methods/ftp.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: ftp.cc,v 1.21 2001/02/20 07:03:18 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the FTP aquire method for APT.
   7
   8    This is a very simple implementation that does not try to optimize
   9    at all. Commands are sent syncronously with the FTP server (as the
  10    rfc recommends, but it is not really necessary..) and no tricks are
  11    done to speed things along.
  12
  13    RFC 2428 describes the IPv6 FTP behavior
  14
  15    ##################################################################### */
  16                                                                         /*}}}*/
  17 // Include Files                                                        /*{{{*/
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/acquire-method.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/md5.h>
  22
  23 #include <sys/stat.h>
  24 #include <sys/time.h>
  25 #include <utime.h>
  26 #include <unistd.h>
  27 #include <signal.h>
  28 #include <stdio.h>
  29 #include <errno.h>
  30 #include <stdarg.h>
  31
  32 // Internet stuff
  33 #include <netinet/in.h>
  34 #include <sys/socket.h>
  35 #include <arpa/inet.h>
  36 #include <netdb.h>
  37
  38 #include "rfc2553emu.h"
  39 #include "connect.h"
  40 #include "ftp.h"
  41                                                                         /*}}}*/
  42
  43 /* This table is for the EPRT and EPSV commands, it maps the OS address
  44    family to the IETF address families */
  45 struct AFMap
  46 {
  47    unsigned long Family;
  48    unsigned long IETFFamily;
  49 };
  50
  51 #ifndef AF_INET6
  52 struct AFMap AFMap[] = {{AF_INET,1},{}};
  53 #else
  54 struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{}};
  55 #endif
  56
  57 unsigned long TimeOut = 120;
  58 URI Proxy;
  59 string FtpMethod::FailFile;
  60 int FtpMethod::FailFd = -1;
  61 time_t FtpMethod::FailTime = 0;
  62
  63 // FTPConn::FTPConn - Constructor                                       /*{{{*/
  64 // ---------------------------------------------------------------------
  65 /* */
  66 FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1),
  67                             DataListenFd(-1), ServerName(Srv)
  68 {
  69    Debug = _config->FindB("Debug::Acquire::Ftp",false);
  70    PasvAddr = 0;
  71 }
  72                                                                         /*}}}*/
  73 // FTPConn::~FTPConn - Destructor                                       /*{{{*/
  74 // ---------------------------------------------------------------------
  75 /* */
  76 FTPConn::~FTPConn()
  77 {
  78    Close();
  79 }
  80                                                                         /*}}}*/
  81 // FTPConn::Close - Close down the connection                           /*{{{*/
  82 // ---------------------------------------------------------------------
  83 /* Just tear down the socket and data socket */
  84 void FTPConn::Close()
  85 {
  86    close(ServerFd);
  87    ServerFd = -1;
  88    close(DataFd);
  89    DataFd = -1;
  90    close(DataListenFd);
  91    DataListenFd = -1;
  92
  93    if (PasvAddr != 0)
  94       freeaddrinfo(PasvAddr);
  95    PasvAddr = 0;
  96 }
  97                                                                         /*}}}*/
  98 // FTPConn::Open - Open a new connection                                /*{{{*/
  99 // ---------------------------------------------------------------------
 100 /* Connect to the server using a non-blocking connection and perform a
 101    login. */
 102 bool FTPConn::Open(pkgAcqMethod *Owner)
 103 {
 104    // Use the already open connection if possible.
 105    if (ServerFd != -1)
 106       return true;
 107
 108    Close();
 109
 110    // Determine the proxy setting
 111    if (getenv("ftp_proxy") == 0)
 112    {
 113       string DefProxy = _config->Find("Acquire::ftp::Proxy");
 114       string SpecificProxy = _config->Find("Acquire::ftp::Proxy::" + ServerName.Host);
 115       if (SpecificProxy.empty() == false)
 116       {
 117          if (SpecificProxy == "DIRECT")
 118             Proxy = "";
 119          else
 120             Proxy = SpecificProxy;
 121       }
 122       else
 123          Proxy = DefProxy;
 124    }
 125    else
 126       Proxy = getenv("ftp_proxy");
 127
 128    // Determine what host and port to use based on the proxy settings
 129    int Port = 0;
 130    string Host;
 131    if (Proxy.empty() == true)
 132    {
 133       if (ServerName.Port != 0)
 134          Port = ServerName.Port;
 135       Host = ServerName.Host;
 136    }
 137    else
 138    {
 139       if (Proxy.Port != 0)
 140          Port = Proxy.Port;
 141       Host = Proxy.Host;
 142    }
 143
 144    /* Connect to the remote server. Since FTP is connection oriented we
 145       want to make sure we get a new server every time we reconnect */
 146    RotateDNS();
 147    if (Connect(Host,Port,"ftp",21,ServerFd,TimeOut,Owner) == false)
 148       return false;
 149
 150    // Get the remote server's address
 151    PeerAddrLen = sizeof(PeerAddr);
 152    if (getpeername(ServerFd,(sockaddr *)&PeerAddr,&PeerAddrLen) != 0)
 153       return _error->Errno("getpeername","Unable to determine the peer name");
 154
 155    // Get the local machine's address
 156    ServerAddrLen = sizeof(ServerAddr);
 157    if (getsockname(ServerFd,(sockaddr *)&ServerAddr,&ServerAddrLen) != 0)
 158       return _error->Errno("getsockname","Unable to determine the local name");
 159
 160    Owner->Status("Logging in");
 161    return Login();
 162 }
 163                                                                         /*}}}*/
 164 // FTPConn::Login - Login to the remote server                          /*{{{*/
 165 // ---------------------------------------------------------------------
 166 /* This performs both normal login and proxy login using a simples script
 167    stored in the config file. */
 168 bool FTPConn::Login()
 169 {
 170    unsigned int Tag;
 171    string Msg;
 172
 173    // Setup the variables needed for authentication
 174    string User = "anonymous";
 175    string Pass = "apt_get_ftp_2.0@debian.linux.user";
 176
 177    // Fill in the user/pass
 178    if (ServerName.User.empty() == false)
 179       User = ServerName.User;
 180    if (ServerName.Password.empty() == false)
 181       Pass = ServerName.Password;
 182
 183    // Perform simple login
 184    if (Proxy.empty() == true)
 185    {
 186       // Read the initial response
 187       if (ReadResp(Tag,Msg) == false)
 188          return false;
 189       if (Tag >= 400)
 190          return _error->Error("Server refused our connection and said: %s",Msg.c_str());
 191
 192       // Send the user
 193       if (WriteMsg(Tag,Msg,"USER %s",User.c_str()) == false)
 194          return false;
 195       if (Tag >= 400)
 196          return _error->Error("USER failed, server said: %s",Msg.c_str());
 197
 198       // Send the Password
 199       if (WriteMsg(Tag,Msg,"PASS %s",Pass.c_str()) == false)
 200          return false;
 201       if (Tag >= 400)
 202          return _error->Error("PASS failed, server said: %s",Msg.c_str());
 203
 204       // Enter passive mode
 205       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 206          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 207       else
 208          TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 209    }
 210    else
 211    {
 212       // Read the initial response
 213       if (ReadResp(Tag,Msg) == false)
 214          return false;
 215       if (Tag >= 400)
 216          return _error->Error("Server refused our connection and said: %s",Msg.c_str());
 217
 218       // Perform proxy script execution
 219       Configuration::Item const *Opts = _config->Tree("Acquire::ftp::ProxyLogin");
 220       if (Opts == 0 || Opts->Child == 0)
 221          return _error->Error("A proxy server was specified but no login "
 222                               "script, Acquire::ftp::ProxyLogin is empty.");
 223       Opts = Opts->Child;
 224
 225       // Iterate over the entire login script
 226       for (; Opts != 0; Opts = Opts->Next)
 227       {
 228          if (Opts->Value.empty() == true)
 229             continue;
 230
 231          // Substitute the variables into the command
 232          char SitePort[20];
 233          if (ServerName.Port != 0)
 234             sprintf(SitePort,"%u",ServerName.Port);
 235          else
 236             strcpy(SitePort,"21");
 237          string Tmp = Opts->Value;
 238          Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
 239          Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
 240          Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
 241          Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
 242          Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort);
 243          Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
 244
 245          // Send the command
 246          if (WriteMsg(Tag,Msg,"%s",Tmp.c_str()) == false)
 247             return false;
 248          if (Tag >= 400)
 249             return _error->Error("Login script command '%s' failed, server said: %s",Tmp.c_str(),Msg.c_str());
 250       }
 251
 252       // Enter passive mode
 253       TryPassive = false;
 254       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 255          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 256       else
 257       {
 258          if (_config->Exists("Acquire::FTP::Proxy::Passive") == true)
 259             TryPassive = _config->FindB("Acquire::FTP::Proxy::Passive",true);
 260          else
 261             TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 262       }
 263    }
 264
 265    // Force the use of extended commands
 266    if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true)
 267       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true);
 268    else
 269       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false);
 270
 271    // Binary mode
 272    if (WriteMsg(Tag,Msg,"TYPE I") == false)
 273       return false;
 274    if (Tag >= 400)
 275       return _error->Error("TYPE failed, server said: %s",Msg.c_str());
 276
 277    return true;
 278 }
 279                                                                         /*}}}*/
 280 // FTPConn::ReadLine - Read a line from the server                      /*{{{*/
 281 // ---------------------------------------------------------------------
 282 /* This performs a very simple buffered read. */
 283 bool FTPConn::ReadLine(string &Text)
 284 {
 285    if (ServerFd == -1)
 286       return false;
 287
 288    // Suck in a line
 289    while (Len < sizeof(Buffer))
 290    {
 291       // Scan the buffer for a new line
 292       for (unsigned int I = 0; I != Len; I++)
 293       {
 294          // Escape some special chars
 295          if (Buffer[I] == 0)
 296             Buffer[I] = '?';
 297
 298          // End of line?
 299          if (Buffer[I] != '\n')
 300             continue;
 301
 302          I++;
 303          Text = string(Buffer,I);
 304          memmove(Buffer,Buffer+I,Len - I);
 305          Len -= I;
 306          return true;
 307       }
 308
 309       // Wait for some data..
 310       if (WaitFd(ServerFd,false,TimeOut) == false)
 311       {
 312          Close();
 313          return _error->Error("Connection timeout");
 314       }
 315
 316       // Suck it back
 317       int Res = read(ServerFd,Buffer + Len,sizeof(Buffer) - Len);
 318       if (Res == 0)
 319          _error->Error("Server closed the connection");
 320       if (Res <= 0)
 321       {
 322          _error->Errno("read","Read error");
 323          Close();
 324          return false;
 325       }
 326       Len += Res;
 327    }
 328
 329    return _error->Error("A response overflowed the buffer.");
 330 }
 331                                                                         /*}}}*/
 332 // FTPConn::ReadResp - Read a full response from the server             /*{{{*/
 333 // ---------------------------------------------------------------------
 334 /* This reads a reply code from the server, it handles both p */
 335 bool FTPConn::ReadResp(unsigned int &Ret,string &Text)
 336 {
 337    // Grab the first line of the response
 338    string Msg;
 339    if (ReadLine(Msg) == false)
 340        return false;
 341
 342    // Get the ID code
 343    char *End;
 344    Ret = strtol(Msg.c_str(),&End,10);
 345    if (End - Msg.c_str() != 3)
 346       return _error->Error("Protocol corruption");
 347
 348    // All done ?
 349    Text = Msg.c_str()+4;
 350    if (*End == ' ')
 351    {
 352       if (Debug == true)
 353          cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 354       return true;
 355    }
 356
 357    if (*End != '-')
 358       return _error->Error("Protocol corruption");
 359
 360    /* Okay, here we do the continued message trick. This is foolish, but
 361       proftpd follows the protocol as specified and wu-ftpd doesn't, so
 362       we filter. I wonder how many clients break if you use proftpd and
 363       put a '- in the 3rd spot in the message? */
 364    char Leader[4];
 365    strncpy(Leader,Msg.c_str(),3);
 366    Leader[3] = 0;
 367    while (ReadLine(Msg) == true)
 368    {
 369       // Short, it must be using RFC continuation..
 370       if (Msg.length() < 4)
 371       {
 372          Text += Msg;
 373          continue;
 374       }
 375
 376       // Oops, finished
 377       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == ' ')
 378       {
 379          Text += Msg.c_str()+4;
 380          break;
 381       }
 382
 383       // This message has the wu-ftpd style reply code prefixed
 384       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == '-')
 385       {
 386          Text += Msg.c_str()+4;
 387          continue;
 388       }
 389
 390       // Must be RFC style prefixing
 391       Text += Msg;
 392    }
 393
 394    if (Debug == true && _error->PendingError() == false)
 395       cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 396
 397    return !_error->PendingError();
 398 }
 399                                                                         /*}}}*/
 400 // FTPConn::WriteMsg - Send a message to the server                     /*{{{*/
 401 // ---------------------------------------------------------------------
 402 /* Simple printf like function.. */
 403 bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...)
 404 {
 405    va_list args;
 406    va_start(args,Fmt);
 407
 408    // sprintf the description
 409    char S[400];
 410    vsnprintf(S,sizeof(S) - 4,Fmt,args);
 411    strcat(S,"\r\n");
 412
 413    if (Debug == true)
 414       cerr << "-> '" << QuoteString(S,"") << "'" << endl;
 415
 416    // Send it off
 417    unsigned long Len = strlen(S);
 418    unsigned long Start = 0;
 419    while (Len != 0)
 420    {
 421       if (WaitFd(ServerFd,true,TimeOut) == false)
 422       {
 423          Close();
 424          return _error->Error("Connection timeout");
 425       }
 426
 427       int Res = write(ServerFd,S + Start,Len);
 428       if (Res <= 0)
 429       {
 430          _error->Errno("write","Write Error");
 431          Close();
 432          return false;
 433       }
 434
 435       Len -= Res;
 436       Start += Res;
 437    }
 438
 439    return ReadResp(Ret,Text);
 440 }
 441                                                                         /*}}}*/
 442 // FTPConn::GoPasv - Enter Passive mode                                 /*{{{*/
 443 // ---------------------------------------------------------------------
 444 /* Try to enter passive mode, the return code does not indicate if passive
 445    mode could or could not be established, only if there was a fatal error.
 446    We have to enter passive mode every time we make a data connection :| */
 447 bool FTPConn::GoPasv()
 448 {
 449    /* The PASV command only works on IPv4 sockets, even though it could
 450       in theory suppory IPv6 via an all zeros reply */
 451    if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET ||
 452        ForceExtended == true)
 453       return ExtGoPasv();
 454
 455    if (PasvAddr != 0)
 456       freeaddrinfo(PasvAddr);
 457    PasvAddr = 0;
 458
 459    // Try to enable pasv mode
 460    unsigned int Tag;
 461    string Msg;
 462    if (WriteMsg(Tag,Msg,"PASV") == false)
 463       return false;
 464
 465    // Unsupported function
 466    string::size_type Pos = Msg.find('(');
 467    if (Tag >= 400 || Pos == string::npos)
 468       return true;
 469
 470    // Scan it
 471    unsigned a0,a1,a2,a3,p0,p1;
 472    if (sscanf(Msg.c_str() + Pos,"(%u,%u,%u,%u,%u,%u)",&a0,&a1,&a2,&a3,&p0,&p1) != 6)
 473       return true;
 474
 475    /* Some evil servers return 0 to mean their addr. We can actually speak
 476       to these servers natively using IPv6 */
 477    if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0)
 478    {
 479       // Get the IP in text form
 480       char Name[NI_MAXHOST];
 481       char Service[NI_MAXSERV];
 482       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 483                   Name,sizeof(Name),Service,sizeof(Service),
 484                   NI_NUMERICHOST|NI_NUMERICSERV);
 485
 486       struct addrinfo Hints;
 487       memset(&Hints,0,sizeof(Hints));
 488       Hints.ai_socktype = SOCK_STREAM;
 489       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 490       Hints.ai_flags |= AI_NUMERICHOST;
 491
 492       // Get a new passive address.
 493       char Port[100];
 494       snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 495       if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 496          return true;
 497       return true;
 498    }
 499
 500    struct addrinfo Hints;
 501    memset(&Hints,0,sizeof(Hints));
 502    Hints.ai_socktype = SOCK_STREAM;
 503    Hints.ai_family = AF_INET;
 504    Hints.ai_flags |= AI_NUMERICHOST;
 505
 506    // Get a new passive address.
 507    char Port[100];
 508    snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 509    char Name[100];
 510    snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3);
 511    if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 512       return true;
 513    return true;
 514 }
 515                                                                         /*}}}*/
 516 // FTPConn::ExtGoPasv - Enter Extended Passive mode                     /*{{{*/
 517 // ---------------------------------------------------------------------
 518 /* Try to enter extended passive mode. See GoPasv above and RFC 2428 */
 519 bool FTPConn::ExtGoPasv()
 520 {
 521    if (PasvAddr != 0)
 522       freeaddrinfo(PasvAddr);
 523    PasvAddr = 0;
 524
 525    // Try to enable pasv mode
 526    unsigned int Tag;
 527    string Msg;
 528    if (WriteMsg(Tag,Msg,"EPSV") == false)
 529       return false;
 530
 531    // Unsupported function
 532    string::size_type Pos = Msg.find('(');
 533    if (Tag >= 400 || Pos == string::npos)
 534       return true;
 535
 536    // Scan it
 537    string::const_iterator List[4];
 538    unsigned Count = 0;
 539    Pos++;
 540    for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); I++)
 541    {
 542       if (*I != Msg[Pos])
 543          continue;
 544       if (Count >= 4)
 545          return true;
 546       List[Count++] = I;
 547    }
 548    if (Count != 4)
 549       return true;
 550
 551    // Break it up ..
 552    unsigned long Proto = 0;
 553    unsigned long Port = 0;
 554    string IP;
 555    IP = string(List[1]+1,List[2]);
 556    Port = atoi(string(List[2]+1,List[3]).c_str());
 557    if (IP.empty() == false)
 558       Proto = atoi(string(List[0]+1,List[1]).c_str());
 559
 560    if (Port == 0)
 561       return false;
 562
 563    // String version of the port
 564    char PStr[100];
 565    snprintf(PStr,sizeof(PStr),"%lu",Port);
 566
 567    // Get the IP in text form
 568    struct addrinfo Hints;
 569    memset(&Hints,0,sizeof(Hints));
 570    Hints.ai_socktype = SOCK_STREAM;
 571    Hints.ai_flags |= AI_NUMERICHOST;
 572
 573    /* The RFC defined case, connect to the old IP/protocol using the
 574       new port. */
 575    if (IP.empty() == true)
 576    {
 577       // Get the IP in text form
 578       char Name[NI_MAXHOST];
 579       char Service[NI_MAXSERV];
 580       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 581                   Name,sizeof(Name),Service,sizeof(Service),
 582                   NI_NUMERICHOST|NI_NUMERICSERV);
 583       IP = Name;
 584       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 585    }
 586    else
 587    {
 588       // Get the family..
 589       Hints.ai_family = 0;
 590       for (unsigned J = 0; AFMap[J].Family != 0; J++)
 591          if (AFMap[J].IETFFamily == Proto)
 592             Hints.ai_family = AFMap[J].Family;
 593       if (Hints.ai_family == 0)
 594          return true;
 595    }
 596
 597    // Get a new passive address.
 598    int Res;
 599    if ((Res = getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr)) != 0)
 600       return true;
 601
 602    return true;
 603 }
 604                                                                         /*}}}*/
 605 // FTPConn::Size - Return the size of a file                            /*{{{*/
 606 // ---------------------------------------------------------------------
 607 /* Grab the file size from the server, 0 means no size or empty file */
 608 bool FTPConn::Size(const char *Path,unsigned long &Size)
 609 {
 610    // Query the size
 611    unsigned int Tag;
 612    string Msg;
 613    Size = 0;
 614    if (WriteMsg(Tag,Msg,"SIZE %s",Path) == false)
 615       return false;
 616
 617    char *End;
 618    Size = strtol(Msg.c_str(),&End,10);
 619    if (Tag >= 400 || End == Msg.c_str())
 620       Size = 0;
 621    return true;
 622 }
 623                                                                         /*}}}*/
 624 // FTPConn::ModTime - Return the modification time of the file          /*{{{*/
 625 // ---------------------------------------------------------------------
 626 /* Like Size no error is returned if the command is not supported. If the
 627    command fails then time is set to the current time of day to fool
 628    date checks. */
 629 bool FTPConn::ModTime(const char *Path, time_t &Time)
 630 {
 631    Time = time(&Time);
 632
 633    // Query the mod time
 634    unsigned int Tag;
 635    string Msg;
 636    if (WriteMsg(Tag,Msg,"MDTM %s",Path) == false)
 637       return false;
 638    if (Tag >= 400 || Msg.empty() == true || isdigit(Msg[0]) == 0)
 639       return true;
 640
 641    // Parse it
 642    StrToTime(Msg,Time);
 643    return true;
 644 }
 645                                                                         /*}}}*/
 646 // FTPConn::CreateDataFd - Get a data connection                        /*{{{*/
 647 // ---------------------------------------------------------------------
 648 /* Create the data connection. Call FinalizeDataFd after this though.. */
 649 bool FTPConn::CreateDataFd()
 650 {
 651    close(DataFd);
 652    DataFd = -1;
 653
 654    // Attempt to enter passive mode.
 655    if (TryPassive == true)
 656    {
 657       if (GoPasv() == false)
 658          return false;
 659
 660       // Oops, didn't work out, don't bother trying again.
 661       if (PasvAddr == 0)
 662          TryPassive = false;
 663    }
 664
 665    // Passive mode?
 666    if (PasvAddr != 0)
 667    {
 668       // Get a socket
 669       if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype,
 670                            PasvAddr->ai_protocol)) < 0)
 671          return _error->Errno("socket","Could not create a socket");
 672
 673       // Connect to the server
 674       SetNonBlock(DataFd,true);
 675       if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 &&
 676           errno != EINPROGRESS)
 677          return _error->Errno("socket","Could not create a socket");
 678
 679       /* This implements a timeout for connect by opening the connection
 680          nonblocking */
 681       if (WaitFd(DataFd,true,TimeOut) == false)
 682          return _error->Error("Could not connect data socket, connection timed out");
 683       unsigned int Err;
 684       unsigned int Len = sizeof(Err);
 685       if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 686          return _error->Errno("getsockopt","Failed");
 687       if (Err != 0)
 688          return _error->Error("Could not connect passive socket.");
 689
 690       return true;
 691    }
 692
 693    // Port mode :<
 694    close(DataListenFd);
 695    DataListenFd = -1;
 696
 697    // Get the information for a listening socket.
 698    struct addrinfo *BindAddr = 0;
 699    struct addrinfo Hints;
 700    memset(&Hints,0,sizeof(Hints));
 701    Hints.ai_socktype = SOCK_STREAM;
 702    Hints.ai_flags |= AI_PASSIVE;
 703    Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family;
 704    int Res;
 705    if ((Res = getaddrinfo(0,"0",&Hints,&BindAddr)) != 0)
 706       return _error->Error("getaddrinfo was unable to get a listening socket");
 707
 708    // Construct the socket
 709    if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype,
 710                               BindAddr->ai_protocol)) < 0)
 711    {
 712       freeaddrinfo(BindAddr);
 713       return _error->Errno("socket","Could not create a socket");
 714    }
 715
 716    // Bind and listen
 717    if (bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0)
 718    {
 719       freeaddrinfo(BindAddr);
 720       return _error->Errno("bind","Could not bind a socket");
 721    }
 722    freeaddrinfo(BindAddr);
 723    if (listen(DataListenFd,1) < 0)
 724       return _error->Errno("listen","Could not listen on the socket");
 725    SetNonBlock(DataListenFd,true);
 726
 727    // Determine the name to send to the remote
 728    struct sockaddr_storage Addr;
 729    socklen_t AddrLen = sizeof(Addr);
 730    if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0)
 731       return _error->Errno("getsockname","Could not determine the socket's name");
 732
 733    // Reverse the address. We need the server address and the data port.
 734    char Name[NI_MAXHOST];
 735    char Service[NI_MAXSERV];
 736    char Service2[NI_MAXSERV];
 737    getnameinfo((struct sockaddr *)&Addr,AddrLen,
 738                Name,sizeof(Name),Service,sizeof(Service),
 739                NI_NUMERICHOST|NI_NUMERICSERV);
 740    getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen,
 741                Name,sizeof(Name),Service2,sizeof(Service2),
 742                NI_NUMERICHOST|NI_NUMERICSERV);
 743
 744    // Send off an IPv4 address in the old port format
 745    if (((struct sockaddr *)&Addr)->sa_family == AF_INET &&
 746        ForceExtended == false)
 747    {
 748       // Convert the dots in the quad into commas
 749       for (char *I = Name; *I != 0; I++)
 750          if (*I == '.')
 751             *I = ',';
 752       unsigned long Port = atoi(Service);
 753
 754       // Send the port command
 755       unsigned int Tag;
 756       string Msg;
 757       if (WriteMsg(Tag,Msg,"PORT %s,%d,%d",
 758                    Name,
 759                    (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false)
 760          return false;
 761       if (Tag >= 400)
 762          return _error->Error("Unable to send PORT command");
 763       return true;
 764    }
 765
 766    // Construct an EPRT command
 767    unsigned Proto = 0;
 768    for (unsigned J = 0; AFMap[J].Family != 0; J++)
 769       if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family)
 770          Proto = AFMap[J].IETFFamily;
 771    if (Proto == 0)
 772       return _error->Error("Unkonwn address family %u (AF_*)",
 773                            ((struct sockaddr *)&Addr)->sa_family);
 774
 775    // Send the EPRT command
 776    unsigned int Tag;
 777    string Msg;
 778    if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false)
 779       return false;
 780    if (Tag >= 400)
 781       return _error->Error("EPRT failed, server said: %s",Msg.c_str());
 782    return true;
 783 }
 784                                                                         /*}}}*/
 785 // FTPConn::Finalize - Complete the Data connection                     /*{{{*/
 786 // ---------------------------------------------------------------------
 787 /* If the connection is in port mode this waits for the other end to hook
 788    up to us. */
 789 bool FTPConn::Finalize()
 790 {
 791    // Passive mode? Do nothing
 792    if (PasvAddr != 0)
 793       return true;
 794
 795    // Close any old socket..
 796    close(DataFd);
 797    DataFd = -1;
 798
 799    // Wait for someone to connect..
 800    if (WaitFd(DataListenFd,false,TimeOut) == false)
 801       return _error->Error("Data socket connect timed out");
 802
 803    // Accept the connection
 804    struct sockaddr_in Addr;
 805    socklen_t Len = sizeof(Addr);
 806    DataFd = accept(DataListenFd,(struct sockaddr *)&Addr,&Len);
 807    if (DataFd < 0)
 808       return _error->Errno("accept","Unable to accept connection");
 809
 810    close(DataListenFd);
 811    DataListenFd = -1;
 812
 813    return true;
 814 }
 815                                                                         /*}}}*/
 816 // FTPConn::Get - Get a file                                            /*{{{*/
 817 // ---------------------------------------------------------------------
 818 /* This opens a data connection, sends REST and RETR and then
 819    transfers the file over. */
 820 bool FTPConn::Get(const char *Path,FileFd &To,unsigned long Resume,
 821                   MD5Summation &MD5,bool &Missing)
 822 {
 823    Missing = false;
 824    if (CreateDataFd() == false)
 825       return false;
 826
 827    unsigned int Tag;
 828    string Msg;
 829    if (Resume != 0)
 830    {
 831       if (WriteMsg(Tag,Msg,"REST %u",Resume) == false)
 832          return false;
 833       if (Tag >= 400)
 834          Resume = 0;
 835    }
 836
 837    if (To.Truncate(Resume) == false)
 838       return false;
 839
 840    if (To.Seek(0) == false)
 841       return false;
 842
 843    if (Resume != 0)
 844    {
 845       if (MD5.AddFD(To.Fd(),Resume) == false)
 846       {
 847          _error->Errno("read","Problem hashing file");
 848          return false;
 849       }
 850    }
 851
 852    // Send the get command
 853    if (WriteMsg(Tag,Msg,"RETR %s",Path) == false)
 854       return false;
 855
 856    if (Tag >= 400)
 857    {
 858       if (Tag == 550)
 859          Missing = true;
 860       return _error->Error("Unable to fetch file, server said '%s'",Msg.c_str());
 861    }
 862
 863    // Finish off the data connection
 864    if (Finalize() == false)
 865       return false;
 866
 867    // Copy loop
 868    unsigned char Buffer[4096];
 869    while (1)
 870    {
 871       // Wait for some data..
 872       if (WaitFd(DataFd,false,TimeOut) == false)
 873       {
 874          Close();
 875          return _error->Error("Data socket timed out");
 876       }
 877
 878       // Read the data..
 879       int Res = read(DataFd,Buffer,sizeof(Buffer));
 880       if (Res == 0)
 881          break;
 882       if (Res < 0)
 883       {
 884          if (errno == EAGAIN)
 885             continue;
 886          break;
 887       }
 888
 889       MD5.Add(Buffer,Res);
 890       if (To.Write(Buffer,Res) == false)
 891       {
 892          Close();
 893          return false;
 894       }
 895    }
 896
 897    // All done
 898    close(DataFd);
 899    DataFd = -1;
 900
 901    // Read the closing message from the server
 902    if (ReadResp(Tag,Msg) == false)
 903       return false;
 904    if (Tag >= 400)
 905       return _error->Error("Data transfer failed, server said '%s'",Msg.c_str());
 906    return true;
 907 }
 908                                                                         /*}}}*/
 909
 910 // FtpMethod::FtpMethod - Constructor                                   /*{{{*/
 911 // ---------------------------------------------------------------------
 912 /* */
 913 FtpMethod::FtpMethod() : pkgAcqMethod("1.0",SendConfig)
 914 {
 915    signal(SIGTERM,SigTerm);
 916    signal(SIGINT,SigTerm);
 917
 918    Server = 0;
 919    FailFd = -1;
 920 }
 921                                                                         /*}}}*/
 922 // FtpMethod::SigTerm - Handle a fatal signal                           /*{{{*/
 923 // ---------------------------------------------------------------------
 924 /* This closes and timestamps the open file. This is neccessary to get
 925    resume behavoir on user abort */
 926 void FtpMethod::SigTerm(int)
 927 {
 928    if (FailFd == -1)
 929       _exit(100);
 930    close(FailFd);
 931
 932    // Timestamp
 933    struct utimbuf UBuf;
 934    UBuf.actime = FailTime;
 935    UBuf.modtime = FailTime;
 936    utime(FailFile.c_str(),&UBuf);
 937
 938    _exit(100);
 939 }
 940                                                                         /*}}}*/
 941 // FtpMethod::Configuration - Handle a configuration message            /*{{{*/
 942 // ---------------------------------------------------------------------
 943 /* We stash the desired pipeline depth */
 944 bool FtpMethod::Configuration(string Message)
 945 {
 946    if (pkgAcqMethod::Configuration(Message) == false)
 947       return false;
 948
 949    TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut);
 950    return true;
 951 }
 952                                                                         /*}}}*/
 953 // FtpMethod::Fetch - Fetch a file                                      /*{{{*/
 954 // ---------------------------------------------------------------------
 955 /* Fetch a single file, called by the base class..  */
 956 bool FtpMethod::Fetch(FetchItem *Itm)
 957 {
 958    URI Get = Itm->Uri;
 959    const char *File = Get.Path.c_str();
 960    FetchResult Res;
 961    Res.Filename = Itm->DestFile;
 962    Res.IMSHit = false;
 963
 964    // Connect to the server
 965    if (Server == 0 || Server->Comp(Get) == false)
 966    {
 967       delete Server;
 968       Server = new FTPConn(Get);
 969    }
 970
 971    // Could not connect is a transient error..
 972    if (Server->Open(this) == false)
 973    {
 974       Server->Close();
 975       Fail(true);
 976       return true;
 977    }
 978
 979    // Get the files information
 980    Status("Query");
 981    unsigned long Size;
 982    if (Server->Size(File,Size) == false ||
 983        Server->ModTime(File,FailTime) == false)
 984    {
 985       Fail(true);
 986       return true;
 987    }
 988    Res.Size = Size;
 989
 990    // See if it is an IMS hit
 991    if (Itm->LastModified == FailTime)
 992    {
 993       Res.Size = 0;
 994       Res.IMSHit = true;
 995       URIDone(Res);
 996       return true;
 997    }
 998
 999    // See if the file exists
1000    struct stat Buf;
1001    if (stat(Itm->DestFile.c_str(),&Buf) == 0)
1002    {
1003       if (Size == (unsigned)Buf.st_size && FailTime == Buf.st_mtime)
1004       {
1005          Res.Size = Buf.st_size;
1006          Res.LastModified = Buf.st_mtime;
1007          Res.ResumePoint = Buf.st_size;
1008          URIDone(Res);
1009          return true;
1010       }
1011
1012       // Resume?
1013       if (FailTime == Buf.st_mtime && Size > (unsigned)Buf.st_size)
1014          Res.ResumePoint = Buf.st_size;
1015    }
1016
1017    // Open the file
1018    MD5Summation MD5;
1019    {
1020       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
1021       if (_error->PendingError() == true)
1022          return false;
1023
1024       URIStart(Res);
1025
1026       FailFile = Itm->DestFile;
1027       FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1028       FailFd = Fd.Fd();
1029
1030       bool Missing;
1031       if (Server->Get(File,Fd,Res.ResumePoint,MD5,Missing) == false)
1032       {
1033          Fd.Close();
1034
1035          // Timestamp
1036          struct utimbuf UBuf;
1037          UBuf.actime = FailTime;
1038          UBuf.modtime = FailTime;
1039          utime(FailFile.c_str(),&UBuf);
1040
1041          // If the file is missing we hard fail otherwise transient fail
1042          if (Missing == true)
1043             return false;
1044          Fail(true);
1045          return true;
1046       }
1047
1048       Res.Size = Fd.Size();
1049    }
1050
1051    Res.LastModified = FailTime;
1052    Res.MD5Sum = MD5.Result();
1053
1054    // Timestamp
1055    struct utimbuf UBuf;
1056    UBuf.actime = FailTime;
1057    UBuf.modtime = FailTime;
1058    utime(Queue->DestFile.c_str(),&UBuf);
1059    FailFd = -1;
1060
1061    URIDone(Res);
1062
1063    return true;
1064 }
1065                                                                         /*}}}*/
1066
1067 int main(int argc,const char *argv[])
1068 {
1069    /* See if we should be come the http client - we do this for http
1070       proxy urls */
1071    if (getenv("ftp_proxy") != 0)
1072    {
1073       URI Proxy = string(getenv("ftp_proxy"));
1074       if (Proxy.Access == "http")
1075       {
1076          // Copy over the environment setting
1077          char S[300];
1078          snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy"));
1079          putenv(S);
1080
1081          // Run the http method
1082          string Path = flNotFile(argv[0]) + "/http";
1083          execl(Path.c_str(),Path.c_str(),0);
1084          cerr << "Unable to invoke " << Path << endl;
1085          exit(100);
1086       }
1087    }
1088
1089    FtpMethod Mth;
1090
1091    return Mth.Run();
1092 }