methods/ftp.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: ftp.cc,v 1.31.2.1 2004/01/16 18:58:50 mdz Exp $
   4 /* ######################################################################
   5
   6    FTP Acquire Method - This is the FTP acquire method for APT.
   7
   8    This is a very simple implementation that does not try to optimize
   9    at all. Commands are sent syncronously with the FTP server (as the
  10    rfc recommends, but it is not really necessary..) and no tricks are
  11    done to speed things along.
  12
  13    RFC 2428 describes the IPv6 FTP behavior
  14
  15    ##################################################################### */
  16                                                                         /*}}}*/
  17 // Include Files                                                        /*{{{*/
  18 #include <config.h>
  19
  20 #include <apt-pkg/fileutl.h>
  21 #include <apt-pkg/acquire-method.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/hashes.h>
  24 #include <apt-pkg/netrc.h>
  25 #include <apt-pkg/configuration.h>
  26 #include <apt-pkg/strutl.h>
  27
  28 #include <ctype.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <sys/stat.h>
  32 #include <sys/time.h>
  33 #include <unistd.h>
  34 #include <signal.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdarg.h>
  38 #include <iostream>
  39
  40 // Internet stuff
  41 #include <netinet/in.h>
  42 #include <sys/socket.h>
  43 #include <arpa/inet.h>
  44 #include <netdb.h>
  45
  46 #include "rfc2553emu.h"
  47 #include "connect.h"
  48 #include "ftp.h"
  49
  50 #include <apti18n.h>
  51                                                                         /*}}}*/
  52
  53 using namespace std;
  54
  55 /* This table is for the EPRT and EPSV commands, it maps the OS address
  56    family to the IETF address families */
  57 struct AFMap
  58 {
  59    unsigned long Family;
  60    unsigned long IETFFamily;
  61 };
  62
  63 #ifndef AF_INET6
  64 struct AFMap AFMap[] = {{AF_INET,1},{0, 0}};
  65 #else
  66 struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{0, 0}};
  67 #endif
  68
  69 unsigned long TimeOut = 120;
  70 URI Proxy;
  71 string FtpMethod::FailFile;
  72 int FtpMethod::FailFd = -1;
  73 time_t FtpMethod::FailTime = 0;
  74
  75 // FTPConn::FTPConn - Constructor                                       /*{{{*/
  76 // ---------------------------------------------------------------------
  77 /* */
  78 FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1),
  79                             DataListenFd(-1), ServerName(Srv),
  80                             ForceExtended(false), TryPassive(true)
  81 {
  82    Debug = _config->FindB("Debug::Acquire::Ftp",false);
  83    PasvAddr = 0;
  84    Buffer[0] = '\0';
  85 }
  86                                                                         /*}}}*/
  87 // FTPConn::~FTPConn - Destructor                                       /*{{{*/
  88 // ---------------------------------------------------------------------
  89 /* */
  90 FTPConn::~FTPConn()
  91 {
  92    Close();
  93 }
  94                                                                         /*}}}*/
  95 // FTPConn::Close - Close down the connection                           /*{{{*/
  96 // ---------------------------------------------------------------------
  97 /* Just tear down the socket and data socket */
  98 void FTPConn::Close()
  99 {
 100    close(ServerFd);
 101    ServerFd = -1;
 102    close(DataFd);
 103    DataFd = -1;
 104    close(DataListenFd);
 105    DataListenFd = -1;
 106
 107    if (PasvAddr != 0)
 108       freeaddrinfo(PasvAddr);
 109    PasvAddr = 0;
 110 }
 111                                                                         /*}}}*/
 112 // FTPConn::Open - Open a new connection                                /*{{{*/
 113 // ---------------------------------------------------------------------
 114 /* Connect to the server using a non-blocking connection and perform a
 115    login. */
 116 bool FTPConn::Open(pkgAcqMethod *Owner)
 117 {
 118    // Use the already open connection if possible.
 119    if (ServerFd != -1)
 120       return true;
 121
 122    Close();
 123
 124    // Determine the proxy setting
 125    string SpecificProxy = _config->Find("Acquire::ftp::Proxy::" + ServerName.Host);
 126    if (!SpecificProxy.empty())
 127    {
 128            if (SpecificProxy == "DIRECT")
 129                    Proxy = "";
 130            else
 131                    Proxy = SpecificProxy;
 132    }
 133    else
 134    {
 135            string DefProxy = _config->Find("Acquire::ftp::Proxy");
 136            if (!DefProxy.empty())
 137            {
 138                    Proxy = DefProxy;
 139            }
 140            else
 141            {
 142                    char* result = getenv("ftp_proxy");
 143                    Proxy = result ? result : "";
 144            }
 145    }
 146
 147    // Parse no_proxy, a , separated list of domains
 148    if (getenv("no_proxy") != 0)
 149    {
 150       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 151          Proxy = "";
 152    }
 153
 154    // Determine what host and port to use based on the proxy settings
 155    int Port = 0;
 156    string Host;
 157    if (Proxy.empty() == true)
 158    {
 159       if (ServerName.Port != 0)
 160          Port = ServerName.Port;
 161       Host = ServerName.Host;
 162    }
 163    else
 164    {
 165       if (Proxy.Port != 0)
 166          Port = Proxy.Port;
 167       Host = Proxy.Host;
 168    }
 169
 170    /* Connect to the remote server. Since FTP is connection oriented we
 171       want to make sure we get a new server every time we reconnect */
 172    RotateDNS();
 173    if (Connect(Host,Port,"ftp",21,ServerFd,TimeOut,Owner) == false)
 174       return false;
 175
 176    // Login must be before getpeername otherwise dante won't work.
 177    Owner->Status(_("Logging in"));
 178    bool Res = Login();
 179
 180    // Get the remote server's address
 181    PeerAddrLen = sizeof(PeerAddr);
 182    if (getpeername(ServerFd,(sockaddr *)&PeerAddr,&PeerAddrLen) != 0)
 183       return _error->Errno("getpeername",_("Unable to determine the peer name"));
 184
 185    // Get the local machine's address
 186    ServerAddrLen = sizeof(ServerAddr);
 187    if (getsockname(ServerFd,(sockaddr *)&ServerAddr,&ServerAddrLen) != 0)
 188       return _error->Errno("getsockname",_("Unable to determine the local name"));
 189
 190    return Res;
 191 }
 192                                                                         /*}}}*/
 193 // FTPConn::Login - Login to the remote server                          /*{{{*/
 194 // ---------------------------------------------------------------------
 195 /* This performs both normal login and proxy login using a simples script
 196    stored in the config file. */
 197 bool FTPConn::Login()
 198 {
 199    unsigned int Tag;
 200    string Msg;
 201
 202    // Setup the variables needed for authentication
 203    string User = "anonymous";
 204    string Pass = "apt_get_ftp_2.1@debian.linux.user";
 205
 206    // Fill in the user/pass
 207    if (ServerName.User.empty() == false)
 208       User = ServerName.User;
 209    if (ServerName.Password.empty() == false)
 210       Pass = ServerName.Password;
 211
 212    // Perform simple login
 213    if (Proxy.empty() == true)
 214    {
 215       // Read the initial response
 216       if (ReadResp(Tag,Msg) == false)
 217          return false;
 218       if (Tag >= 400)
 219          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 220
 221       // Send the user
 222       if (WriteMsg(Tag,Msg,"USER %s",User.c_str()) == false)
 223          return false;
 224       if (Tag >= 400)
 225          return _error->Error(_("USER failed, server said: %s"),Msg.c_str());
 226
 227       if (Tag == 331) { // 331 User name okay, need password.
 228          // Send the Password
 229          if (WriteMsg(Tag,Msg,"PASS %s",Pass.c_str()) == false)
 230             return false;
 231          if (Tag >= 400)
 232             return _error->Error(_("PASS failed, server said: %s"),Msg.c_str());
 233       }
 234
 235       // Enter passive mode
 236       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 237          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 238       else
 239          TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 240    }
 241    else
 242    {
 243       // Read the initial response
 244       if (ReadResp(Tag,Msg) == false)
 245          return false;
 246       if (Tag >= 400)
 247          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 248
 249       // Perform proxy script execution
 250       Configuration::Item const *Opts = _config->Tree("Acquire::ftp::ProxyLogin");
 251       if (Opts == 0 || Opts->Child == 0)
 252          return _error->Error(_("A proxy server was specified but no login "
 253                               "script, Acquire::ftp::ProxyLogin is empty."));
 254       Opts = Opts->Child;
 255
 256       // Iterate over the entire login script
 257       for (; Opts != 0; Opts = Opts->Next)
 258       {
 259          if (Opts->Value.empty() == true)
 260             continue;
 261
 262          // Substitute the variables into the command
 263          char SitePort[20];
 264          if (ServerName.Port != 0)
 265             sprintf(SitePort,"%u",ServerName.Port);
 266          else
 267             strcpy(SitePort,"21");
 268          string Tmp = Opts->Value;
 269          Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
 270          Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
 271          Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
 272          Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
 273          Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort);
 274          Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
 275
 276          // Send the command
 277          if (WriteMsg(Tag,Msg,"%s",Tmp.c_str()) == false)
 278             return false;
 279          if (Tag >= 400)
 280             return _error->Error(_("Login script command '%s' failed, server said: %s"),Tmp.c_str(),Msg.c_str());
 281       }
 282
 283       // Enter passive mode
 284       TryPassive = false;
 285       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 286          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 287       else
 288       {
 289          if (_config->Exists("Acquire::FTP::Proxy::Passive") == true)
 290             TryPassive = _config->FindB("Acquire::FTP::Proxy::Passive",true);
 291          else
 292             TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 293       }
 294    }
 295
 296    // Force the use of extended commands
 297    if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true)
 298       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true);
 299    else
 300       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false);
 301
 302    // Binary mode
 303    if (WriteMsg(Tag,Msg,"TYPE I") == false)
 304       return false;
 305    if (Tag >= 400)
 306       return _error->Error(_("TYPE failed, server said: %s"),Msg.c_str());
 307
 308    return true;
 309 }
 310                                                                         /*}}}*/
 311 // FTPConn::ReadLine - Read a line from the server                      /*{{{*/
 312 // ---------------------------------------------------------------------
 313 /* This performs a very simple buffered read. */
 314 bool FTPConn::ReadLine(string &Text)
 315 {
 316    if (ServerFd == -1)
 317       return false;
 318
 319    // Suck in a line
 320    while (Len < sizeof(Buffer))
 321    {
 322       // Scan the buffer for a new line
 323       for (unsigned int I = 0; I != Len; I++)
 324       {
 325          // Escape some special chars
 326          if (Buffer[I] == 0)
 327             Buffer[I] = '?';
 328
 329          // End of line?
 330          if (Buffer[I] != '\n')
 331             continue;
 332
 333          I++;
 334          Text = string(Buffer,I);
 335          memmove(Buffer,Buffer+I,Len - I);
 336          Len -= I;
 337          return true;
 338       }
 339
 340       // Wait for some data..
 341       if (WaitFd(ServerFd,false,TimeOut) == false)
 342       {
 343          Close();
 344          return _error->Error(_("Connection timeout"));
 345       }
 346
 347       // Suck it back
 348       int Res = read(ServerFd,Buffer + Len,sizeof(Buffer) - Len);
 349       if (Res == 0)
 350          _error->Error(_("Server closed the connection"));
 351       if (Res <= 0)
 352       {
 353          _error->Errno("read",_("Read error"));
 354          Close();
 355          return false;
 356       }
 357       Len += Res;
 358    }
 359
 360    return _error->Error(_("A response overflowed the buffer."));
 361 }
 362                                                                         /*}}}*/
 363 // FTPConn::ReadResp - Read a full response from the server             /*{{{*/
 364 // ---------------------------------------------------------------------
 365 /* This reads a reply code from the server, it handles both p */
 366 bool FTPConn::ReadResp(unsigned int &Ret,string &Text)
 367 {
 368    // Grab the first line of the response
 369    string Msg;
 370    if (ReadLine(Msg) == false)
 371        return false;
 372
 373    // Get the ID code
 374    char *End;
 375    Ret = strtol(Msg.c_str(),&End,10);
 376    if (End - Msg.c_str() != 3)
 377       return _error->Error(_("Protocol corruption"));
 378
 379    // All done ?
 380    Text = Msg.c_str()+4;
 381    if (*End == ' ')
 382    {
 383       if (Debug == true)
 384          cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 385       return true;
 386    }
 387
 388    if (*End != '-')
 389       return _error->Error(_("Protocol corruption"));
 390
 391    /* Okay, here we do the continued message trick. This is foolish, but
 392       proftpd follows the protocol as specified and wu-ftpd doesn't, so
 393       we filter. I wonder how many clients break if you use proftpd and
 394       put a '- in the 3rd spot in the message? */
 395    char Leader[4];
 396    strncpy(Leader,Msg.c_str(),3);
 397    Leader[3] = 0;
 398    while (ReadLine(Msg) == true)
 399    {
 400       // Short, it must be using RFC continuation..
 401       if (Msg.length() < 4)
 402       {
 403          Text += Msg;
 404          continue;
 405       }
 406
 407       // Oops, finished
 408       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == ' ')
 409       {
 410          Text += Msg.c_str()+4;
 411          break;
 412       }
 413
 414       // This message has the wu-ftpd style reply code prefixed
 415       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == '-')
 416       {
 417          Text += Msg.c_str()+4;
 418          continue;
 419       }
 420
 421       // Must be RFC style prefixing
 422       Text += Msg;
 423    }
 424
 425    if (Debug == true && _error->PendingError() == false)
 426       cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 427
 428    return !_error->PendingError();
 429 }
 430                                                                         /*}}}*/
 431 // FTPConn::WriteMsg - Send a message to the server                     /*{{{*/
 432 // ---------------------------------------------------------------------
 433 /* Simple printf like function.. */
 434 bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...)
 435 {
 436    va_list args;
 437    va_start(args,Fmt);
 438
 439    // sprintf the description
 440    char S[400];
 441    vsnprintf(S,sizeof(S) - 4,Fmt,args);
 442    strcat(S,"\r\n");
 443    va_end(args);
 444
 445    if (Debug == true)
 446       cerr << "-> '" << QuoteString(S,"") << "'" << endl;
 447
 448    // Send it off
 449    unsigned long Len = strlen(S);
 450    unsigned long Start = 0;
 451    while (Len != 0)
 452    {
 453       if (WaitFd(ServerFd,true,TimeOut) == false)
 454       {
 455          Close();
 456          return _error->Error(_("Connection timeout"));
 457       }
 458
 459       int Res = write(ServerFd,S + Start,Len);
 460       if (Res <= 0)
 461       {
 462          _error->Errno("write",_("Write error"));
 463          Close();
 464          return false;
 465       }
 466
 467       Len -= Res;
 468       Start += Res;
 469    }
 470
 471    return ReadResp(Ret,Text);
 472 }
 473                                                                         /*}}}*/
 474 // FTPConn::GoPasv - Enter Passive mode                                 /*{{{*/
 475 // ---------------------------------------------------------------------
 476 /* Try to enter passive mode, the return code does not indicate if passive
 477    mode could or could not be established, only if there was a fatal error.
 478    We have to enter passive mode every time we make a data connection :| */
 479 bool FTPConn::GoPasv()
 480 {
 481    /* The PASV command only works on IPv4 sockets, even though it could
 482       in theory suppory IPv6 via an all zeros reply */
 483    if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET ||
 484        ForceExtended == true)
 485       return ExtGoPasv();
 486
 487    if (PasvAddr != 0)
 488       freeaddrinfo(PasvAddr);
 489    PasvAddr = 0;
 490
 491    // Try to enable pasv mode
 492    unsigned int Tag;
 493    string Msg;
 494    if (WriteMsg(Tag,Msg,"PASV") == false)
 495       return false;
 496
 497    // Unsupported function
 498    string::size_type Pos = Msg.find('(');
 499    if (Tag >= 400 || Pos == string::npos)
 500       return true;
 501
 502    // Scan it
 503    unsigned a0,a1,a2,a3,p0,p1;
 504    if (sscanf(Msg.c_str() + Pos,"(%u,%u,%u,%u,%u,%u)",&a0,&a1,&a2,&a3,&p0,&p1) != 6)
 505       return true;
 506
 507    /* Some evil servers return 0 to mean their addr. We can actually speak
 508       to these servers natively using IPv6 */
 509    if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0)
 510    {
 511       // Get the IP in text form
 512       char Name[NI_MAXHOST];
 513       char Service[NI_MAXSERV];
 514       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 515                   Name,sizeof(Name),Service,sizeof(Service),
 516                   NI_NUMERICHOST|NI_NUMERICSERV);
 517
 518       struct addrinfo Hints;
 519       memset(&Hints,0,sizeof(Hints));
 520       Hints.ai_socktype = SOCK_STREAM;
 521       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 522       Hints.ai_flags |= AI_NUMERICHOST;
 523
 524       // Get a new passive address.
 525       char Port[100];
 526       snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 527       if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 528          return true;
 529       return true;
 530    }
 531
 532    struct addrinfo Hints;
 533    memset(&Hints,0,sizeof(Hints));
 534    Hints.ai_socktype = SOCK_STREAM;
 535    Hints.ai_family = AF_INET;
 536    Hints.ai_flags |= AI_NUMERICHOST;
 537
 538    // Get a new passive address.
 539    char Port[100];
 540    snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 541    char Name[100];
 542    snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3);
 543    if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 544       return true;
 545    return true;
 546 }
 547                                                                         /*}}}*/
 548 // FTPConn::ExtGoPasv - Enter Extended Passive mode                     /*{{{*/
 549 // ---------------------------------------------------------------------
 550 /* Try to enter extended passive mode. See GoPasv above and RFC 2428 */
 551 bool FTPConn::ExtGoPasv()
 552 {
 553    if (PasvAddr != 0)
 554       freeaddrinfo(PasvAddr);
 555    PasvAddr = 0;
 556
 557    // Try to enable pasv mode
 558    unsigned int Tag;
 559    string Msg;
 560    if (WriteMsg(Tag,Msg,"EPSV") == false)
 561       return false;
 562
 563    // Unsupported function
 564    string::size_type Pos = Msg.find('(');
 565    if (Tag >= 400 || Pos == string::npos)
 566       return true;
 567
 568    // Scan it
 569    string::const_iterator List[4];
 570    unsigned Count = 0;
 571    Pos++;
 572    for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); ++I)
 573    {
 574       if (*I != Msg[Pos])
 575          continue;
 576       if (Count >= 4)
 577          return true;
 578       List[Count++] = I;
 579    }
 580    if (Count != 4)
 581       return true;
 582
 583    // Break it up ..
 584    unsigned long Proto = 0;
 585    unsigned long Port = 0;
 586    string IP;
 587    IP = string(List[1]+1,List[2]);
 588    Port = atoi(string(List[2]+1,List[3]).c_str());
 589    if (IP.empty() == false)
 590       Proto = atoi(string(List[0]+1,List[1]).c_str());
 591
 592    if (Port == 0)
 593       return false;
 594
 595    // String version of the port
 596    char PStr[100];
 597    snprintf(PStr,sizeof(PStr),"%lu",Port);
 598
 599    // Get the IP in text form
 600    struct addrinfo Hints;
 601    memset(&Hints,0,sizeof(Hints));
 602    Hints.ai_socktype = SOCK_STREAM;
 603    Hints.ai_flags |= AI_NUMERICHOST;
 604
 605    /* The RFC defined case, connect to the old IP/protocol using the
 606       new port. */
 607    if (IP.empty() == true)
 608    {
 609       // Get the IP in text form
 610       char Name[NI_MAXHOST];
 611       char Service[NI_MAXSERV];
 612       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 613                   Name,sizeof(Name),Service,sizeof(Service),
 614                   NI_NUMERICHOST|NI_NUMERICSERV);
 615       IP = Name;
 616       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 617    }
 618    else
 619    {
 620       // Get the family..
 621       Hints.ai_family = 0;
 622       for (unsigned J = 0; AFMap[J].Family != 0; J++)
 623          if (AFMap[J].IETFFamily == Proto)
 624             Hints.ai_family = AFMap[J].Family;
 625       if (Hints.ai_family == 0)
 626          return true;
 627    }
 628
 629    // Get a new passive address.
 630    if (getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr) != 0)
 631       return true;
 632
 633    return true;
 634 }
 635                                                                         /*}}}*/
 636 // FTPConn::Size - Return the size of a file                            /*{{{*/
 637 // ---------------------------------------------------------------------
 638 /* Grab the file size from the server, 0 means no size or empty file */
 639 bool FTPConn::Size(const char *Path,unsigned long long &Size)
 640 {
 641    // Query the size
 642    unsigned int Tag;
 643    string Msg;
 644    Size = 0;
 645    if (WriteMsg(Tag,Msg,"SIZE %s",Path) == false)
 646       return false;
 647
 648    char *End;
 649    Size = strtoull(Msg.c_str(),&End,10);
 650    if (Tag >= 400 || End == Msg.c_str())
 651       Size = 0;
 652    return true;
 653 }
 654                                                                         /*}}}*/
 655 // FTPConn::ModTime - Return the modification time of the file          /*{{{*/
 656 // ---------------------------------------------------------------------
 657 /* Like Size no error is returned if the command is not supported. If the
 658    command fails then time is set to the current time of day to fool
 659    date checks. */
 660 bool FTPConn::ModTime(const char *Path, time_t &Time)
 661 {
 662    Time = time(&Time);
 663
 664    // Query the mod time
 665    unsigned int Tag;
 666    string Msg;
 667    if (WriteMsg(Tag,Msg,"MDTM %s",Path) == false)
 668       return false;
 669    if (Tag >= 400 || Msg.empty() == true || isdigit(Msg[0]) == 0)
 670       return true;
 671
 672    // Parse it
 673    return FTPMDTMStrToTime(Msg.c_str(), Time);
 674 }
 675                                                                         /*}}}*/
 676 // FTPConn::CreateDataFd - Get a data connection                        /*{{{*/
 677 // ---------------------------------------------------------------------
 678 /* Create the data connection. Call FinalizeDataFd after this though.. */
 679 bool FTPConn::CreateDataFd()
 680 {
 681    close(DataFd);
 682    DataFd = -1;
 683
 684    // Attempt to enter passive mode.
 685    if (TryPassive == true)
 686    {
 687       if (GoPasv() == false)
 688          return false;
 689
 690       // Oops, didn't work out, don't bother trying again.
 691       if (PasvAddr == 0)
 692          TryPassive = false;
 693    }
 694
 695    // Passive mode?
 696    if (PasvAddr != 0)
 697    {
 698       // Get a socket
 699       if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype,
 700                            PasvAddr->ai_protocol)) < 0)
 701          return _error->Errno("socket",_("Could not create a socket"));
 702
 703       // Connect to the server
 704       SetNonBlock(DataFd,true);
 705       if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 &&
 706           errno != EINPROGRESS)
 707          return _error->Errno("socket",_("Could not create a socket"));
 708
 709       /* This implements a timeout for connect by opening the connection
 710          nonblocking */
 711       if (WaitFd(DataFd,true,TimeOut) == false)
 712          return _error->Error(_("Could not connect data socket, connection timed out"));
 713       unsigned int Err;
 714       unsigned int Len = sizeof(Err);
 715       if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 716          return _error->Errno("getsockopt",_("Failed"));
 717       if (Err != 0)
 718          return _error->Error(_("Could not connect passive socket."));
 719
 720       return true;
 721    }
 722
 723    // Port mode :<
 724    close(DataListenFd);
 725    DataListenFd = -1;
 726
 727    // Get the information for a listening socket.
 728    struct addrinfo *BindAddr = NULL;
 729    struct addrinfo Hints;
 730    memset(&Hints,0,sizeof(Hints));
 731    Hints.ai_socktype = SOCK_STREAM;
 732    Hints.ai_flags |= AI_PASSIVE;
 733    Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family;
 734    if (getaddrinfo(0,"0",&Hints,&BindAddr) != 0 || BindAddr == NULL)
 735       return _error->Error(_("getaddrinfo was unable to get a listening socket"));
 736
 737    // Construct the socket
 738    if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype,
 739                               BindAddr->ai_protocol)) < 0)
 740    {
 741       freeaddrinfo(BindAddr);
 742       return _error->Errno("socket",_("Could not create a socket"));
 743    }
 744
 745    // Bind and listen
 746    if (bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0)
 747    {
 748       freeaddrinfo(BindAddr);
 749       return _error->Errno("bind",_("Could not bind a socket"));
 750    }
 751    freeaddrinfo(BindAddr);
 752    if (listen(DataListenFd,1) < 0)
 753       return _error->Errno("listen",_("Could not listen on the socket"));
 754    SetNonBlock(DataListenFd,true);
 755
 756    // Determine the name to send to the remote
 757    struct sockaddr_storage Addr;
 758    socklen_t AddrLen = sizeof(Addr);
 759    if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0)
 760       return _error->Errno("getsockname",_("Could not determine the socket's name"));
 761
 762
 763    // Reverse the address. We need the server address and the data port.
 764    char Name[NI_MAXHOST];
 765    char Service[NI_MAXSERV];
 766    char Service2[NI_MAXSERV];
 767    getnameinfo((struct sockaddr *)&Addr,AddrLen,
 768                Name,sizeof(Name),Service,sizeof(Service),
 769                NI_NUMERICHOST|NI_NUMERICSERV);
 770    getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen,
 771                Name,sizeof(Name),Service2,sizeof(Service2),
 772                NI_NUMERICHOST|NI_NUMERICSERV);
 773
 774    // Send off an IPv4 address in the old port format
 775    if (((struct sockaddr *)&Addr)->sa_family == AF_INET &&
 776        ForceExtended == false)
 777    {
 778       // Convert the dots in the quad into commas
 779       for (char *I = Name; *I != 0; I++)
 780          if (*I == '.')
 781             *I = ',';
 782       unsigned long Port = atoi(Service);
 783
 784       // Send the port command
 785       unsigned int Tag;
 786       string Msg;
 787       if (WriteMsg(Tag,Msg,"PORT %s,%d,%d",
 788                    Name,
 789                    (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false)
 790          return false;
 791       if (Tag >= 400)
 792          return _error->Error(_("Unable to send PORT command"));
 793       return true;
 794    }
 795
 796    // Construct an EPRT command
 797    unsigned Proto = 0;
 798    for (unsigned J = 0; AFMap[J].Family != 0; J++)
 799       if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family)
 800          Proto = AFMap[J].IETFFamily;
 801    if (Proto == 0)
 802       return _error->Error(_("Unknown address family %u (AF_*)"),
 803                            ((struct sockaddr *)&Addr)->sa_family);
 804
 805    // Send the EPRT command
 806    unsigned int Tag;
 807    string Msg;
 808    if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false)
 809       return false;
 810    if (Tag >= 400)
 811       return _error->Error(_("EPRT failed, server said: %s"),Msg.c_str());
 812    return true;
 813 }
 814                                                                         /*}}}*/
 815 // FTPConn::Finalize - Complete the Data connection                     /*{{{*/
 816 // ---------------------------------------------------------------------
 817 /* If the connection is in port mode this waits for the other end to hook
 818    up to us. */
 819 bool FTPConn::Finalize()
 820 {
 821    // Passive mode? Do nothing
 822    if (PasvAddr != 0)
 823       return true;
 824
 825    // Close any old socket..
 826    close(DataFd);
 827    DataFd = -1;
 828
 829    // Wait for someone to connect..
 830    if (WaitFd(DataListenFd,false,TimeOut) == false)
 831       return _error->Error(_("Data socket connect timed out"));
 832
 833    // Accept the connection
 834    struct sockaddr_in Addr;
 835    socklen_t Len = sizeof(Addr);
 836    DataFd = accept(DataListenFd,(struct sockaddr *)&Addr,&Len);
 837    if (DataFd < 0)
 838       return _error->Errno("accept",_("Unable to accept connection"));
 839
 840    close(DataListenFd);
 841    DataListenFd = -1;
 842
 843    return true;
 844 }
 845                                                                         /*}}}*/
 846 // FTPConn::Get - Get a file                                            /*{{{*/
 847 // ---------------------------------------------------------------------
 848 /* This opens a data connection, sends REST and RETR and then
 849    transfers the file over. */
 850 bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
 851                   Hashes &Hash,bool &Missing)
 852 {
 853    Missing = false;
 854    if (CreateDataFd() == false)
 855       return false;
 856
 857    unsigned int Tag;
 858    string Msg;
 859    if (Resume != 0)
 860    {
 861       if (WriteMsg(Tag,Msg,"REST %u",Resume) == false)
 862          return false;
 863       if (Tag >= 400)
 864          Resume = 0;
 865    }
 866
 867    if (To.Truncate(Resume) == false)
 868       return false;
 869
 870    if (To.Seek(0) == false)
 871       return false;
 872
 873    if (Resume != 0)
 874    {
 875       if (Hash.AddFD(To,Resume) == false)
 876       {
 877          _error->Errno("read",_("Problem hashing file"));
 878          return false;
 879       }
 880    }
 881
 882    // Send the get command
 883    if (WriteMsg(Tag,Msg,"RETR %s",Path) == false)
 884       return false;
 885
 886    if (Tag >= 400)
 887    {
 888       if (Tag == 550)
 889          Missing = true;
 890       return _error->Error(_("Unable to fetch file, server said '%s'"),Msg.c_str());
 891    }
 892
 893    // Finish off the data connection
 894    if (Finalize() == false)
 895       return false;
 896
 897    // Copy loop
 898    unsigned char Buffer[4096];
 899    while (1)
 900    {
 901       // Wait for some data..
 902       if (WaitFd(DataFd,false,TimeOut) == false)
 903       {
 904          Close();
 905          return _error->Error(_("Data socket timed out"));
 906       }
 907
 908       // Read the data..
 909       int Res = read(DataFd,Buffer,sizeof(Buffer));
 910       if (Res == 0)
 911          break;
 912       if (Res < 0)
 913       {
 914          if (errno == EAGAIN)
 915             continue;
 916          break;
 917       }
 918
 919       Hash.Add(Buffer,Res);
 920       if (To.Write(Buffer,Res) == false)
 921       {
 922          Close();
 923          return false;
 924       }
 925    }
 926
 927    // All done
 928    close(DataFd);
 929    DataFd = -1;
 930
 931    // Read the closing message from the server
 932    if (ReadResp(Tag,Msg) == false)
 933       return false;
 934    if (Tag >= 400)
 935       return _error->Error(_("Data transfer failed, server said '%s'"),Msg.c_str());
 936    return true;
 937 }
 938                                                                         /*}}}*/
 939
 940 // FtpMethod::FtpMethod - Constructor                                   /*{{{*/
 941 // ---------------------------------------------------------------------
 942 /* */
 943 FtpMethod::FtpMethod() : pkgAcqMethod("1.0",SendConfig)
 944 {
 945    signal(SIGTERM,SigTerm);
 946    signal(SIGINT,SigTerm);
 947
 948    Server = 0;
 949    FailFd = -1;
 950 }
 951                                                                         /*}}}*/
 952 // FtpMethod::SigTerm - Handle a fatal signal                           /*{{{*/
 953 // ---------------------------------------------------------------------
 954 /* This closes and timestamps the open file. This is necessary to get
 955    resume behavoir on user abort */
 956 void FtpMethod::SigTerm(int)
 957 {
 958    if (FailFd == -1)
 959       _exit(100);
 960
 961    // Timestamp
 962    struct timeval times[2];
 963    times[0].tv_sec = FailTime;
 964    times[1].tv_sec = FailTime;
 965    times[0].tv_usec = times[1].tv_usec = 0;
 966    utimes(FailFile.c_str(), times);
 967
 968    close(FailFd);
 969
 970    _exit(100);
 971 }
 972                                                                         /*}}}*/
 973 // FtpMethod::Configuration - Handle a configuration message            /*{{{*/
 974 // ---------------------------------------------------------------------
 975 /* We stash the desired pipeline depth */
 976 bool FtpMethod::Configuration(string Message)
 977 {
 978    if (pkgAcqMethod::Configuration(Message) == false)
 979       return false;
 980
 981    TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut);
 982    return true;
 983 }
 984                                                                         /*}}}*/
 985 // FtpMethod::Fetch - Fetch a file                                      /*{{{*/
 986 // ---------------------------------------------------------------------
 987 /* Fetch a single file, called by the base class..  */
 988 bool FtpMethod::Fetch(FetchItem *Itm)
 989 {
 990    URI Get = Itm->Uri;
 991    const char *File = Get.Path.c_str();
 992    FetchResult Res;
 993    Res.Filename = Itm->DestFile;
 994    Res.IMSHit = false;
 995
 996    maybe_add_auth (Get, _config->FindFile("Dir::Etc::netrc"));
 997
 998    // Connect to the server
 999    if (Server == 0 || Server->Comp(Get) == false)
1000    {
1001       delete Server;
1002       Server = new FTPConn(Get);
1003    }
1004
1005    // Could not connect is a transient error..
1006    if (Server->Open(this) == false)
1007    {
1008       Server->Close();
1009       Fail(true);
1010       return true;
1011    }
1012
1013    // Get the files information
1014    Status(_("Query"));
1015    unsigned long long Size;
1016    if (Server->Size(File,Size) == false ||
1017        Server->ModTime(File,FailTime) == false)
1018    {
1019       Fail(true);
1020       return true;
1021    }
1022    Res.Size = Size;
1023
1024    // See if it is an IMS hit
1025    if (Itm->LastModified == FailTime)
1026    {
1027       Res.Size = 0;
1028       Res.IMSHit = true;
1029       URIDone(Res);
1030       return true;
1031    }
1032
1033    // See if the file exists
1034    struct stat Buf;
1035    if (stat(Itm->DestFile.c_str(),&Buf) == 0)
1036    {
1037       if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime)
1038       {
1039          Res.Size = Buf.st_size;
1040          Res.LastModified = Buf.st_mtime;
1041          Res.ResumePoint = Buf.st_size;
1042          URIDone(Res);
1043          return true;
1044       }
1045
1046       // Resume?
1047       if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size)
1048          Res.ResumePoint = Buf.st_size;
1049    }
1050
1051    // Open the file
1052    Hashes Hash;
1053    {
1054       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
1055       if (_error->PendingError() == true)
1056          return false;
1057
1058       URIStart(Res);
1059
1060       FailFile = Itm->DestFile;
1061       FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1062       FailFd = Fd.Fd();
1063
1064       bool Missing;
1065       if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing) == false)
1066       {
1067          Fd.Close();
1068
1069          // Timestamp
1070          struct timeval times[2];
1071          times[0].tv_sec = FailTime;
1072          times[1].tv_sec = FailTime;
1073          times[0].tv_usec = times[1].tv_usec = 0;
1074          utimes(FailFile.c_str(), times);
1075
1076          // If the file is missing we hard fail and delete the destfile
1077          // otherwise transient fail
1078          if (Missing == true) {
1079             unlink(FailFile.c_str());
1080             return false;
1081          }
1082          Fail(true);
1083          return true;
1084       }
1085
1086       Res.Size = Fd.Size();
1087
1088       // Timestamp
1089       struct timeval times[2];
1090       times[0].tv_sec = FailTime;
1091       times[1].tv_sec = FailTime;
1092       times[0].tv_usec = times[1].tv_usec = 0;
1093       utimes(Fd.Name().c_str(), times);
1094       FailFd = -1;
1095    }
1096
1097    Res.LastModified = FailTime;
1098    Res.TakeHashes(Hash);
1099
1100    URIDone(Res);
1101
1102    return true;
1103 }
1104                                                                         /*}}}*/
1105
1106 int main(int, const char *argv[])
1107 {
1108    setlocale(LC_ALL, "");
1109
1110    /* See if we should be come the http client - we do this for http
1111       proxy urls */
1112    if (getenv("ftp_proxy") != 0)
1113    {
1114       URI Proxy = string(getenv("ftp_proxy"));
1115
1116       // Run the HTTP method
1117       if (Proxy.Access == "http")
1118       {
1119          // Copy over the environment setting
1120          char S[300];
1121          snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy"));
1122          putenv(S);
1123          putenv((char *)"no_proxy=");
1124
1125          // Run the http method
1126          string Path = flNotFile(argv[0]) + "http";
1127          execl(Path.c_str(),Path.c_str(),(char *)NULL);
1128          cerr << _("Unable to invoke ") << Path << endl;
1129          exit(100);
1130       }
1131    }
1132
1133    FtpMethod Mth;
1134
1135    return Mth.Run();
1136 }