methods/ftp.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: ftp.cc,v 1.31.2.1 2004/01/16 18:58:50 mdz Exp $
   4 /* ######################################################################
   5
   6    FTP Acquire Method - This is the FTP acquire method for APT.
   7
   8    This is a very simple implementation that does not try to optimize
   9    at all. Commands are sent syncronously with the FTP server (as the
  10    rfc recommends, but it is not really necessary..) and no tricks are
  11    done to speed things along.
  12
  13    RFC 2428 describes the IPv6 FTP behavior
  14
  15    ##################################################################### */
  16                                                                         /*}}}*/
  17 // Include Files                                                        /*{{{*/
  18 #include <config.h>
  19
  20 #include <apt-pkg/fileutl.h>
  21 #include <apt-pkg/acquire-method.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/hashes.h>
  24 #include <apt-pkg/netrc.h>
  25 #include <apt-pkg/configuration.h>
  26 #include <apt-pkg/strutl.h>
  27
  28 #include <ctype.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <sys/stat.h>
  32 #include <sys/time.h>
  33 #include <unistd.h>
  34 #include <signal.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdarg.h>
  38 #include <iostream>
  39
  40 // Internet stuff
  41 #include <netinet/in.h>
  42 #include <sys/socket.h>
  43 #include <arpa/inet.h>
  44 #include <netdb.h>
  45
  46 #include "rfc2553emu.h"
  47 #include "connect.h"
  48 #include "ftp.h"
  49
  50 #include <apti18n.h>
  51                                                                         /*}}}*/
  52
  53 using namespace std;
  54
  55 /* This table is for the EPRT and EPSV commands, it maps the OS address
  56    family to the IETF address families */
  57 struct AFMap
  58 {
  59    unsigned long Family;
  60    unsigned long IETFFamily;
  61 };
  62
  63 #ifndef AF_INET6
  64 struct AFMap AFMap[] = {{AF_INET,1},{0, 0}};
  65 #else
  66 struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{0, 0}};
  67 #endif
  68
  69 unsigned long TimeOut = 120;
  70 URI Proxy;
  71 string FtpMethod::FailFile;
  72 int FtpMethod::FailFd = -1;
  73 time_t FtpMethod::FailTime = 0;
  74
  75 // FTPConn::FTPConn - Constructor                                       /*{{{*/
  76 // ---------------------------------------------------------------------
  77 /* */
  78 FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1),
  79                             DataListenFd(-1), ServerName(Srv),
  80                             ForceExtended(false), TryPassive(true),
  81                             PeerAddrLen(0), ServerAddrLen(0)
  82 {
  83    Debug = _config->FindB("Debug::Acquire::Ftp",false);
  84    PasvAddr = 0;
  85    Buffer[0] = '\0';
  86 }
  87                                                                         /*}}}*/
  88 // FTPConn::~FTPConn - Destructor                                       /*{{{*/
  89 // ---------------------------------------------------------------------
  90 /* */
  91 FTPConn::~FTPConn()
  92 {
  93    Close();
  94 }
  95                                                                         /*}}}*/
  96 // FTPConn::Close - Close down the connection                           /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* Just tear down the socket and data socket */
  99 void FTPConn::Close()
 100 {
 101    close(ServerFd);
 102    ServerFd = -1;
 103    close(DataFd);
 104    DataFd = -1;
 105    close(DataListenFd);
 106    DataListenFd = -1;
 107
 108    if (PasvAddr != 0)
 109       freeaddrinfo(PasvAddr);
 110    PasvAddr = 0;
 111 }
 112                                                                         /*}}}*/
 113 // FTPConn::Open - Open a new connection                                /*{{{*/
 114 // ---------------------------------------------------------------------
 115 /* Connect to the server using a non-blocking connection and perform a
 116    login. */
 117 bool FTPConn::Open(pkgAcqMethod *Owner)
 118 {
 119    // Use the already open connection if possible.
 120    if (ServerFd != -1)
 121       return true;
 122
 123    Close();
 124
 125    // Determine the proxy setting
 126    string SpecificProxy = _config->Find("Acquire::ftp::Proxy::" + ServerName.Host);
 127    if (!SpecificProxy.empty())
 128    {
 129            if (SpecificProxy == "DIRECT")
 130                    Proxy = "";
 131            else
 132                    Proxy = SpecificProxy;
 133    }
 134    else
 135    {
 136            string DefProxy = _config->Find("Acquire::ftp::Proxy");
 137            if (!DefProxy.empty())
 138            {
 139                    Proxy = DefProxy;
 140            }
 141            else
 142            {
 143                    char* result = getenv("ftp_proxy");
 144                    Proxy = result ? result : "";
 145            }
 146    }
 147
 148    // Parse no_proxy, a , separated list of domains
 149    if (getenv("no_proxy") != 0)
 150    {
 151       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 152          Proxy = "";
 153    }
 154
 155    // Determine what host and port to use based on the proxy settings
 156    int Port = 0;
 157    string Host;
 158    if (Proxy.empty() == true)
 159    {
 160       if (ServerName.Port != 0)
 161          Port = ServerName.Port;
 162       Host = ServerName.Host;
 163    }
 164    else
 165    {
 166       if (Proxy.Port != 0)
 167          Port = Proxy.Port;
 168       Host = Proxy.Host;
 169    }
 170
 171    /* Connect to the remote server. Since FTP is connection oriented we
 172       want to make sure we get a new server every time we reconnect */
 173    RotateDNS();
 174    if (Connect(Host,Port,"ftp",21,ServerFd,TimeOut,Owner) == false)
 175       return false;
 176
 177    // Login must be before getpeername otherwise dante won't work.
 178    Owner->Status(_("Logging in"));
 179    bool Res = Login();
 180
 181    // Get the remote server's address
 182    PeerAddrLen = sizeof(PeerAddr);
 183    if (getpeername(ServerFd,(sockaddr *)&PeerAddr,&PeerAddrLen) != 0)
 184       return _error->Errno("getpeername",_("Unable to determine the peer name"));
 185
 186    // Get the local machine's address
 187    ServerAddrLen = sizeof(ServerAddr);
 188    if (getsockname(ServerFd,(sockaddr *)&ServerAddr,&ServerAddrLen) != 0)
 189       return _error->Errno("getsockname",_("Unable to determine the local name"));
 190
 191    return Res;
 192 }
 193                                                                         /*}}}*/
 194 // FTPConn::Login - Login to the remote server                          /*{{{*/
 195 // ---------------------------------------------------------------------
 196 /* This performs both normal login and proxy login using a simples script
 197    stored in the config file. */
 198 bool FTPConn::Login()
 199 {
 200    unsigned int Tag;
 201    string Msg;
 202
 203    // Setup the variables needed for authentication
 204    string User = "anonymous";
 205    string Pass = "apt_get_ftp_2.1@debian.linux.user";
 206
 207    // Fill in the user/pass
 208    if (ServerName.User.empty() == false)
 209       User = ServerName.User;
 210    if (ServerName.Password.empty() == false)
 211       Pass = ServerName.Password;
 212
 213    // Perform simple login
 214    if (Proxy.empty() == true)
 215    {
 216       // Read the initial response
 217       if (ReadResp(Tag,Msg) == false)
 218          return false;
 219       if (Tag >= 400)
 220          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 221
 222       // Send the user
 223       if (WriteMsg(Tag,Msg,"USER %s",User.c_str()) == false)
 224          return false;
 225       if (Tag >= 400)
 226          return _error->Error(_("USER failed, server said: %s"),Msg.c_str());
 227
 228       if (Tag == 331) { // 331 User name okay, need password.
 229          // Send the Password
 230          if (WriteMsg(Tag,Msg,"PASS %s",Pass.c_str()) == false)
 231             return false;
 232          if (Tag >= 400)
 233             return _error->Error(_("PASS failed, server said: %s"),Msg.c_str());
 234       }
 235
 236       // Enter passive mode
 237       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 238          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 239       else
 240          TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 241    }
 242    else
 243    {
 244       // Read the initial response
 245       if (ReadResp(Tag,Msg) == false)
 246          return false;
 247       if (Tag >= 400)
 248          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 249
 250       // Perform proxy script execution
 251       Configuration::Item const *Opts = _config->Tree("Acquire::ftp::ProxyLogin");
 252       if (Opts == 0 || Opts->Child == 0)
 253          return _error->Error(_("A proxy server was specified but no login "
 254                               "script, Acquire::ftp::ProxyLogin is empty."));
 255       Opts = Opts->Child;
 256
 257       // Iterate over the entire login script
 258       for (; Opts != 0; Opts = Opts->Next)
 259       {
 260          if (Opts->Value.empty() == true)
 261             continue;
 262
 263          // Substitute the variables into the command
 264          char SitePort[20];
 265          if (ServerName.Port != 0)
 266             sprintf(SitePort,"%u",ServerName.Port);
 267          else
 268             strcpy(SitePort,"21");
 269          string Tmp = Opts->Value;
 270          Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
 271          Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
 272          Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
 273          Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
 274          Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort);
 275          Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
 276
 277          // Send the command
 278          if (WriteMsg(Tag,Msg,"%s",Tmp.c_str()) == false)
 279             return false;
 280          if (Tag >= 400)
 281             return _error->Error(_("Login script command '%s' failed, server said: %s"),Tmp.c_str(),Msg.c_str());
 282       }
 283
 284       // Enter passive mode
 285       TryPassive = false;
 286       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 287          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 288       else
 289       {
 290          if (_config->Exists("Acquire::FTP::Proxy::Passive") == true)
 291             TryPassive = _config->FindB("Acquire::FTP::Proxy::Passive",true);
 292          else
 293             TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 294       }
 295    }
 296
 297    // Force the use of extended commands
 298    if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true)
 299       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true);
 300    else
 301       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false);
 302
 303    // Binary mode
 304    if (WriteMsg(Tag,Msg,"TYPE I") == false)
 305       return false;
 306    if (Tag >= 400)
 307       return _error->Error(_("TYPE failed, server said: %s"),Msg.c_str());
 308
 309    return true;
 310 }
 311                                                                         /*}}}*/
 312 // FTPConn::ReadLine - Read a line from the server                      /*{{{*/
 313 // ---------------------------------------------------------------------
 314 /* This performs a very simple buffered read. */
 315 bool FTPConn::ReadLine(string &Text)
 316 {
 317    if (ServerFd == -1)
 318       return false;
 319
 320    // Suck in a line
 321    while (Len < sizeof(Buffer))
 322    {
 323       // Scan the buffer for a new line
 324       for (unsigned int I = 0; I != Len; I++)
 325       {
 326          // Escape some special chars
 327          if (Buffer[I] == 0)
 328             Buffer[I] = '?';
 329
 330          // End of line?
 331          if (Buffer[I] != '\n')
 332             continue;
 333
 334          I++;
 335          Text = string(Buffer,I);
 336          memmove(Buffer,Buffer+I,Len - I);
 337          Len -= I;
 338          return true;
 339       }
 340
 341       // Wait for some data..
 342       if (WaitFd(ServerFd,false,TimeOut) == false)
 343       {
 344          Close();
 345          return _error->Error(_("Connection timeout"));
 346       }
 347
 348       // Suck it back
 349       int Res = read(ServerFd,Buffer + Len,sizeof(Buffer) - Len);
 350       if (Res == 0)
 351          _error->Error(_("Server closed the connection"));
 352       if (Res <= 0)
 353       {
 354          _error->Errno("read",_("Read error"));
 355          Close();
 356          return false;
 357       }
 358       Len += Res;
 359    }
 360
 361    return _error->Error(_("A response overflowed the buffer."));
 362 }
 363                                                                         /*}}}*/
 364 // FTPConn::ReadResp - Read a full response from the server             /*{{{*/
 365 // ---------------------------------------------------------------------
 366 /* This reads a reply code from the server, it handles both p */
 367 bool FTPConn::ReadResp(unsigned int &Ret,string &Text)
 368 {
 369    // Grab the first line of the response
 370    string Msg;
 371    if (ReadLine(Msg) == false)
 372        return false;
 373
 374    // Get the ID code
 375    char *End;
 376    Ret = strtol(Msg.c_str(),&End,10);
 377    if (End - Msg.c_str() != 3)
 378       return _error->Error(_("Protocol corruption"));
 379
 380    // All done ?
 381    Text = Msg.c_str()+4;
 382    if (*End == ' ')
 383    {
 384       if (Debug == true)
 385          cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 386       return true;
 387    }
 388
 389    if (*End != '-')
 390       return _error->Error(_("Protocol corruption"));
 391
 392    /* Okay, here we do the continued message trick. This is foolish, but
 393       proftpd follows the protocol as specified and wu-ftpd doesn't, so
 394       we filter. I wonder how many clients break if you use proftpd and
 395       put a '- in the 3rd spot in the message? */
 396    char Leader[4];
 397    strncpy(Leader,Msg.c_str(),3);
 398    Leader[3] = 0;
 399    while (ReadLine(Msg) == true)
 400    {
 401       // Short, it must be using RFC continuation..
 402       if (Msg.length() < 4)
 403       {
 404          Text += Msg;
 405          continue;
 406       }
 407
 408       // Oops, finished
 409       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == ' ')
 410       {
 411          Text += Msg.c_str()+4;
 412          break;
 413       }
 414
 415       // This message has the wu-ftpd style reply code prefixed
 416       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == '-')
 417       {
 418          Text += Msg.c_str()+4;
 419          continue;
 420       }
 421
 422       // Must be RFC style prefixing
 423       Text += Msg;
 424    }
 425
 426    if (Debug == true && _error->PendingError() == false)
 427       cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 428
 429    return !_error->PendingError();
 430 }
 431                                                                         /*}}}*/
 432 // FTPConn::WriteMsg - Send a message to the server                     /*{{{*/
 433 // ---------------------------------------------------------------------
 434 /* Simple printf like function.. */
 435 bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...)
 436 {
 437    va_list args;
 438    va_start(args,Fmt);
 439
 440    // sprintf the description
 441    char S[400];
 442    vsnprintf(S,sizeof(S) - 4,Fmt,args);
 443    strcat(S,"\r\n");
 444    va_end(args);
 445
 446    if (Debug == true)
 447       cerr << "-> '" << QuoteString(S,"") << "'" << endl;
 448
 449    // Send it off
 450    unsigned long Len = strlen(S);
 451    unsigned long Start = 0;
 452    while (Len != 0)
 453    {
 454       if (WaitFd(ServerFd,true,TimeOut) == false)
 455       {
 456          Close();
 457          return _error->Error(_("Connection timeout"));
 458       }
 459
 460       int Res = write(ServerFd,S + Start,Len);
 461       if (Res <= 0)
 462       {
 463          _error->Errno("write",_("Write error"));
 464          Close();
 465          return false;
 466       }
 467
 468       Len -= Res;
 469       Start += Res;
 470    }
 471
 472    return ReadResp(Ret,Text);
 473 }
 474                                                                         /*}}}*/
 475 // FTPConn::GoPasv - Enter Passive mode                                 /*{{{*/
 476 // ---------------------------------------------------------------------
 477 /* Try to enter passive mode, the return code does not indicate if passive
 478    mode could or could not be established, only if there was a fatal error.
 479    We have to enter passive mode every time we make a data connection :| */
 480 bool FTPConn::GoPasv()
 481 {
 482    /* The PASV command only works on IPv4 sockets, even though it could
 483       in theory suppory IPv6 via an all zeros reply */
 484    if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET ||
 485        ForceExtended == true)
 486       return ExtGoPasv();
 487
 488    if (PasvAddr != 0)
 489       freeaddrinfo(PasvAddr);
 490    PasvAddr = 0;
 491
 492    // Try to enable pasv mode
 493    unsigned int Tag;
 494    string Msg;
 495    if (WriteMsg(Tag,Msg,"PASV") == false)
 496       return false;
 497
 498    // Unsupported function
 499    string::size_type Pos = Msg.find('(');
 500    if (Tag >= 400 || Pos == string::npos)
 501       return true;
 502
 503    // Scan it
 504    unsigned a0,a1,a2,a3,p0,p1;
 505    if (sscanf(Msg.c_str() + Pos,"(%u,%u,%u,%u,%u,%u)",&a0,&a1,&a2,&a3,&p0,&p1) != 6)
 506       return true;
 507
 508    /* Some evil servers return 0 to mean their addr. We can actually speak
 509       to these servers natively using IPv6 */
 510    if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0)
 511    {
 512       // Get the IP in text form
 513       char Name[NI_MAXHOST];
 514       char Service[NI_MAXSERV];
 515       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 516                   Name,sizeof(Name),Service,sizeof(Service),
 517                   NI_NUMERICHOST|NI_NUMERICSERV);
 518
 519       struct addrinfo Hints;
 520       memset(&Hints,0,sizeof(Hints));
 521       Hints.ai_socktype = SOCK_STREAM;
 522       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 523       Hints.ai_flags |= AI_NUMERICHOST;
 524
 525       // Get a new passive address.
 526       char Port[100];
 527       snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 528       if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 529          return true;
 530       return true;
 531    }
 532
 533    struct addrinfo Hints;
 534    memset(&Hints,0,sizeof(Hints));
 535    Hints.ai_socktype = SOCK_STREAM;
 536    Hints.ai_family = AF_INET;
 537    Hints.ai_flags |= AI_NUMERICHOST;
 538
 539    // Get a new passive address.
 540    char Port[100];
 541    snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 542    char Name[100];
 543    snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3);
 544    if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 545       return true;
 546    return true;
 547 }
 548                                                                         /*}}}*/
 549 // FTPConn::ExtGoPasv - Enter Extended Passive mode                     /*{{{*/
 550 // ---------------------------------------------------------------------
 551 /* Try to enter extended passive mode. See GoPasv above and RFC 2428 */
 552 bool FTPConn::ExtGoPasv()
 553 {
 554    if (PasvAddr != 0)
 555       freeaddrinfo(PasvAddr);
 556    PasvAddr = 0;
 557
 558    // Try to enable pasv mode
 559    unsigned int Tag;
 560    string Msg;
 561    if (WriteMsg(Tag,Msg,"EPSV") == false)
 562       return false;
 563
 564    // Unsupported function
 565    string::size_type Pos = Msg.find('(');
 566    if (Tag >= 400 || Pos == string::npos)
 567       return true;
 568
 569    // Scan it
 570    string::const_iterator List[4];
 571    unsigned Count = 0;
 572    Pos++;
 573    for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); ++I)
 574    {
 575       if (*I != Msg[Pos])
 576          continue;
 577       if (Count >= 4)
 578          return true;
 579       List[Count++] = I;
 580    }
 581    if (Count != 4)
 582       return true;
 583
 584    // Break it up ..
 585    unsigned long Proto = 0;
 586    unsigned long Port = 0;
 587    string IP;
 588    IP = string(List[1]+1,List[2]);
 589    Port = atoi(string(List[2]+1,List[3]).c_str());
 590    if (IP.empty() == false)
 591       Proto = atoi(string(List[0]+1,List[1]).c_str());
 592
 593    if (Port == 0)
 594       return false;
 595
 596    // String version of the port
 597    char PStr[100];
 598    snprintf(PStr,sizeof(PStr),"%lu",Port);
 599
 600    // Get the IP in text form
 601    struct addrinfo Hints;
 602    memset(&Hints,0,sizeof(Hints));
 603    Hints.ai_socktype = SOCK_STREAM;
 604    Hints.ai_flags |= AI_NUMERICHOST;
 605
 606    /* The RFC defined case, connect to the old IP/protocol using the
 607       new port. */
 608    if (IP.empty() == true)
 609    {
 610       // Get the IP in text form
 611       char Name[NI_MAXHOST];
 612       char Service[NI_MAXSERV];
 613       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 614                   Name,sizeof(Name),Service,sizeof(Service),
 615                   NI_NUMERICHOST|NI_NUMERICSERV);
 616       IP = Name;
 617       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 618    }
 619    else
 620    {
 621       // Get the family..
 622       Hints.ai_family = 0;
 623       for (unsigned J = 0; AFMap[J].Family != 0; J++)
 624          if (AFMap[J].IETFFamily == Proto)
 625             Hints.ai_family = AFMap[J].Family;
 626       if (Hints.ai_family == 0)
 627          return true;
 628    }
 629
 630    // Get a new passive address.
 631    if (getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr) != 0)
 632       return true;
 633
 634    return true;
 635 }
 636                                                                         /*}}}*/
 637 // FTPConn::Size - Return the size of a file                            /*{{{*/
 638 // ---------------------------------------------------------------------
 639 /* Grab the file size from the server, 0 means no size or empty file */
 640 bool FTPConn::Size(const char *Path,unsigned long long &Size)
 641 {
 642    // Query the size
 643    unsigned int Tag;
 644    string Msg;
 645    Size = 0;
 646    if (WriteMsg(Tag,Msg,"SIZE %s",Path) == false)
 647       return false;
 648
 649    char *End;
 650    Size = strtoull(Msg.c_str(),&End,10);
 651    if (Tag >= 400 || End == Msg.c_str())
 652       Size = 0;
 653    return true;
 654 }
 655                                                                         /*}}}*/
 656 // FTPConn::ModTime - Return the modification time of the file          /*{{{*/
 657 // ---------------------------------------------------------------------
 658 /* Like Size no error is returned if the command is not supported. If the
 659    command fails then time is set to the current time of day to fool
 660    date checks. */
 661 bool FTPConn::ModTime(const char *Path, time_t &Time)
 662 {
 663    Time = time(&Time);
 664
 665    // Query the mod time
 666    unsigned int Tag;
 667    string Msg;
 668    if (WriteMsg(Tag,Msg,"MDTM %s",Path) == false)
 669       return false;
 670    if (Tag >= 400 || Msg.empty() == true || isdigit(Msg[0]) == 0)
 671       return true;
 672
 673    // Parse it
 674    return FTPMDTMStrToTime(Msg.c_str(), Time);
 675 }
 676                                                                         /*}}}*/
 677 // FTPConn::CreateDataFd - Get a data connection                        /*{{{*/
 678 // ---------------------------------------------------------------------
 679 /* Create the data connection. Call FinalizeDataFd after this though.. */
 680 bool FTPConn::CreateDataFd()
 681 {
 682    close(DataFd);
 683    DataFd = -1;
 684
 685    // Attempt to enter passive mode.
 686    if (TryPassive == true)
 687    {
 688       if (GoPasv() == false)
 689          return false;
 690
 691       // Oops, didn't work out, don't bother trying again.
 692       if (PasvAddr == 0)
 693          TryPassive = false;
 694    }
 695
 696    // Passive mode?
 697    if (PasvAddr != 0)
 698    {
 699       // Get a socket
 700       if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype,
 701                            PasvAddr->ai_protocol)) < 0)
 702          return _error->Errno("socket",_("Could not create a socket"));
 703
 704       // Connect to the server
 705       SetNonBlock(DataFd,true);
 706       if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 &&
 707           errno != EINPROGRESS)
 708          return _error->Errno("socket",_("Could not create a socket"));
 709
 710       /* This implements a timeout for connect by opening the connection
 711          nonblocking */
 712       if (WaitFd(DataFd,true,TimeOut) == false)
 713          return _error->Error(_("Could not connect data socket, connection timed out"));
 714       unsigned int Err;
 715       unsigned int Len = sizeof(Err);
 716       if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 717          return _error->Errno("getsockopt",_("Failed"));
 718       if (Err != 0)
 719          return _error->Error(_("Could not connect passive socket."));
 720
 721       return true;
 722    }
 723
 724    // Port mode :<
 725    close(DataListenFd);
 726    DataListenFd = -1;
 727
 728    // Get the information for a listening socket.
 729    struct addrinfo *BindAddr = NULL;
 730    struct addrinfo Hints;
 731    memset(&Hints,0,sizeof(Hints));
 732    Hints.ai_socktype = SOCK_STREAM;
 733    Hints.ai_flags |= AI_PASSIVE;
 734    Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family;
 735    if (getaddrinfo(0,"0",&Hints,&BindAddr) != 0 || BindAddr == NULL)
 736       return _error->Error(_("getaddrinfo was unable to get a listening socket"));
 737
 738    // Construct the socket
 739    if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype,
 740                               BindAddr->ai_protocol)) < 0)
 741    {
 742       freeaddrinfo(BindAddr);
 743       return _error->Errno("socket",_("Could not create a socket"));
 744    }
 745
 746    // Bind and listen
 747    if (bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0)
 748    {
 749       freeaddrinfo(BindAddr);
 750       return _error->Errno("bind",_("Could not bind a socket"));
 751    }
 752    freeaddrinfo(BindAddr);
 753    if (listen(DataListenFd,1) < 0)
 754       return _error->Errno("listen",_("Could not listen on the socket"));
 755    SetNonBlock(DataListenFd,true);
 756
 757    // Determine the name to send to the remote
 758    struct sockaddr_storage Addr;
 759    socklen_t AddrLen = sizeof(Addr);
 760    if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0)
 761       return _error->Errno("getsockname",_("Could not determine the socket's name"));
 762
 763
 764    // Reverse the address. We need the server address and the data port.
 765    char Name[NI_MAXHOST];
 766    char Service[NI_MAXSERV];
 767    char Service2[NI_MAXSERV];
 768    getnameinfo((struct sockaddr *)&Addr,AddrLen,
 769                Name,sizeof(Name),Service,sizeof(Service),
 770                NI_NUMERICHOST|NI_NUMERICSERV);
 771    getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen,
 772                Name,sizeof(Name),Service2,sizeof(Service2),
 773                NI_NUMERICHOST|NI_NUMERICSERV);
 774
 775    // Send off an IPv4 address in the old port format
 776    if (((struct sockaddr *)&Addr)->sa_family == AF_INET &&
 777        ForceExtended == false)
 778    {
 779       // Convert the dots in the quad into commas
 780       for (char *I = Name; *I != 0; I++)
 781          if (*I == '.')
 782             *I = ',';
 783       unsigned long Port = atoi(Service);
 784
 785       // Send the port command
 786       unsigned int Tag;
 787       string Msg;
 788       if (WriteMsg(Tag,Msg,"PORT %s,%d,%d",
 789                    Name,
 790                    (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false)
 791          return false;
 792       if (Tag >= 400)
 793          return _error->Error(_("Unable to send PORT command"));
 794       return true;
 795    }
 796
 797    // Construct an EPRT command
 798    unsigned Proto = 0;
 799    for (unsigned J = 0; AFMap[J].Family != 0; J++)
 800       if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family)
 801          Proto = AFMap[J].IETFFamily;
 802    if (Proto == 0)
 803       return _error->Error(_("Unknown address family %u (AF_*)"),
 804                            ((struct sockaddr *)&Addr)->sa_family);
 805
 806    // Send the EPRT command
 807    unsigned int Tag;
 808    string Msg;
 809    if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false)
 810       return false;
 811    if (Tag >= 400)
 812       return _error->Error(_("EPRT failed, server said: %s"),Msg.c_str());
 813    return true;
 814 }
 815                                                                         /*}}}*/
 816 // FTPConn::Finalize - Complete the Data connection                     /*{{{*/
 817 // ---------------------------------------------------------------------
 818 /* If the connection is in port mode this waits for the other end to hook
 819    up to us. */
 820 bool FTPConn::Finalize()
 821 {
 822    // Passive mode? Do nothing
 823    if (PasvAddr != 0)
 824       return true;
 825
 826    // Close any old socket..
 827    close(DataFd);
 828    DataFd = -1;
 829
 830    // Wait for someone to connect..
 831    if (WaitFd(DataListenFd,false,TimeOut) == false)
 832       return _error->Error(_("Data socket connect timed out"));
 833
 834    // Accept the connection
 835    struct sockaddr_in Addr;
 836    socklen_t Len = sizeof(Addr);
 837    DataFd = accept(DataListenFd,(struct sockaddr *)&Addr,&Len);
 838    if (DataFd < 0)
 839       return _error->Errno("accept",_("Unable to accept connection"));
 840
 841    close(DataListenFd);
 842    DataListenFd = -1;
 843
 844    return true;
 845 }
 846                                                                         /*}}}*/
 847 // FTPConn::Get - Get a file                                            /*{{{*/
 848 // ---------------------------------------------------------------------
 849 /* This opens a data connection, sends REST and RETR and then
 850    transfers the file over. */
 851 bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
 852                   Hashes &Hash,bool &Missing, unsigned long long MaximumSize,
 853                   pkgAcqMethod *Owner)
 854 {
 855    Missing = false;
 856    if (CreateDataFd() == false)
 857       return false;
 858
 859    unsigned int Tag;
 860    string Msg;
 861    if (Resume != 0)
 862    {
 863       if (WriteMsg(Tag,Msg,"REST %u",Resume) == false)
 864          return false;
 865       if (Tag >= 400)
 866          Resume = 0;
 867    }
 868
 869    if (To.Truncate(Resume) == false)
 870       return false;
 871
 872    if (To.Seek(0) == false)
 873       return false;
 874
 875    if (Resume != 0)
 876    {
 877       if (Hash.AddFD(To,Resume) == false)
 878       {
 879          _error->Errno("read",_("Problem hashing file"));
 880          return false;
 881       }
 882    }
 883
 884    // Send the get command
 885    if (WriteMsg(Tag,Msg,"RETR %s",Path) == false)
 886       return false;
 887
 888    if (Tag >= 400)
 889    {
 890       if (Tag == 550)
 891          Missing = true;
 892       return _error->Error(_("Unable to fetch file, server said '%s'"),Msg.c_str());
 893    }
 894
 895    // Finish off the data connection
 896    if (Finalize() == false)
 897       return false;
 898
 899    // Copy loop
 900    unsigned char Buffer[4096];
 901    while (1)
 902    {
 903       // Wait for some data..
 904       if (WaitFd(DataFd,false,TimeOut) == false)
 905       {
 906          Close();
 907          return _error->Error(_("Data socket timed out"));
 908       }
 909
 910       // Read the data..
 911       int Res = read(DataFd,Buffer,sizeof(Buffer));
 912       if (Res == 0)
 913          break;
 914       if (Res < 0)
 915       {
 916          if (errno == EAGAIN)
 917             continue;
 918          break;
 919       }
 920
 921       Hash.Add(Buffer,Res);
 922       if (To.Write(Buffer,Res) == false)
 923       {
 924          Close();
 925          return false;
 926       }
 927
 928       if (MaximumSize > 0 && To.Tell() > MaximumSize)
 929       {
 930          Owner->SetFailReason("MaximumSizeExceeded");
 931          return _error->Error("Writing more data than expected (%llu > %llu)",
 932                               To.Tell(), MaximumSize);
 933       }
 934    }
 935
 936    // All done
 937    close(DataFd);
 938    DataFd = -1;
 939
 940    // Read the closing message from the server
 941    if (ReadResp(Tag,Msg) == false)
 942       return false;
 943    if (Tag >= 400)
 944       return _error->Error(_("Data transfer failed, server said '%s'"),Msg.c_str());
 945    return true;
 946 }
 947                                                                         /*}}}*/
 948
 949 // FtpMethod::FtpMethod - Constructor                                   /*{{{*/
 950 // ---------------------------------------------------------------------
 951 /* */
 952 FtpMethod::FtpMethod() : pkgAcqMethod("1.0",SendConfig)
 953 {
 954    signal(SIGTERM,SigTerm);
 955    signal(SIGINT,SigTerm);
 956
 957    Server = 0;
 958    FailFd = -1;
 959 }
 960                                                                         /*}}}*/
 961 // FtpMethod::SigTerm - Handle a fatal signal                           /*{{{*/
 962 // ---------------------------------------------------------------------
 963 /* This closes and timestamps the open file. This is necessary to get
 964    resume behavoir on user abort */
 965 void FtpMethod::SigTerm(int)
 966 {
 967    if (FailFd == -1)
 968       _exit(100);
 969
 970    // Timestamp
 971    struct timeval times[2];
 972    times[0].tv_sec = FailTime;
 973    times[1].tv_sec = FailTime;
 974    times[0].tv_usec = times[1].tv_usec = 0;
 975    utimes(FailFile.c_str(), times);
 976
 977    close(FailFd);
 978
 979    _exit(100);
 980 }
 981                                                                         /*}}}*/
 982 // FtpMethod::Configuration - Handle a configuration message            /*{{{*/
 983 // ---------------------------------------------------------------------
 984 /* We stash the desired pipeline depth */
 985 bool FtpMethod::Configuration(string Message)
 986 {
 987    if (pkgAcqMethod::Configuration(Message) == false)
 988       return false;
 989
 990    TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut);
 991    return true;
 992 }
 993                                                                         /*}}}*/
 994 // FtpMethod::Fetch - Fetch a file                                      /*{{{*/
 995 // ---------------------------------------------------------------------
 996 /* Fetch a single file, called by the base class..  */
 997 bool FtpMethod::Fetch(FetchItem *Itm)
 998 {
 999    URI Get = Itm->Uri;
1000    const char *File = Get.Path.c_str();
1001    FetchResult Res;
1002    Res.Filename = Itm->DestFile;
1003    Res.IMSHit = false;
1004
1005    maybe_add_auth (Get, _config->FindFile("Dir::Etc::netrc"));
1006
1007    // Connect to the server
1008    if (Server == 0 || Server->Comp(Get) == false)
1009    {
1010       delete Server;
1011       Server = new FTPConn(Get);
1012    }
1013
1014    // Could not connect is a transient error..
1015    if (Server->Open(this) == false)
1016    {
1017       Server->Close();
1018       Fail(true);
1019       return true;
1020    }
1021
1022    // Get the files information
1023    Status(_("Query"));
1024    unsigned long long Size;
1025    if (Server->Size(File,Size) == false ||
1026        Server->ModTime(File,FailTime) == false)
1027    {
1028       Fail(true);
1029       return true;
1030    }
1031    Res.Size = Size;
1032
1033    // See if it is an IMS hit
1034    if (Itm->LastModified == FailTime)
1035    {
1036       Res.Size = 0;
1037       Res.IMSHit = true;
1038       URIDone(Res);
1039       return true;
1040    }
1041
1042    // See if the file exists
1043    struct stat Buf;
1044    if (stat(Itm->DestFile.c_str(),&Buf) == 0)
1045    {
1046       if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime)
1047       {
1048          Res.Size = Buf.st_size;
1049          Res.LastModified = Buf.st_mtime;
1050          Res.ResumePoint = Buf.st_size;
1051          URIDone(Res);
1052          return true;
1053       }
1054
1055       // Resume?
1056       if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size)
1057          Res.ResumePoint = Buf.st_size;
1058    }
1059
1060    // Open the file
1061    Hashes Hash;
1062    {
1063       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
1064       if (_error->PendingError() == true)
1065          return false;
1066
1067       URIStart(Res);
1068
1069       FailFile = Itm->DestFile;
1070       FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1071       FailFd = Fd.Fd();
1072
1073       bool Missing;
1074       if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->MaximumSize,this) == false)
1075       {
1076          Fd.Close();
1077
1078          // Timestamp
1079          struct timeval times[2];
1080          times[0].tv_sec = FailTime;
1081          times[1].tv_sec = FailTime;
1082          times[0].tv_usec = times[1].tv_usec = 0;
1083          utimes(FailFile.c_str(), times);
1084
1085          // If the file is missing we hard fail and delete the destfile
1086          // otherwise transient fail
1087          if (Missing == true) {
1088             unlink(FailFile.c_str());
1089             return false;
1090          }
1091          Fail(true);
1092          return true;
1093       }
1094
1095       Res.Size = Fd.Size();
1096
1097       // Timestamp
1098       struct timeval times[2];
1099       times[0].tv_sec = FailTime;
1100       times[1].tv_sec = FailTime;
1101       times[0].tv_usec = times[1].tv_usec = 0;
1102       utimes(Fd.Name().c_str(), times);
1103       FailFd = -1;
1104    }
1105
1106    Res.LastModified = FailTime;
1107    Res.TakeHashes(Hash);
1108
1109    URIDone(Res);
1110
1111    return true;
1112 }
1113                                                                         /*}}}*/
1114
1115 int main(int, const char *argv[])
1116 {
1117    setlocale(LC_ALL, "");
1118
1119    /* See if we should be come the http client - we do this for http
1120       proxy urls */
1121    if (getenv("ftp_proxy") != 0)
1122    {
1123       URI Proxy = string(getenv("ftp_proxy"));
1124
1125       // Run the HTTP method
1126       if (Proxy.Access == "http")
1127       {
1128          // Copy over the environment setting
1129          char S[300];
1130          snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy"));
1131          putenv(S);
1132          putenv((char *)"no_proxy=");
1133
1134          // Run the http method
1135          string Path = flNotFile(argv[0]) + "http";
1136          execl(Path.c_str(),Path.c_str(),(char *)NULL);
1137          cerr << _("Unable to invoke ") << Path << endl;
1138          exit(100);
1139       }
1140    }
1141
1142    FtpMethod Mth;
1143
1144    // no more active ftp, sorry
1145    Mth.DropPrivsOrDie();
1146
1147    return Mth.Run();
1148 }