methods/ftp.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: ftp.cc,v 1.31.2.1 2004/01/16 18:58:50 mdz Exp $
   4 /* ######################################################################
   5
   6    FTP Acquire Method - This is the FTP acquire method for APT.
   7
   8    This is a very simple implementation that does not try to optimize
   9    at all. Commands are sent syncronously with the FTP server (as the
  10    rfc recommends, but it is not really necessary..) and no tricks are
  11    done to speed things along.
  12
  13    RFC 2428 describes the IPv6 FTP behavior
  14
  15    ##################################################################### */
  16                                                                         /*}}}*/
  17 // Include Files                                                        /*{{{*/
  18 #include <config.h>
  19
  20 #include <apt-pkg/fileutl.h>
  21 #include <apt-pkg/acquire-method.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/hashes.h>
  24 #include <apt-pkg/netrc.h>
  25 #include <apt-pkg/configuration.h>
  26 #include <apt-pkg/strutl.h>
  27
  28 #include <ctype.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <sys/stat.h>
  32 #include <sys/time.h>
  33 #include <unistd.h>
  34 #include <signal.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdarg.h>
  38 #include <iostream>
  39
  40 // Internet stuff
  41 #include <netinet/in.h>
  42 #include <sys/socket.h>
  43 #include <arpa/inet.h>
  44 #include <netdb.h>
  45
  46 #include "rfc2553emu.h"
  47 #include "connect.h"
  48 #include "ftp.h"
  49
  50 #include <apti18n.h>
  51                                                                         /*}}}*/
  52
  53 using namespace std;
  54
  55 /* This table is for the EPRT and EPSV commands, it maps the OS address
  56    family to the IETF address families */
  57 struct AFMap
  58 {
  59    unsigned long Family;
  60    unsigned long IETFFamily;
  61 };
  62
  63 #ifndef AF_INET6
  64 struct AFMap AFMap[] = {{AF_INET,1},{0, 0}};
  65 #else
  66 struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{0, 0}};
  67 #endif
  68
  69 unsigned long TimeOut = 120;
  70 URI Proxy;
  71 string FtpMethod::FailFile;
  72 int FtpMethod::FailFd = -1;
  73 time_t FtpMethod::FailTime = 0;
  74
  75 // FTPConn::FTPConn - Constructor                                       /*{{{*/
  76 // ---------------------------------------------------------------------
  77 /* */
  78 FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1),
  79                             DataListenFd(-1), ServerName(Srv),
  80                             ForceExtended(false), TryPassive(true),
  81                             PeerAddrLen(0), ServerAddrLen(0)
  82 {
  83    Debug = _config->FindB("Debug::Acquire::Ftp",false);
  84    PasvAddr = 0;
  85    Buffer[0] = '\0';
  86 }
  87                                                                         /*}}}*/
  88 // FTPConn::~FTPConn - Destructor                                       /*{{{*/
  89 // ---------------------------------------------------------------------
  90 /* */
  91 FTPConn::~FTPConn()
  92 {
  93    Close();
  94 }
  95                                                                         /*}}}*/
  96 // FTPConn::Close - Close down the connection                           /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* Just tear down the socket and data socket */
  99 void FTPConn::Close()
 100 {
 101    close(ServerFd);
 102    ServerFd = -1;
 103    close(DataFd);
 104    DataFd = -1;
 105    close(DataListenFd);
 106    DataListenFd = -1;
 107
 108    if (PasvAddr != 0)
 109       freeaddrinfo(PasvAddr);
 110    PasvAddr = 0;
 111 }
 112                                                                         /*}}}*/
 113 // FTPConn::Open - Open a new connection                                /*{{{*/
 114 // ---------------------------------------------------------------------
 115 /* Connect to the server using a non-blocking connection and perform a
 116    login. */
 117 bool FTPConn::Open(pkgAcqMethod *Owner)
 118 {
 119    // Use the already open connection if possible.
 120    if (ServerFd != -1)
 121       return true;
 122
 123    Close();
 124
 125    // Determine the proxy setting
 126    string SpecificProxy = _config->Find("Acquire::ftp::Proxy::" + ServerName.Host);
 127    if (!SpecificProxy.empty())
 128    {
 129            if (SpecificProxy == "DIRECT")
 130                    Proxy = "";
 131            else
 132                    Proxy = SpecificProxy;
 133    }
 134    else
 135    {
 136            string DefProxy = _config->Find("Acquire::ftp::Proxy");
 137            if (!DefProxy.empty())
 138            {
 139                    Proxy = DefProxy;
 140            }
 141            else
 142            {
 143                    char* result = getenv("ftp_proxy");
 144                    Proxy = result ? result : "";
 145            }
 146    }
 147
 148    // Parse no_proxy, a , separated list of domains
 149    if (getenv("no_proxy") != 0)
 150    {
 151       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 152          Proxy = "";
 153    }
 154
 155    // Determine what host and port to use based on the proxy settings
 156    int Port = 0;
 157    string Host;
 158    if (Proxy.empty() == true)
 159    {
 160       if (ServerName.Port != 0)
 161          Port = ServerName.Port;
 162       Host = ServerName.Host;
 163    }
 164    else
 165    {
 166       if (Proxy.Port != 0)
 167          Port = Proxy.Port;
 168       Host = Proxy.Host;
 169    }
 170
 171    /* Connect to the remote server. Since FTP is connection oriented we
 172       want to make sure we get a new server every time we reconnect */
 173    RotateDNS();
 174    if (Connect(Host,Port,"ftp",21,ServerFd,TimeOut,Owner) == false)
 175       return false;
 176
 177    // Login must be before getpeername otherwise dante won't work.
 178    Owner->Status(_("Logging in"));
 179    bool Res = Login();
 180
 181    // Get the remote server's address
 182    PeerAddrLen = sizeof(PeerAddr);
 183    if (getpeername(ServerFd,(sockaddr *)&PeerAddr,&PeerAddrLen) != 0)
 184       return _error->Errno("getpeername",_("Unable to determine the peer name"));
 185
 186    // Get the local machine's address
 187    ServerAddrLen = sizeof(ServerAddr);
 188    if (getsockname(ServerFd,(sockaddr *)&ServerAddr,&ServerAddrLen) != 0)
 189       return _error->Errno("getsockname",_("Unable to determine the local name"));
 190
 191    return Res;
 192 }
 193                                                                         /*}}}*/
 194 // FTPConn::Login - Login to the remote server                          /*{{{*/
 195 // ---------------------------------------------------------------------
 196 /* This performs both normal login and proxy login using a simples script
 197    stored in the config file. */
 198 bool FTPConn::Login()
 199 {
 200    unsigned int Tag;
 201    string Msg;
 202
 203    // Setup the variables needed for authentication
 204    string User = "anonymous";
 205    string Pass = "apt_get_ftp_2.1@debian.linux.user";
 206
 207    // Fill in the user/pass
 208    if (ServerName.User.empty() == false)
 209       User = ServerName.User;
 210    if (ServerName.Password.empty() == false)
 211       Pass = ServerName.Password;
 212
 213    // Perform simple login
 214    if (Proxy.empty() == true)
 215    {
 216       // Read the initial response
 217       if (ReadResp(Tag,Msg) == false)
 218          return false;
 219       if (Tag >= 400)
 220          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 221
 222       // Send the user
 223       if (WriteMsg(Tag,Msg,"USER %s",User.c_str()) == false)
 224          return false;
 225       if (Tag >= 400)
 226          return _error->Error(_("USER failed, server said: %s"),Msg.c_str());
 227
 228       if (Tag == 331) { // 331 User name okay, need password.
 229          // Send the Password
 230          if (WriteMsg(Tag,Msg,"PASS %s",Pass.c_str()) == false)
 231             return false;
 232          if (Tag >= 400)
 233             return _error->Error(_("PASS failed, server said: %s"),Msg.c_str());
 234       }
 235
 236       // Enter passive mode
 237       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 238          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 239       else
 240          TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 241    }
 242    else
 243    {
 244       // Read the initial response
 245       if (ReadResp(Tag,Msg) == false)
 246          return false;
 247       if (Tag >= 400)
 248          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 249
 250       // Perform proxy script execution
 251       Configuration::Item const *Opts = _config->Tree("Acquire::ftp::ProxyLogin");
 252       if (Opts == 0 || Opts->Child == 0)
 253          return _error->Error(_("A proxy server was specified but no login "
 254                               "script, Acquire::ftp::ProxyLogin is empty."));
 255       Opts = Opts->Child;
 256
 257       // Iterate over the entire login script
 258       for (; Opts != 0; Opts = Opts->Next)
 259       {
 260          if (Opts->Value.empty() == true)
 261             continue;
 262
 263          // Substitute the variables into the command
 264          char SitePort[20];
 265          if (ServerName.Port != 0)
 266             sprintf(SitePort,"%u",ServerName.Port);
 267          else
 268             strcpy(SitePort,"21");
 269          string Tmp = Opts->Value;
 270          Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
 271          Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
 272          Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
 273          Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
 274          Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort);
 275          Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
 276
 277          // Send the command
 278          if (WriteMsg(Tag,Msg,"%s",Tmp.c_str()) == false)
 279             return false;
 280          if (Tag >= 400)
 281             return _error->Error(_("Login script command '%s' failed, server said: %s"),Tmp.c_str(),Msg.c_str());
 282       }
 283
 284       // Enter passive mode
 285       TryPassive = false;
 286       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 287          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 288       else
 289       {
 290          if (_config->Exists("Acquire::FTP::Proxy::Passive") == true)
 291             TryPassive = _config->FindB("Acquire::FTP::Proxy::Passive",true);
 292          else
 293             TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 294       }
 295    }
 296
 297    // Force the use of extended commands
 298    if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true)
 299       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true);
 300    else
 301       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false);
 302
 303    // Binary mode
 304    if (WriteMsg(Tag,Msg,"TYPE I") == false)
 305       return false;
 306    if (Tag >= 400)
 307       return _error->Error(_("TYPE failed, server said: %s"),Msg.c_str());
 308
 309    return true;
 310 }
 311                                                                         /*}}}*/
 312 // FTPConn::ReadLine - Read a line from the server                      /*{{{*/
 313 // ---------------------------------------------------------------------
 314 /* This performs a very simple buffered read. */
 315 bool FTPConn::ReadLine(string &Text)
 316 {
 317    if (ServerFd == -1)
 318       return false;
 319
 320    // Suck in a line
 321    while (Len < sizeof(Buffer))
 322    {
 323       // Scan the buffer for a new line
 324       for (unsigned int I = 0; I != Len; I++)
 325       {
 326          // Escape some special chars
 327          if (Buffer[I] == 0)
 328             Buffer[I] = '?';
 329
 330          // End of line?
 331          if (Buffer[I] != '\n')
 332             continue;
 333
 334          I++;
 335          Text = string(Buffer,I);
 336          memmove(Buffer,Buffer+I,Len - I);
 337          Len -= I;
 338          return true;
 339       }
 340
 341       // Wait for some data..
 342       if (WaitFd(ServerFd,false,TimeOut) == false)
 343       {
 344          Close();
 345          return _error->Error(_("Connection timeout"));
 346       }
 347
 348       // Suck it back
 349       int Res = read(ServerFd,Buffer + Len,sizeof(Buffer) - Len);
 350       if (Res == 0)
 351          _error->Error(_("Server closed the connection"));
 352       if (Res <= 0)
 353       {
 354          _error->Errno("read",_("Read error"));
 355          Close();
 356          return false;
 357       }
 358       Len += Res;
 359    }
 360
 361    return _error->Error(_("A response overflowed the buffer."));
 362 }
 363                                                                         /*}}}*/
 364 // FTPConn::ReadResp - Read a full response from the server             /*{{{*/
 365 // ---------------------------------------------------------------------
 366 /* This reads a reply code from the server, it handles both p */
 367 bool FTPConn::ReadResp(unsigned int &Ret,string &Text)
 368 {
 369    // Grab the first line of the response
 370    string Msg;
 371    if (ReadLine(Msg) == false)
 372        return false;
 373
 374    // Get the ID code
 375    char *End;
 376    Ret = strtol(Msg.c_str(),&End,10);
 377    if (End - Msg.c_str() != 3)
 378       return _error->Error(_("Protocol corruption"));
 379
 380    // All done ?
 381    Text = Msg.c_str()+4;
 382    if (*End == ' ')
 383    {
 384       if (Debug == true)
 385          cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 386       return true;
 387    }
 388
 389    if (*End != '-')
 390       return _error->Error(_("Protocol corruption"));
 391
 392    /* Okay, here we do the continued message trick. This is foolish, but
 393       proftpd follows the protocol as specified and wu-ftpd doesn't, so
 394       we filter. I wonder how many clients break if you use proftpd and
 395       put a '- in the 3rd spot in the message? */
 396    char Leader[4];
 397    strncpy(Leader,Msg.c_str(),3);
 398    Leader[3] = 0;
 399    while (ReadLine(Msg) == true)
 400    {
 401       // Short, it must be using RFC continuation..
 402       if (Msg.length() < 4)
 403       {
 404          Text += Msg;
 405          continue;
 406       }
 407
 408       // Oops, finished
 409       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == ' ')
 410       {
 411          Text += Msg.c_str()+4;
 412          break;
 413       }
 414
 415       // This message has the wu-ftpd style reply code prefixed
 416       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == '-')
 417       {
 418          Text += Msg.c_str()+4;
 419          continue;
 420       }
 421
 422       // Must be RFC style prefixing
 423       Text += Msg;
 424    }
 425
 426    if (Debug == true && _error->PendingError() == false)
 427       cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 428
 429    return !_error->PendingError();
 430 }
 431                                                                         /*}}}*/
 432 // FTPConn::WriteMsg - Send a message to the server                     /*{{{*/
 433 // ---------------------------------------------------------------------
 434 /* Simple printf like function.. */
 435 bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...)
 436 {
 437    va_list args;
 438    va_start(args,Fmt);
 439
 440    // sprintf the description
 441    char S[400];
 442    vsnprintf(S,sizeof(S) - 4,Fmt,args);
 443    strcat(S,"\r\n");
 444    va_end(args);
 445
 446    if (Debug == true)
 447       cerr << "-> '" << QuoteString(S,"") << "'" << endl;
 448
 449    // Send it off
 450    unsigned long Len = strlen(S);
 451    unsigned long Start = 0;
 452    while (Len != 0)
 453    {
 454       if (WaitFd(ServerFd,true,TimeOut) == false)
 455       {
 456          Close();
 457          return _error->Error(_("Connection timeout"));
 458       }
 459
 460       int Res = write(ServerFd,S + Start,Len);
 461       if (Res <= 0)
 462       {
 463          _error->Errno("write",_("Write error"));
 464          Close();
 465          return false;
 466       }
 467
 468       Len -= Res;
 469       Start += Res;
 470    }
 471
 472    return ReadResp(Ret,Text);
 473 }
 474                                                                         /*}}}*/
 475 // FTPConn::GoPasv - Enter Passive mode                                 /*{{{*/
 476 // ---------------------------------------------------------------------
 477 /* Try to enter passive mode, the return code does not indicate if passive
 478    mode could or could not be established, only if there was a fatal error.
 479    We have to enter passive mode every time we make a data connection :| */
 480 bool FTPConn::GoPasv()
 481 {
 482    /* The PASV command only works on IPv4 sockets, even though it could
 483       in theory suppory IPv6 via an all zeros reply */
 484    if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET ||
 485        ForceExtended == true)
 486       return ExtGoPasv();
 487
 488    if (PasvAddr != 0)
 489       freeaddrinfo(PasvAddr);
 490    PasvAddr = 0;
 491
 492    // Try to enable pasv mode
 493    unsigned int Tag;
 494    string Msg;
 495    if (WriteMsg(Tag,Msg,"PASV") == false)
 496       return false;
 497
 498    // Unsupported function
 499    string::size_type Pos = Msg.find('(');
 500    if (Tag >= 400 || Pos == string::npos)
 501       return true;
 502
 503    // Scan it
 504    unsigned a0,a1,a2,a3,p0,p1;
 505    if (sscanf(Msg.c_str() + Pos,"(%u,%u,%u,%u,%u,%u)",&a0,&a1,&a2,&a3,&p0,&p1) != 6)
 506       return true;
 507
 508    /* Some evil servers return 0 to mean their addr. We can actually speak
 509       to these servers natively using IPv6 */
 510    if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0)
 511    {
 512       // Get the IP in text form
 513       char Name[NI_MAXHOST];
 514       char Service[NI_MAXSERV];
 515       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 516                   Name,sizeof(Name),Service,sizeof(Service),
 517                   NI_NUMERICHOST|NI_NUMERICSERV);
 518
 519       struct addrinfo Hints;
 520       memset(&Hints,0,sizeof(Hints));
 521       Hints.ai_socktype = SOCK_STREAM;
 522       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 523       Hints.ai_flags |= AI_NUMERICHOST;
 524
 525       // Get a new passive address.
 526       char Port[100];
 527       snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 528       if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 529          return true;
 530       return true;
 531    }
 532
 533    struct addrinfo Hints;
 534    memset(&Hints,0,sizeof(Hints));
 535    Hints.ai_socktype = SOCK_STREAM;
 536    Hints.ai_family = AF_INET;
 537    Hints.ai_flags |= AI_NUMERICHOST;
 538
 539    // Get a new passive address.
 540    char Port[100];
 541    snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 542    char Name[100];
 543    snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3);
 544    if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 545       return true;
 546    return true;
 547 }
 548                                                                         /*}}}*/
 549 // FTPConn::ExtGoPasv - Enter Extended Passive mode                     /*{{{*/
 550 // ---------------------------------------------------------------------
 551 /* Try to enter extended passive mode. See GoPasv above and RFC 2428 */
 552 bool FTPConn::ExtGoPasv()
 553 {
 554    if (PasvAddr != 0)
 555       freeaddrinfo(PasvAddr);
 556    PasvAddr = 0;
 557
 558    // Try to enable pasv mode
 559    unsigned int Tag;
 560    string Msg;
 561    if (WriteMsg(Tag,Msg,"EPSV") == false)
 562       return false;
 563
 564    // Unsupported function
 565    string::size_type Pos = Msg.find('(');
 566    if (Tag >= 400 || Pos == string::npos)
 567       return true;
 568
 569    // Scan it
 570    string::const_iterator List[4];
 571    unsigned Count = 0;
 572    Pos++;
 573    for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); ++I)
 574    {
 575       if (*I != Msg[Pos])
 576          continue;
 577       if (Count >= 4)
 578          return true;
 579       List[Count++] = I;
 580    }
 581    if (Count != 4)
 582       return true;
 583
 584    // Break it up ..
 585    unsigned long Proto = 0;
 586    unsigned long Port = 0;
 587    string IP;
 588    IP = string(List[1]+1,List[2]);
 589    Port = atoi(string(List[2]+1,List[3]).c_str());
 590    if (IP.empty() == false)
 591       Proto = atoi(string(List[0]+1,List[1]).c_str());
 592
 593    if (Port == 0)
 594       return false;
 595
 596    // String version of the port
 597    char PStr[100];
 598    snprintf(PStr,sizeof(PStr),"%lu",Port);
 599
 600    // Get the IP in text form
 601    struct addrinfo Hints;
 602    memset(&Hints,0,sizeof(Hints));
 603    Hints.ai_socktype = SOCK_STREAM;
 604    Hints.ai_flags |= AI_NUMERICHOST;
 605
 606    /* The RFC defined case, connect to the old IP/protocol using the
 607       new port. */
 608    if (IP.empty() == true)
 609    {
 610       // Get the IP in text form
 611       char Name[NI_MAXHOST];
 612       char Service[NI_MAXSERV];
 613       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 614                   Name,sizeof(Name),Service,sizeof(Service),
 615                   NI_NUMERICHOST|NI_NUMERICSERV);
 616       IP = Name;
 617       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 618    }
 619    else
 620    {
 621       // Get the family..
 622       Hints.ai_family = 0;
 623       for (unsigned J = 0; AFMap[J].Family != 0; J++)
 624          if (AFMap[J].IETFFamily == Proto)
 625             Hints.ai_family = AFMap[J].Family;
 626       if (Hints.ai_family == 0)
 627          return true;
 628    }
 629
 630    // Get a new passive address.
 631    if (getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr) != 0)
 632       return true;
 633
 634    return true;
 635 }
 636                                                                         /*}}}*/
 637 // FTPConn::Size - Return the size of a file                            /*{{{*/
 638 // ---------------------------------------------------------------------
 639 /* Grab the file size from the server, 0 means no size or empty file */
 640 bool FTPConn::Size(const char *Path,unsigned long long &Size)
 641 {
 642    // Query the size
 643    unsigned int Tag;
 644    string Msg;
 645    Size = 0;
 646    if (WriteMsg(Tag,Msg,"SIZE %s",Path) == false)
 647       return false;
 648
 649    char *End;
 650    Size = strtoull(Msg.c_str(),&End,10);
 651    if (Tag >= 400 || End == Msg.c_str())
 652       Size = 0;
 653    return true;
 654 }
 655                                                                         /*}}}*/
 656 // FTPConn::ModTime - Return the modification time of the file          /*{{{*/
 657 // ---------------------------------------------------------------------
 658 /* Like Size no error is returned if the command is not supported. If the
 659    command fails then time is set to the current time of day to fool
 660    date checks. */
 661 bool FTPConn::ModTime(const char *Path, time_t &Time)
 662 {
 663    Time = time(&Time);
 664
 665    // Query the mod time
 666    unsigned int Tag;
 667    string Msg;
 668    if (WriteMsg(Tag,Msg,"MDTM %s",Path) == false)
 669       return false;
 670    if (Tag >= 400 || Msg.empty() == true || isdigit(Msg[0]) == 0)
 671       return true;
 672
 673    // Parse it
 674    return FTPMDTMStrToTime(Msg.c_str(), Time);
 675 }
 676                                                                         /*}}}*/
 677 // FTPConn::CreateDataFd - Get a data connection                        /*{{{*/
 678 // ---------------------------------------------------------------------
 679 /* Create the data connection. Call FinalizeDataFd after this though.. */
 680 bool FTPConn::CreateDataFd()
 681 {
 682    close(DataFd);
 683    DataFd = -1;
 684
 685    // Attempt to enter passive mode.
 686    if (TryPassive == true)
 687    {
 688       if (GoPasv() == false)
 689          return false;
 690
 691       // Oops, didn't work out, don't bother trying again.
 692       if (PasvAddr == 0)
 693          TryPassive = false;
 694    }
 695
 696    // Passive mode?
 697    if (PasvAddr != 0)
 698    {
 699       // Get a socket
 700       if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype,
 701                            PasvAddr->ai_protocol)) < 0)
 702          return _error->Errno("socket",_("Could not create a socket"));
 703
 704       // Connect to the server
 705       SetNonBlock(DataFd,true);
 706       if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 &&
 707           errno != EINPROGRESS)
 708          return _error->Errno("socket",_("Could not create a socket"));
 709
 710       /* This implements a timeout for connect by opening the connection
 711          nonblocking */
 712       if (WaitFd(DataFd,true,TimeOut) == false)
 713          return _error->Error(_("Could not connect data socket, connection timed out"));
 714       unsigned int Err;
 715       unsigned int Len = sizeof(Err);
 716       if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 717          return _error->Errno("getsockopt",_("Failed"));
 718       if (Err != 0)
 719          return _error->Error(_("Could not connect passive socket."));
 720
 721       return true;
 722    }
 723
 724    // Port mode :<
 725    close(DataListenFd);
 726    DataListenFd = -1;
 727
 728    // Get the information for a listening socket.
 729    struct addrinfo *BindAddr = NULL;
 730    struct addrinfo Hints;
 731    memset(&Hints,0,sizeof(Hints));
 732    Hints.ai_socktype = SOCK_STREAM;
 733    Hints.ai_flags |= AI_PASSIVE;
 734    Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family;
 735    if (getaddrinfo(0,"0",&Hints,&BindAddr) != 0 || BindAddr == NULL)
 736       return _error->Error(_("getaddrinfo was unable to get a listening socket"));
 737
 738    // Construct the socket
 739    if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype,
 740                               BindAddr->ai_protocol)) < 0)
 741    {
 742       freeaddrinfo(BindAddr);
 743       return _error->Errno("socket",_("Could not create a socket"));
 744    }
 745
 746    // Bind and listen
 747    if (bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0)
 748    {
 749       freeaddrinfo(BindAddr);
 750       return _error->Errno("bind",_("Could not bind a socket"));
 751    }
 752    freeaddrinfo(BindAddr);
 753    if (listen(DataListenFd,1) < 0)
 754       return _error->Errno("listen",_("Could not listen on the socket"));
 755    SetNonBlock(DataListenFd,true);
 756
 757    // Determine the name to send to the remote
 758    struct sockaddr_storage Addr;
 759    socklen_t AddrLen = sizeof(Addr);
 760    if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0)
 761       return _error->Errno("getsockname",_("Could not determine the socket's name"));
 762
 763
 764    // Reverse the address. We need the server address and the data port.
 765    char Name[NI_MAXHOST];
 766    char Service[NI_MAXSERV];
 767    char Service2[NI_MAXSERV];
 768    getnameinfo((struct sockaddr *)&Addr,AddrLen,
 769                Name,sizeof(Name),Service,sizeof(Service),
 770                NI_NUMERICHOST|NI_NUMERICSERV);
 771    getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen,
 772                Name,sizeof(Name),Service2,sizeof(Service2),
 773                NI_NUMERICHOST|NI_NUMERICSERV);
 774
 775    // Send off an IPv4 address in the old port format
 776    if (((struct sockaddr *)&Addr)->sa_family == AF_INET &&
 777        ForceExtended == false)
 778    {
 779       // Convert the dots in the quad into commas
 780       for (char *I = Name; *I != 0; I++)
 781          if (*I == '.')
 782             *I = ',';
 783       unsigned long Port = atoi(Service);
 784
 785       // Send the port command
 786       unsigned int Tag;
 787       string Msg;
 788       if (WriteMsg(Tag,Msg,"PORT %s,%d,%d",
 789                    Name,
 790                    (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false)
 791          return false;
 792       if (Tag >= 400)
 793          return _error->Error(_("Unable to send PORT command"));
 794       return true;
 795    }
 796
 797    // Construct an EPRT command
 798    unsigned Proto = 0;
 799    for (unsigned J = 0; AFMap[J].Family != 0; J++)
 800       if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family)
 801          Proto = AFMap[J].IETFFamily;
 802    if (Proto == 0)
 803       return _error->Error(_("Unknown address family %u (AF_*)"),
 804                            ((struct sockaddr *)&Addr)->sa_family);
 805
 806    // Send the EPRT command
 807    unsigned int Tag;
 808    string Msg;
 809    if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false)
 810       return false;
 811    if (Tag >= 400)
 812       return _error->Error(_("EPRT failed, server said: %s"),Msg.c_str());
 813    return true;
 814 }
 815                                                                         /*}}}*/
 816 // FTPConn::Finalize - Complete the Data connection                     /*{{{*/
 817 // ---------------------------------------------------------------------
 818 /* If the connection is in port mode this waits for the other end to hook
 819    up to us. */
 820 bool FTPConn::Finalize()
 821 {
 822    // Passive mode? Do nothing
 823    if (PasvAddr != 0)
 824       return true;
 825
 826    // Close any old socket..
 827    close(DataFd);
 828    DataFd = -1;
 829
 830    // Wait for someone to connect..
 831    if (WaitFd(DataListenFd,false,TimeOut) == false)
 832       return _error->Error(_("Data socket connect timed out"));
 833
 834    // Accept the connection
 835    struct sockaddr_in Addr;
 836    socklen_t Len = sizeof(Addr);
 837    DataFd = accept(DataListenFd,(struct sockaddr *)&Addr,&Len);
 838    if (DataFd < 0)
 839       return _error->Errno("accept",_("Unable to accept connection"));
 840
 841    close(DataListenFd);
 842    DataListenFd = -1;
 843
 844    return true;
 845 }
 846                                                                         /*}}}*/
 847 // FTPConn::Get - Get a file                                            /*{{{*/
 848 // ---------------------------------------------------------------------
 849 /* This opens a data connection, sends REST and RETR and then
 850    transfers the file over. */
 851 bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
 852                   Hashes &Hash,bool &Missing, unsigned long long MaximumSize)
 853 {
 854    Missing = false;
 855    if (CreateDataFd() == false)
 856       return false;
 857
 858    unsigned int Tag;
 859    string Msg;
 860    if (Resume != 0)
 861    {
 862       if (WriteMsg(Tag,Msg,"REST %u",Resume) == false)
 863          return false;
 864       if (Tag >= 400)
 865          Resume = 0;
 866    }
 867
 868    if (To.Truncate(Resume) == false)
 869       return false;
 870
 871    if (To.Seek(0) == false)
 872       return false;
 873
 874    if (Resume != 0)
 875    {
 876       if (Hash.AddFD(To,Resume) == false)
 877       {
 878          _error->Errno("read",_("Problem hashing file"));
 879          return false;
 880       }
 881    }
 882
 883    // Send the get command
 884    if (WriteMsg(Tag,Msg,"RETR %s",Path) == false)
 885       return false;
 886
 887    if (Tag >= 400)
 888    {
 889       if (Tag == 550)
 890          Missing = true;
 891       return _error->Error(_("Unable to fetch file, server said '%s'"),Msg.c_str());
 892    }
 893
 894    // Finish off the data connection
 895    if (Finalize() == false)
 896       return false;
 897
 898    // Copy loop
 899    unsigned char Buffer[4096];
 900    while (1)
 901    {
 902       // Wait for some data..
 903       if (WaitFd(DataFd,false,TimeOut) == false)
 904       {
 905          Close();
 906          return _error->Error(_("Data socket timed out"));
 907       }
 908
 909       // Read the data..
 910       int Res = read(DataFd,Buffer,sizeof(Buffer));
 911       if (Res == 0)
 912          break;
 913       if (Res < 0)
 914       {
 915          if (errno == EAGAIN)
 916             continue;
 917          break;
 918       }
 919
 920       Hash.Add(Buffer,Res);
 921       if (To.Write(Buffer,Res) == false)
 922       {
 923          Close();
 924          return false;
 925       }
 926
 927       if (MaximumSize > 0 && To.Tell() > MaximumSize)
 928          return _error->Error("Writing more data than expected (%llu > %llu)",
 929                               To.Tell(), MaximumSize);
 930    }
 931
 932    // All done
 933    close(DataFd);
 934    DataFd = -1;
 935
 936    // Read the closing message from the server
 937    if (ReadResp(Tag,Msg) == false)
 938       return false;
 939    if (Tag >= 400)
 940       return _error->Error(_("Data transfer failed, server said '%s'"),Msg.c_str());
 941    return true;
 942 }
 943                                                                         /*}}}*/
 944
 945 // FtpMethod::FtpMethod - Constructor                                   /*{{{*/
 946 // ---------------------------------------------------------------------
 947 /* */
 948 FtpMethod::FtpMethod() : pkgAcqMethod("1.0",SendConfig)
 949 {
 950    signal(SIGTERM,SigTerm);
 951    signal(SIGINT,SigTerm);
 952
 953    Server = 0;
 954    FailFd = -1;
 955 }
 956                                                                         /*}}}*/
 957 // FtpMethod::SigTerm - Handle a fatal signal                           /*{{{*/
 958 // ---------------------------------------------------------------------
 959 /* This closes and timestamps the open file. This is necessary to get
 960    resume behavoir on user abort */
 961 void FtpMethod::SigTerm(int)
 962 {
 963    if (FailFd == -1)
 964       _exit(100);
 965
 966    // Timestamp
 967    struct timeval times[2];
 968    times[0].tv_sec = FailTime;
 969    times[1].tv_sec = FailTime;
 970    times[0].tv_usec = times[1].tv_usec = 0;
 971    utimes(FailFile.c_str(), times);
 972
 973    close(FailFd);
 974
 975    _exit(100);
 976 }
 977                                                                         /*}}}*/
 978 // FtpMethod::Configuration - Handle a configuration message            /*{{{*/
 979 // ---------------------------------------------------------------------
 980 /* We stash the desired pipeline depth */
 981 bool FtpMethod::Configuration(string Message)
 982 {
 983    if (pkgAcqMethod::Configuration(Message) == false)
 984       return false;
 985
 986    TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut);
 987    return true;
 988 }
 989                                                                         /*}}}*/
 990 // FtpMethod::Fetch - Fetch a file                                      /*{{{*/
 991 // ---------------------------------------------------------------------
 992 /* Fetch a single file, called by the base class..  */
 993 bool FtpMethod::Fetch(FetchItem *Itm)
 994 {
 995    URI Get = Itm->Uri;
 996    const char *File = Get.Path.c_str();
 997    FetchResult Res;
 998    Res.Filename = Itm->DestFile;
 999    Res.IMSHit = false;
1000
1001    maybe_add_auth (Get, _config->FindFile("Dir::Etc::netrc"));
1002
1003    // Connect to the server
1004    if (Server == 0 || Server->Comp(Get) == false)
1005    {
1006       delete Server;
1007       Server = new FTPConn(Get);
1008    }
1009
1010    // Could not connect is a transient error..
1011    if (Server->Open(this) == false)
1012    {
1013       Server->Close();
1014       Fail(true);
1015       return true;
1016    }
1017
1018    // Get the files information
1019    Status(_("Query"));
1020    unsigned long long Size;
1021    if (Server->Size(File,Size) == false ||
1022        Server->ModTime(File,FailTime) == false)
1023    {
1024       Fail(true);
1025       return true;
1026    }
1027    Res.Size = Size;
1028
1029    // See if it is an IMS hit
1030    if (Itm->LastModified == FailTime)
1031    {
1032       Res.Size = 0;
1033       Res.IMSHit = true;
1034       URIDone(Res);
1035       return true;
1036    }
1037
1038    // See if the file exists
1039    struct stat Buf;
1040    if (stat(Itm->DestFile.c_str(),&Buf) == 0)
1041    {
1042       if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime)
1043       {
1044          Res.Size = Buf.st_size;
1045          Res.LastModified = Buf.st_mtime;
1046          Res.ResumePoint = Buf.st_size;
1047          URIDone(Res);
1048          return true;
1049       }
1050
1051       // Resume?
1052       if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size)
1053          Res.ResumePoint = Buf.st_size;
1054    }
1055
1056    // Open the file
1057    Hashes Hash;
1058    {
1059       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
1060       if (_error->PendingError() == true)
1061          return false;
1062
1063       URIStart(Res);
1064
1065       FailFile = Itm->DestFile;
1066       FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1067       FailFd = Fd.Fd();
1068
1069       bool Missing;
1070       if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->MaximumSize) == false)
1071       {
1072          Fd.Close();
1073
1074          // Timestamp
1075          struct timeval times[2];
1076          times[0].tv_sec = FailTime;
1077          times[1].tv_sec = FailTime;
1078          times[0].tv_usec = times[1].tv_usec = 0;
1079          utimes(FailFile.c_str(), times);
1080
1081          // If the file is missing we hard fail and delete the destfile
1082          // otherwise transient fail
1083          if (Missing == true) {
1084             unlink(FailFile.c_str());
1085             return false;
1086          }
1087          Fail(true);
1088          return true;
1089       }
1090
1091       Res.Size = Fd.Size();
1092
1093       // Timestamp
1094       struct timeval times[2];
1095       times[0].tv_sec = FailTime;
1096       times[1].tv_sec = FailTime;
1097       times[0].tv_usec = times[1].tv_usec = 0;
1098       utimes(Fd.Name().c_str(), times);
1099       FailFd = -1;
1100    }
1101
1102    Res.LastModified = FailTime;
1103    Res.TakeHashes(Hash);
1104
1105    URIDone(Res);
1106
1107    return true;
1108 }
1109                                                                         /*}}}*/
1110
1111 int main(int, const char *argv[])
1112 {
1113    setlocale(LC_ALL, "");
1114
1115    /* See if we should be come the http client - we do this for http
1116       proxy urls */
1117    if (getenv("ftp_proxy") != 0)
1118    {
1119       URI Proxy = string(getenv("ftp_proxy"));
1120
1121       // Run the HTTP method
1122       if (Proxy.Access == "http")
1123       {
1124          // Copy over the environment setting
1125          char S[300];
1126          snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy"));
1127          putenv(S);
1128          putenv((char *)"no_proxy=");
1129
1130          // Run the http method
1131          string Path = flNotFile(argv[0]) + "http";
1132          execl(Path.c_str(),Path.c_str(),(char *)NULL);
1133          cerr << _("Unable to invoke ") << Path << endl;
1134          exit(100);
1135       }
1136    }
1137
1138    FtpMethod Mth;
1139
1140    // no more active ftp, sorry
1141    Mth.DropPrivsOrDie();
1142
1143    return Mth.Run();
1144 }