methods/ftp.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: ftp.cc,v 1.31.2.1 2004/01/16 18:58:50 mdz Exp $
   4 /* ######################################################################
   5
   6    FTP Acquire Method - This is the FTP acquire method for APT.
   7
   8    This is a very simple implementation that does not try to optimize
   9    at all. Commands are sent syncronously with the FTP server (as the
  10    rfc recommends, but it is not really necessary..) and no tricks are
  11    done to speed things along.
  12
  13    RFC 2428 describes the IPv6 FTP behavior
  14
  15    ##################################################################### */
  16                                                                         /*}}}*/
  17 // Include Files                                                        /*{{{*/
  18 #include <config.h>
  19
  20 #include <apt-pkg/fileutl.h>
  21 #include <apt-pkg/acquire-method.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/hashes.h>
  24 #include <apt-pkg/netrc.h>
  25 #include <apt-pkg/configuration.h>
  26 #include <apt-pkg/strutl.h>
  27
  28 #include <ctype.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <sys/stat.h>
  32 #include <sys/time.h>
  33 #include <unistd.h>
  34 #include <signal.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdarg.h>
  38 #include <iostream>
  39
  40 // Internet stuff
  41 #include <netinet/in.h>
  42 #include <sys/socket.h>
  43 #include <arpa/inet.h>
  44 #include <netdb.h>
  45
  46 #include "rfc2553emu.h"
  47 #include "connect.h"
  48 #include "ftp.h"
  49
  50 #include <apti18n.h>
  51                                                                         /*}}}*/
  52
  53 using namespace std;
  54
  55 /* This table is for the EPRT and EPSV commands, it maps the OS address
  56    family to the IETF address families */
  57 struct AFMap
  58 {
  59    unsigned long Family;
  60    unsigned long IETFFamily;
  61 };
  62
  63 #ifndef AF_INET6
  64 struct AFMap AFMap[] = {{AF_INET,1},{0, 0}};
  65 #else
  66 struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{0, 0}};
  67 #endif
  68
  69 unsigned long TimeOut = 120;
  70 URI Proxy;
  71 string FtpMethod::FailFile;
  72 int FtpMethod::FailFd = -1;
  73 time_t FtpMethod::FailTime = 0;
  74
  75 // FTPConn::FTPConn - Constructor                                       /*{{{*/
  76 // ---------------------------------------------------------------------
  77 /* */
  78 FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1),
  79                             DataListenFd(-1), ServerName(Srv),
  80                             ForceExtended(false), TryPassive(true),
  81                             PeerAddrLen(0), ServerAddrLen(0)
  82 {
  83    Debug = _config->FindB("Debug::Acquire::Ftp",false);
  84    PasvAddr = 0;
  85    Buffer[0] = '\0';
  86 }
  87                                                                         /*}}}*/
  88 // FTPConn::~FTPConn - Destructor                                       /*{{{*/
  89 // ---------------------------------------------------------------------
  90 /* */
  91 FTPConn::~FTPConn()
  92 {
  93    Close();
  94 }
  95                                                                         /*}}}*/
  96 // FTPConn::Close - Close down the connection                           /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* Just tear down the socket and data socket */
  99 void FTPConn::Close()
 100 {
 101    close(ServerFd);
 102    ServerFd = -1;
 103    close(DataFd);
 104    DataFd = -1;
 105    close(DataListenFd);
 106    DataListenFd = -1;
 107
 108    if (PasvAddr != 0)
 109       freeaddrinfo(PasvAddr);
 110    PasvAddr = 0;
 111 }
 112                                                                         /*}}}*/
 113 // FTPConn::Open - Open a new connection                                /*{{{*/
 114 // ---------------------------------------------------------------------
 115 /* Connect to the server using a non-blocking connection and perform a
 116    login. */
 117 bool FTPConn::Open(pkgAcqMethod *Owner)
 118 {
 119    // Use the already open connection if possible.
 120    if (ServerFd != -1)
 121       return true;
 122
 123    Close();
 124
 125    // Determine the proxy setting
 126    string SpecificProxy = _config->Find("Acquire::ftp::Proxy::" + ServerName.Host);
 127    if (!SpecificProxy.empty())
 128    {
 129            if (SpecificProxy == "DIRECT")
 130                    Proxy = "";
 131            else
 132                    Proxy = SpecificProxy;
 133    }
 134    else
 135    {
 136            string DefProxy = _config->Find("Acquire::ftp::Proxy");
 137            if (!DefProxy.empty())
 138            {
 139                    Proxy = DefProxy;
 140            }
 141            else
 142            {
 143                    char* result = getenv("ftp_proxy");
 144                    Proxy = result ? result : "";
 145            }
 146    }
 147
 148    // Parse no_proxy, a , separated list of domains
 149    if (getenv("no_proxy") != 0)
 150    {
 151       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 152          Proxy = "";
 153    }
 154
 155    // Determine what host and port to use based on the proxy settings
 156    int Port = 0;
 157    string Host;
 158    if (Proxy.empty() == true)
 159    {
 160       if (ServerName.Port != 0)
 161          Port = ServerName.Port;
 162       Host = ServerName.Host;
 163    }
 164    else
 165    {
 166       if (Proxy.Port != 0)
 167          Port = Proxy.Port;
 168       Host = Proxy.Host;
 169    }
 170
 171    /* Connect to the remote server. Since FTP is connection oriented we
 172       want to make sure we get a new server every time we reconnect */
 173    RotateDNS();
 174    if (Connect(Host,Port,"ftp",21,ServerFd,TimeOut,Owner) == false)
 175       return false;
 176
 177    // Login must be before getpeername otherwise dante won't work.
 178    Owner->Status(_("Logging in"));
 179    bool Res = Login();
 180
 181    // Get the remote server's address
 182    PeerAddrLen = sizeof(PeerAddr);
 183    if (getpeername(ServerFd,(sockaddr *)&PeerAddr,&PeerAddrLen) != 0)
 184       return _error->Errno("getpeername",_("Unable to determine the peer name"));
 185
 186    // Get the local machine's address
 187    ServerAddrLen = sizeof(ServerAddr);
 188    if (getsockname(ServerFd,(sockaddr *)&ServerAddr,&ServerAddrLen) != 0)
 189       return _error->Errno("getsockname",_("Unable to determine the local name"));
 190
 191    return Res;
 192 }
 193                                                                         /*}}}*/
 194 // FTPConn::Login - Login to the remote server                          /*{{{*/
 195 // ---------------------------------------------------------------------
 196 /* This performs both normal login and proxy login using a simples script
 197    stored in the config file. */
 198 bool FTPConn::Login()
 199 {
 200    unsigned int Tag;
 201    string Msg;
 202
 203    // Setup the variables needed for authentication
 204    string User = "anonymous";
 205    string Pass = "apt_get_ftp_2.1@debian.linux.user";
 206
 207    // Fill in the user/pass
 208    if (ServerName.User.empty() == false)
 209       User = ServerName.User;
 210    if (ServerName.Password.empty() == false)
 211       Pass = ServerName.Password;
 212
 213    // Perform simple login
 214    if (Proxy.empty() == true)
 215    {
 216       // Read the initial response
 217       if (ReadResp(Tag,Msg) == false)
 218          return false;
 219       if (Tag >= 400)
 220          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 221
 222       // Send the user
 223       if (WriteMsg(Tag,Msg,"USER %s",User.c_str()) == false)
 224          return false;
 225       if (Tag >= 400)
 226          return _error->Error(_("USER failed, server said: %s"),Msg.c_str());
 227
 228       if (Tag == 331) { // 331 User name okay, need password.
 229          // Send the Password
 230          if (WriteMsg(Tag,Msg,"PASS %s",Pass.c_str()) == false)
 231             return false;
 232          if (Tag >= 400)
 233             return _error->Error(_("PASS failed, server said: %s"),Msg.c_str());
 234       }
 235
 236       // Enter passive mode
 237       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 238          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 239       else
 240          TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 241    }
 242    else
 243    {
 244       // Read the initial response
 245       if (ReadResp(Tag,Msg) == false)
 246          return false;
 247       if (Tag >= 400)
 248          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 249
 250       // Perform proxy script execution
 251       Configuration::Item const *Opts = _config->Tree("Acquire::ftp::ProxyLogin");
 252       if (Opts == 0 || Opts->Child == 0)
 253          return _error->Error(_("A proxy server was specified but no login "
 254                               "script, Acquire::ftp::ProxyLogin is empty."));
 255       Opts = Opts->Child;
 256
 257       // Iterate over the entire login script
 258       for (; Opts != 0; Opts = Opts->Next)
 259       {
 260          if (Opts->Value.empty() == true)
 261             continue;
 262
 263          // Substitute the variables into the command
 264          char SitePort[20];
 265          if (ServerName.Port != 0)
 266             sprintf(SitePort,"%u",ServerName.Port);
 267          else
 268             strcpy(SitePort,"21");
 269          string Tmp = Opts->Value;
 270          Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
 271          Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
 272          Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
 273          Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
 274          Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort);
 275          Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
 276
 277          // Send the command
 278          if (WriteMsg(Tag,Msg,"%s",Tmp.c_str()) == false)
 279             return false;
 280          if (Tag >= 400)
 281             return _error->Error(_("Login script command '%s' failed, server said: %s"),Tmp.c_str(),Msg.c_str());
 282       }
 283
 284       // Enter passive mode
 285       TryPassive = false;
 286       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 287          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 288       else
 289       {
 290          if (_config->Exists("Acquire::FTP::Proxy::Passive") == true)
 291             TryPassive = _config->FindB("Acquire::FTP::Proxy::Passive",true);
 292          else
 293             TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 294       }
 295    }
 296
 297    // Force the use of extended commands
 298    if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true)
 299       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true);
 300    else
 301       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false);
 302
 303    // Binary mode
 304    if (WriteMsg(Tag,Msg,"TYPE I") == false)
 305       return false;
 306    if (Tag >= 400)
 307       return _error->Error(_("TYPE failed, server said: %s"),Msg.c_str());
 308
 309    return true;
 310 }
 311                                                                         /*}}}*/
 312 // FTPConn::ReadLine - Read a line from the server                      /*{{{*/
 313 // ---------------------------------------------------------------------
 314 /* This performs a very simple buffered read. */
 315 bool FTPConn::ReadLine(string &Text)
 316 {
 317    if (ServerFd == -1)
 318       return false;
 319
 320    // Suck in a line
 321    while (Len < sizeof(Buffer))
 322    {
 323       // Scan the buffer for a new line
 324       for (unsigned int I = 0; I != Len; I++)
 325       {
 326          // Escape some special chars
 327          if (Buffer[I] == 0)
 328             Buffer[I] = '?';
 329
 330          // End of line?
 331          if (Buffer[I] != '\n')
 332             continue;
 333
 334          I++;
 335          Text = string(Buffer,I);
 336          memmove(Buffer,Buffer+I,Len - I);
 337          Len -= I;
 338          return true;
 339       }
 340
 341       // Wait for some data..
 342       if (WaitFd(ServerFd,false,TimeOut) == false)
 343       {
 344          Close();
 345          return _error->Error(_("Connection timeout"));
 346       }
 347
 348       // Suck it back
 349       int Res = read(ServerFd,Buffer + Len,sizeof(Buffer) - Len);
 350       if (Res == 0)
 351          _error->Error(_("Server closed the connection"));
 352       if (Res <= 0)
 353       {
 354          _error->Errno("read",_("Read error"));
 355          Close();
 356          return false;
 357       }
 358       Len += Res;
 359    }
 360
 361    return _error->Error(_("A response overflowed the buffer."));
 362 }
 363                                                                         /*}}}*/
 364 // FTPConn::ReadResp - Read a full response from the server             /*{{{*/
 365 // ---------------------------------------------------------------------
 366 /* This reads a reply code from the server, it handles both p */
 367 bool FTPConn::ReadResp(unsigned int &Ret,string &Text)
 368 {
 369    // Grab the first line of the response
 370    string Msg;
 371    if (ReadLine(Msg) == false)
 372        return false;
 373
 374    // Get the ID code
 375    char *End;
 376    Ret = strtol(Msg.c_str(),&End,10);
 377    if (End - Msg.c_str() != 3)
 378       return _error->Error(_("Protocol corruption"));
 379
 380    // All done ?
 381    Text = Msg.c_str()+4;
 382    if (*End == ' ')
 383    {
 384       if (Debug == true)
 385          cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 386       return true;
 387    }
 388
 389    if (*End != '-')
 390       return _error->Error(_("Protocol corruption"));
 391
 392    /* Okay, here we do the continued message trick. This is foolish, but
 393       proftpd follows the protocol as specified and wu-ftpd doesn't, so
 394       we filter. I wonder how many clients break if you use proftpd and
 395       put a '- in the 3rd spot in the message? */
 396    char Leader[4];
 397    strncpy(Leader,Msg.c_str(),3);
 398    Leader[3] = 0;
 399    while (ReadLine(Msg) == true)
 400    {
 401       // Short, it must be using RFC continuation..
 402       if (Msg.length() < 4)
 403       {
 404          Text += Msg;
 405          continue;
 406       }
 407
 408       // Oops, finished
 409       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == ' ')
 410       {
 411          Text += Msg.c_str()+4;
 412          break;
 413       }
 414
 415       // This message has the wu-ftpd style reply code prefixed
 416       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == '-')
 417       {
 418          Text += Msg.c_str()+4;
 419          continue;
 420       }
 421
 422       // Must be RFC style prefixing
 423       Text += Msg;
 424    }
 425
 426    if (Debug == true && _error->PendingError() == false)
 427       cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 428
 429    return !_error->PendingError();
 430 }
 431                                                                         /*}}}*/
 432 // FTPConn::WriteMsg - Send a message to the server                     /*{{{*/
 433 // ---------------------------------------------------------------------
 434 /* Simple printf like function.. */
 435 bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...)
 436 {
 437    va_list args;
 438    va_start(args,Fmt);
 439
 440    // sprintf the description
 441    char S[400];
 442    vsnprintf(S,sizeof(S) - 4,Fmt,args);
 443    strcat(S,"\r\n");
 444    va_end(args);
 445
 446    if (Debug == true)
 447       cerr << "-> '" << QuoteString(S,"") << "'" << endl;
 448
 449    // Send it off
 450    unsigned long Len = strlen(S);
 451    unsigned long Start = 0;
 452    while (Len != 0)
 453    {
 454       if (WaitFd(ServerFd,true,TimeOut) == false)
 455       {
 456          Close();
 457          return _error->Error(_("Connection timeout"));
 458       }
 459
 460       int Res = write(ServerFd,S + Start,Len);
 461       if (Res <= 0)
 462       {
 463          _error->Errno("write",_("Write error"));
 464          Close();
 465          return false;
 466       }
 467
 468       Len -= Res;
 469       Start += Res;
 470    }
 471
 472    return ReadResp(Ret,Text);
 473 }
 474                                                                         /*}}}*/
 475 // FTPConn::GoPasv - Enter Passive mode                                 /*{{{*/
 476 // ---------------------------------------------------------------------
 477 /* Try to enter passive mode, the return code does not indicate if passive
 478    mode could or could not be established, only if there was a fatal error.
 479    We have to enter passive mode every time we make a data connection :| */
 480 bool FTPConn::GoPasv()
 481 {
 482    /* The PASV command only works on IPv4 sockets, even though it could
 483       in theory suppory IPv6 via an all zeros reply */
 484    if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET ||
 485        ForceExtended == true)
 486       return ExtGoPasv();
 487
 488    if (PasvAddr != 0)
 489       freeaddrinfo(PasvAddr);
 490    PasvAddr = 0;
 491
 492    // Try to enable pasv mode
 493    unsigned int Tag;
 494    string Msg;
 495    if (WriteMsg(Tag,Msg,"PASV") == false)
 496       return false;
 497
 498    // Unsupported function
 499    string::size_type Pos = Msg.find('(');
 500    if (Tag >= 400 || Pos == string::npos)
 501       return true;
 502
 503    // Scan it
 504    unsigned a0,a1,a2,a3,p0,p1;
 505    if (sscanf(Msg.c_str() + Pos,"(%u,%u,%u,%u,%u,%u)",&a0,&a1,&a2,&a3,&p0,&p1) != 6)
 506       return true;
 507
 508    /* Some evil servers return 0 to mean their addr. We can actually speak
 509       to these servers natively using IPv6 */
 510    if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0)
 511    {
 512       // Get the IP in text form
 513       char Name[NI_MAXHOST];
 514       char Service[NI_MAXSERV];
 515       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 516                   Name,sizeof(Name),Service,sizeof(Service),
 517                   NI_NUMERICHOST|NI_NUMERICSERV);
 518
 519       struct addrinfo Hints;
 520       memset(&Hints,0,sizeof(Hints));
 521       Hints.ai_socktype = SOCK_STREAM;
 522       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 523       Hints.ai_flags |= AI_NUMERICHOST;
 524
 525       // Get a new passive address.
 526       char Port[100];
 527       snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 528       if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 529          return true;
 530       return true;
 531    }
 532
 533    struct addrinfo Hints;
 534    memset(&Hints,0,sizeof(Hints));
 535    Hints.ai_socktype = SOCK_STREAM;
 536    Hints.ai_family = AF_INET;
 537    Hints.ai_flags |= AI_NUMERICHOST;
 538
 539    // Get a new passive address.
 540    char Port[100];
 541    snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 542    char Name[100];
 543    snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3);
 544    if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 545       return true;
 546    return true;
 547 }
 548                                                                         /*}}}*/
 549 // FTPConn::ExtGoPasv - Enter Extended Passive mode                     /*{{{*/
 550 // ---------------------------------------------------------------------
 551 /* Try to enter extended passive mode. See GoPasv above and RFC 2428 */
 552 bool FTPConn::ExtGoPasv()
 553 {
 554    if (PasvAddr != 0)
 555       freeaddrinfo(PasvAddr);
 556    PasvAddr = 0;
 557
 558    // Try to enable pasv mode
 559    unsigned int Tag;
 560    string Msg;
 561    if (WriteMsg(Tag,Msg,"EPSV") == false)
 562       return false;
 563
 564    // Unsupported function
 565    string::size_type Pos = Msg.find('(');
 566    if (Tag >= 400 || Pos == string::npos)
 567       return true;
 568
 569    // Scan it
 570    string::const_iterator List[4];
 571    unsigned Count = 0;
 572    Pos++;
 573    for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); ++I)
 574    {
 575       if (*I != Msg[Pos])
 576          continue;
 577       if (Count >= 4)
 578          return true;
 579       List[Count++] = I;
 580    }
 581    if (Count != 4)
 582       return true;
 583
 584    // Break it up ..
 585    unsigned long Proto = 0;
 586    unsigned long Port = 0;
 587    string IP;
 588    IP = string(List[1]+1,List[2]);
 589    Port = atoi(string(List[2]+1,List[3]).c_str());
 590    if (IP.empty() == false)
 591       Proto = atoi(string(List[0]+1,List[1]).c_str());
 592
 593    if (Port == 0)
 594       return false;
 595
 596    // String version of the port
 597    char PStr[100];
 598    snprintf(PStr,sizeof(PStr),"%lu",Port);
 599
 600    // Get the IP in text form
 601    struct addrinfo Hints;
 602    memset(&Hints,0,sizeof(Hints));
 603    Hints.ai_socktype = SOCK_STREAM;
 604    Hints.ai_flags |= AI_NUMERICHOST;
 605
 606    /* The RFC defined case, connect to the old IP/protocol using the
 607       new port. */
 608    if (IP.empty() == true)
 609    {
 610       // Get the IP in text form
 611       char Name[NI_MAXHOST];
 612       char Service[NI_MAXSERV];
 613       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 614                   Name,sizeof(Name),Service,sizeof(Service),
 615                   NI_NUMERICHOST|NI_NUMERICSERV);
 616       IP = Name;
 617       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 618    }
 619    else
 620    {
 621       // Get the family..
 622       Hints.ai_family = 0;
 623       for (unsigned J = 0; AFMap[J].Family != 0; J++)
 624          if (AFMap[J].IETFFamily == Proto)
 625             Hints.ai_family = AFMap[J].Family;
 626       if (Hints.ai_family == 0)
 627          return true;
 628    }
 629
 630    // Get a new passive address.
 631    if (getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr) != 0)
 632       return true;
 633
 634    return true;
 635 }
 636                                                                         /*}}}*/
 637 // FTPConn::Size - Return the size of a file                            /*{{{*/
 638 // ---------------------------------------------------------------------
 639 /* Grab the file size from the server, 0 means no size or empty file */
 640 bool FTPConn::Size(const char *Path,unsigned long long &Size)
 641 {
 642    // Query the size
 643    unsigned int Tag;
 644    string Msg;
 645    Size = 0;
 646    if (WriteMsg(Tag,Msg,"SIZE %s",Path) == false)
 647       return false;
 648
 649    char *End;
 650    Size = strtoull(Msg.c_str(),&End,10);
 651    if (Tag >= 400 || End == Msg.c_str())
 652       Size = 0;
 653    return true;
 654 }
 655                                                                         /*}}}*/
 656 // FTPConn::ModTime - Return the modification time of the file          /*{{{*/
 657 // ---------------------------------------------------------------------
 658 /* Like Size no error is returned if the command is not supported. If the
 659    command fails then time is set to the current time of day to fool
 660    date checks. */
 661 bool FTPConn::ModTime(const char *Path, time_t &Time)
 662 {
 663    Time = time(&Time);
 664
 665    // Query the mod time
 666    unsigned int Tag;
 667    string Msg;
 668    if (WriteMsg(Tag,Msg,"MDTM %s",Path) == false)
 669       return false;
 670    if (Tag >= 400 || Msg.empty() == true || isdigit(Msg[0]) == 0)
 671       return true;
 672
 673    // Parse it
 674    return FTPMDTMStrToTime(Msg.c_str(), Time);
 675 }
 676                                                                         /*}}}*/
 677 // FTPConn::CreateDataFd - Get a data connection                        /*{{{*/
 678 // ---------------------------------------------------------------------
 679 /* Create the data connection. Call FinalizeDataFd after this though.. */
 680 bool FTPConn::CreateDataFd()
 681 {
 682    close(DataFd);
 683    DataFd = -1;
 684
 685    // Attempt to enter passive mode.
 686    if (TryPassive == true)
 687    {
 688       if (GoPasv() == false)
 689          return false;
 690
 691       // Oops, didn't work out, don't bother trying again.
 692       if (PasvAddr == 0)
 693          TryPassive = false;
 694    }
 695
 696    // Passive mode?
 697    if (PasvAddr != 0)
 698    {
 699       // Get a socket
 700       if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype,
 701                            PasvAddr->ai_protocol)) < 0)
 702          return _error->Errno("socket",_("Could not create a socket"));
 703
 704       // Connect to the server
 705       SetNonBlock(DataFd,true);
 706       if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 &&
 707           errno != EINPROGRESS)
 708          return _error->Errno("socket",_("Could not create a socket"));
 709
 710       /* This implements a timeout for connect by opening the connection
 711          nonblocking */
 712       if (WaitFd(DataFd,true,TimeOut) == false)
 713          return _error->Error(_("Could not connect data socket, connection timed out"));
 714       unsigned int Err;
 715       unsigned int Len = sizeof(Err);
 716       if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 717          return _error->Errno("getsockopt",_("Failed"));
 718       if (Err != 0)
 719          return _error->Error(_("Could not connect passive socket."));
 720
 721       return true;
 722    }
 723
 724    // Port mode :<
 725    close(DataListenFd);
 726    DataListenFd = -1;
 727
 728    // Get the information for a listening socket.
 729    struct addrinfo *BindAddr = NULL;
 730    struct addrinfo Hints;
 731    memset(&Hints,0,sizeof(Hints));
 732    Hints.ai_socktype = SOCK_STREAM;
 733    Hints.ai_flags |= AI_PASSIVE;
 734    Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family;
 735    if (getaddrinfo(0,"0",&Hints,&BindAddr) != 0 || BindAddr == NULL)
 736       return _error->Error(_("getaddrinfo was unable to get a listening socket"));
 737
 738    // Construct the socket
 739    if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype,
 740                               BindAddr->ai_protocol)) < 0)
 741    {
 742       freeaddrinfo(BindAddr);
 743       return _error->Errno("socket",_("Could not create a socket"));
 744    }
 745
 746    // Bind and listen
 747    if (bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0)
 748    {
 749       freeaddrinfo(BindAddr);
 750       return _error->Errno("bind",_("Could not bind a socket"));
 751    }
 752    freeaddrinfo(BindAddr);
 753    if (listen(DataListenFd,1) < 0)
 754       return _error->Errno("listen",_("Could not listen on the socket"));
 755    SetNonBlock(DataListenFd,true);
 756
 757    // Determine the name to send to the remote
 758    struct sockaddr_storage Addr;
 759    socklen_t AddrLen = sizeof(Addr);
 760    if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0)
 761       return _error->Errno("getsockname",_("Could not determine the socket's name"));
 762
 763
 764    // Reverse the address. We need the server address and the data port.
 765    char Name[NI_MAXHOST];
 766    char Service[NI_MAXSERV];
 767    char Service2[NI_MAXSERV];
 768    getnameinfo((struct sockaddr *)&Addr,AddrLen,
 769                Name,sizeof(Name),Service,sizeof(Service),
 770                NI_NUMERICHOST|NI_NUMERICSERV);
 771    getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen,
 772                Name,sizeof(Name),Service2,sizeof(Service2),
 773                NI_NUMERICHOST|NI_NUMERICSERV);
 774
 775    // Send off an IPv4 address in the old port format
 776    if (((struct sockaddr *)&Addr)->sa_family == AF_INET &&
 777        ForceExtended == false)
 778    {
 779       // Convert the dots in the quad into commas
 780       for (char *I = Name; *I != 0; I++)
 781          if (*I == '.')
 782             *I = ',';
 783       unsigned long Port = atoi(Service);
 784
 785       // Send the port command
 786       unsigned int Tag;
 787       string Msg;
 788       if (WriteMsg(Tag,Msg,"PORT %s,%d,%d",
 789                    Name,
 790                    (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false)
 791          return false;
 792       if (Tag >= 400)
 793          return _error->Error(_("Unable to send PORT command"));
 794       return true;
 795    }
 796
 797    // Construct an EPRT command
 798    unsigned Proto = 0;
 799    for (unsigned J = 0; AFMap[J].Family != 0; J++)
 800       if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family)
 801          Proto = AFMap[J].IETFFamily;
 802    if (Proto == 0)
 803       return _error->Error(_("Unknown address family %u (AF_*)"),
 804                            ((struct sockaddr *)&Addr)->sa_family);
 805
 806    // Send the EPRT command
 807    unsigned int Tag;
 808    string Msg;
 809    if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false)
 810       return false;
 811    if (Tag >= 400)
 812       return _error->Error(_("EPRT failed, server said: %s"),Msg.c_str());
 813    return true;
 814 }
 815                                                                         /*}}}*/
 816 // FTPConn::Finalize - Complete the Data connection                     /*{{{*/
 817 // ---------------------------------------------------------------------
 818 /* If the connection is in port mode this waits for the other end to hook
 819    up to us. */
 820 bool FTPConn::Finalize()
 821 {
 822    // Passive mode? Do nothing
 823    if (PasvAddr != 0)
 824       return true;
 825
 826    // Close any old socket..
 827    close(DataFd);
 828    DataFd = -1;
 829
 830    // Wait for someone to connect..
 831    if (WaitFd(DataListenFd,false,TimeOut) == false)
 832       return _error->Error(_("Data socket connect timed out"));
 833
 834    // Accept the connection
 835    struct sockaddr_in Addr;
 836    socklen_t Len = sizeof(Addr);
 837    DataFd = accept(DataListenFd,(struct sockaddr *)&Addr,&Len);
 838    if (DataFd < 0)
 839       return _error->Errno("accept",_("Unable to accept connection"));
 840
 841    close(DataListenFd);
 842    DataListenFd = -1;
 843
 844    return true;
 845 }
 846                                                                         /*}}}*/
 847 // FTPConn::Get - Get a file                                            /*{{{*/
 848 // ---------------------------------------------------------------------
 849 /* This opens a data connection, sends REST and RETR and then
 850    transfers the file over. */
 851 bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
 852                   Hashes &Hash,bool &Missing)
 853 {
 854    Missing = false;
 855    if (CreateDataFd() == false)
 856       return false;
 857
 858    unsigned int Tag;
 859    string Msg;
 860    if (Resume != 0)
 861    {
 862       if (WriteMsg(Tag,Msg,"REST %u",Resume) == false)
 863          return false;
 864       if (Tag >= 400)
 865          Resume = 0;
 866    }
 867
 868    if (To.Truncate(Resume) == false)
 869       return false;
 870
 871    if (To.Seek(0) == false)
 872       return false;
 873
 874    if (Resume != 0)
 875    {
 876       if (Hash.AddFD(To,Resume) == false)
 877       {
 878          _error->Errno("read",_("Problem hashing file"));
 879          return false;
 880       }
 881    }
 882
 883    // Send the get command
 884    if (WriteMsg(Tag,Msg,"RETR %s",Path) == false)
 885       return false;
 886
 887    if (Tag >= 400)
 888    {
 889       if (Tag == 550)
 890          Missing = true;
 891       return _error->Error(_("Unable to fetch file, server said '%s'"),Msg.c_str());
 892    }
 893
 894    // Finish off the data connection
 895    if (Finalize() == false)
 896       return false;
 897
 898    // Copy loop
 899    unsigned char Buffer[4096];
 900    while (1)
 901    {
 902       // Wait for some data..
 903       if (WaitFd(DataFd,false,TimeOut) == false)
 904       {
 905          Close();
 906          return _error->Error(_("Data socket timed out"));
 907       }
 908
 909       // Read the data..
 910       int Res = read(DataFd,Buffer,sizeof(Buffer));
 911       if (Res == 0)
 912          break;
 913       if (Res < 0)
 914       {
 915          if (errno == EAGAIN)
 916             continue;
 917          break;
 918       }
 919
 920       Hash.Add(Buffer,Res);
 921       if (To.Write(Buffer,Res) == false)
 922       {
 923          Close();
 924          return false;
 925       }
 926    }
 927
 928    // All done
 929    close(DataFd);
 930    DataFd = -1;
 931
 932    // Read the closing message from the server
 933    if (ReadResp(Tag,Msg) == false)
 934       return false;
 935    if (Tag >= 400)
 936       return _error->Error(_("Data transfer failed, server said '%s'"),Msg.c_str());
 937    return true;
 938 }
 939                                                                         /*}}}*/
 940
 941 // FtpMethod::FtpMethod - Constructor                                   /*{{{*/
 942 // ---------------------------------------------------------------------
 943 /* */
 944 FtpMethod::FtpMethod() : pkgAcqMethod("1.0",SendConfig)
 945 {
 946    signal(SIGTERM,SigTerm);
 947    signal(SIGINT,SigTerm);
 948
 949    Server = 0;
 950    FailFd = -1;
 951 }
 952                                                                         /*}}}*/
 953 // FtpMethod::SigTerm - Handle a fatal signal                           /*{{{*/
 954 // ---------------------------------------------------------------------
 955 /* This closes and timestamps the open file. This is necessary to get
 956    resume behavoir on user abort */
 957 void FtpMethod::SigTerm(int)
 958 {
 959    if (FailFd == -1)
 960       _exit(100);
 961
 962    // Timestamp
 963    struct timeval times[2];
 964    times[0].tv_sec = FailTime;
 965    times[1].tv_sec = FailTime;
 966    times[0].tv_usec = times[1].tv_usec = 0;
 967    utimes(FailFile.c_str(), times);
 968
 969    close(FailFd);
 970
 971    _exit(100);
 972 }
 973                                                                         /*}}}*/
 974 // FtpMethod::Configuration - Handle a configuration message            /*{{{*/
 975 // ---------------------------------------------------------------------
 976 /* We stash the desired pipeline depth */
 977 bool FtpMethod::Configuration(string Message)
 978 {
 979    if (pkgAcqMethod::Configuration(Message) == false)
 980       return false;
 981
 982    TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut);
 983    return true;
 984 }
 985                                                                         /*}}}*/
 986 // FtpMethod::Fetch - Fetch a file                                      /*{{{*/
 987 // ---------------------------------------------------------------------
 988 /* Fetch a single file, called by the base class..  */
 989 bool FtpMethod::Fetch(FetchItem *Itm)
 990 {
 991    URI Get = Itm->Uri;
 992    const char *File = Get.Path.c_str();
 993    FetchResult Res;
 994    Res.Filename = Itm->DestFile;
 995    Res.IMSHit = false;
 996
 997    maybe_add_auth (Get, _config->FindFile("Dir::Etc::netrc"));
 998
 999    // Connect to the server
1000    if (Server == 0 || Server->Comp(Get) == false)
1001    {
1002       delete Server;
1003       Server = new FTPConn(Get);
1004    }
1005
1006    // Could not connect is a transient error..
1007    if (Server->Open(this) == false)
1008    {
1009       Server->Close();
1010       Fail(true);
1011       return true;
1012    }
1013
1014    // Get the files information
1015    Status(_("Query"));
1016    unsigned long long Size;
1017    if (Server->Size(File,Size) == false ||
1018        Server->ModTime(File,FailTime) == false)
1019    {
1020       Fail(true);
1021       return true;
1022    }
1023    Res.Size = Size;
1024
1025    // See if it is an IMS hit
1026    if (Itm->LastModified == FailTime)
1027    {
1028       Res.Size = 0;
1029       Res.IMSHit = true;
1030       URIDone(Res);
1031       return true;
1032    }
1033
1034    // See if the file exists
1035    struct stat Buf;
1036    if (stat(Itm->DestFile.c_str(),&Buf) == 0)
1037    {
1038       if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime)
1039       {
1040          Res.Size = Buf.st_size;
1041          Res.LastModified = Buf.st_mtime;
1042          Res.ResumePoint = Buf.st_size;
1043          URIDone(Res);
1044          return true;
1045       }
1046
1047       // Resume?
1048       if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size)
1049          Res.ResumePoint = Buf.st_size;
1050    }
1051
1052    // Open the file
1053    Hashes Hash;
1054    {
1055       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
1056       if (_error->PendingError() == true)
1057          return false;
1058
1059       URIStart(Res);
1060
1061       FailFile = Itm->DestFile;
1062       FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1063       FailFd = Fd.Fd();
1064
1065       bool Missing;
1066       if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing) == false)
1067       {
1068          Fd.Close();
1069
1070          // Timestamp
1071          struct timeval times[2];
1072          times[0].tv_sec = FailTime;
1073          times[1].tv_sec = FailTime;
1074          times[0].tv_usec = times[1].tv_usec = 0;
1075          utimes(FailFile.c_str(), times);
1076
1077          // If the file is missing we hard fail and delete the destfile
1078          // otherwise transient fail
1079          if (Missing == true) {
1080             unlink(FailFile.c_str());
1081             return false;
1082          }
1083          Fail(true);
1084          return true;
1085       }
1086
1087       Res.Size = Fd.Size();
1088
1089       // Timestamp
1090       struct timeval times[2];
1091       times[0].tv_sec = FailTime;
1092       times[1].tv_sec = FailTime;
1093       times[0].tv_usec = times[1].tv_usec = 0;
1094       utimes(Fd.Name().c_str(), times);
1095       FailFd = -1;
1096    }
1097
1098    Res.LastModified = FailTime;
1099    Res.TakeHashes(Hash);
1100
1101    URIDone(Res);
1102
1103    return true;
1104 }
1105                                                                         /*}}}*/
1106
1107 int main(int, const char *argv[])
1108 {
1109    setlocale(LC_ALL, "");
1110
1111    /* See if we should be come the http client - we do this for http
1112       proxy urls */
1113    if (getenv("ftp_proxy") != 0)
1114    {
1115       URI Proxy = string(getenv("ftp_proxy"));
1116
1117       // Run the HTTP method
1118       if (Proxy.Access == "http")
1119       {
1120          // Copy over the environment setting
1121          char S[300];
1122          snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy"));
1123          putenv(S);
1124          putenv((char *)"no_proxy=");
1125
1126          // Run the http method
1127          string Path = flNotFile(argv[0]) + "http";
1128          execl(Path.c_str(),Path.c_str(),(char *)NULL);
1129          cerr << _("Unable to invoke ") << Path << endl;
1130          exit(100);
1131       }
1132    }
1133
1134    FtpMethod Mth;
1135
1136    // no more active ftp, sorry
1137    Mth.DropPrivsOrDie();
1138
1139    return Mth.Run();
1140 }