methods/ftp.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: ftp.cc,v 1.31.2.1 2004/01/16 18:58:50 mdz Exp $
   4 /* ######################################################################
   5
   6    FTP Acquire Method - This is the FTP acquire method for APT.
   7
   8    This is a very simple implementation that does not try to optimize
   9    at all. Commands are sent syncronously with the FTP server (as the
  10    rfc recommends, but it is not really necessary..) and no tricks are
  11    done to speed things along.
  12
  13    RFC 2428 describes the IPv6 FTP behavior
  14
  15    ##################################################################### */
  16                                                                         /*}}}*/
  17 // Include Files                                                        /*{{{*/
  18 #include <config.h>
  19
  20 #include <apt-pkg/fileutl.h>
  21 #include <apt-pkg/acquire-method.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/hashes.h>
  24 #include <apt-pkg/netrc.h>
  25 #include <apt-pkg/configuration.h>
  26 #include <apt-pkg/strutl.h>
  27
  28 #include <ctype.h>
  29 #include <stdlib.h>
  30 #include <string.h>
  31 #include <sys/stat.h>
  32 #include <sys/time.h>
  33 #include <unistd.h>
  34 #include <signal.h>
  35 #include <stdio.h>
  36 #include <errno.h>
  37 #include <stdarg.h>
  38 #include <iostream>
  39
  40 // Internet stuff
  41 #include <netinet/in.h>
  42 #include <arpa/inet.h>
  43 #include <netdb.h>
  44
  45 #include "rfc2553emu.h"
  46 #include "connect.h"
  47 #include "ftp.h"
  48
  49 #include <apti18n.h>
  50                                                                         /*}}}*/
  51
  52 using namespace std;
  53
  54 /* This table is for the EPRT and EPSV commands, it maps the OS address
  55    family to the IETF address families */
  56 struct AFMap
  57 {
  58    unsigned long Family;
  59    unsigned long IETFFamily;
  60 };
  61
  62 #ifndef AF_INET6
  63 struct AFMap AFMap[] = {{AF_INET,1},{0, 0}};
  64 #else
  65 struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{0, 0}};
  66 #endif
  67
  68 unsigned long TimeOut = 120;
  69 URI Proxy;
  70 string FtpMethod::FailFile;
  71 int FtpMethod::FailFd = -1;
  72 time_t FtpMethod::FailTime = 0;
  73
  74 // FTPConn::FTPConn - Constructor                                       /*{{{*/
  75 // ---------------------------------------------------------------------
  76 /* */
  77 FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1),
  78                             DataListenFd(-1), ServerName(Srv),
  79                             ForceExtended(false), TryPassive(true),
  80                             PeerAddrLen(0), ServerAddrLen(0)
  81 {
  82    Debug = _config->FindB("Debug::Acquire::Ftp",false);
  83    PasvAddr = 0;
  84    Buffer[0] = '\0';
  85 }
  86                                                                         /*}}}*/
  87 // FTPConn::~FTPConn - Destructor                                       /*{{{*/
  88 // ---------------------------------------------------------------------
  89 /* */
  90 FTPConn::~FTPConn()
  91 {
  92    Close();
  93 }
  94                                                                         /*}}}*/
  95 // FTPConn::Close - Close down the connection                           /*{{{*/
  96 // ---------------------------------------------------------------------
  97 /* Just tear down the socket and data socket */
  98 void FTPConn::Close()
  99 {
 100    close(ServerFd);
 101    ServerFd = -1;
 102    close(DataFd);
 103    DataFd = -1;
 104    close(DataListenFd);
 105    DataListenFd = -1;
 106
 107    if (PasvAddr != 0)
 108       freeaddrinfo(PasvAddr);
 109    PasvAddr = 0;
 110 }
 111                                                                         /*}}}*/
 112 // FTPConn::Open - Open a new connection                                /*{{{*/
 113 // ---------------------------------------------------------------------
 114 /* Connect to the server using a non-blocking connection and perform a
 115    login. */
 116 bool FTPConn::Open(pkgAcqMethod *Owner)
 117 {
 118    // Use the already open connection if possible.
 119    if (ServerFd != -1)
 120       return true;
 121
 122    Close();
 123
 124    // Determine the proxy setting
 125    string SpecificProxy = _config->Find("Acquire::ftp::Proxy::" + ServerName.Host);
 126    if (!SpecificProxy.empty())
 127    {
 128            if (SpecificProxy == "DIRECT")
 129                    Proxy = "";
 130            else
 131                    Proxy = SpecificProxy;
 132    }
 133    else
 134    {
 135            string DefProxy = _config->Find("Acquire::ftp::Proxy");
 136            if (!DefProxy.empty())
 137            {
 138                    Proxy = DefProxy;
 139            }
 140            else
 141            {
 142                    char* result = getenv("ftp_proxy");
 143                    Proxy = result ? result : "";
 144            }
 145    }
 146
 147    // Parse no_proxy, a , separated list of domains
 148    if (getenv("no_proxy") != 0)
 149    {
 150       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 151          Proxy = "";
 152    }
 153
 154    // Determine what host and port to use based on the proxy settings
 155    int Port = 0;
 156    string Host;
 157    if (Proxy.empty() == true)
 158    {
 159       if (ServerName.Port != 0)
 160          Port = ServerName.Port;
 161       Host = ServerName.Host;
 162    }
 163    else
 164    {
 165       if (Proxy.Port != 0)
 166          Port = Proxy.Port;
 167       Host = Proxy.Host;
 168    }
 169
 170    /* Connect to the remote server. Since FTP is connection oriented we
 171       want to make sure we get a new server every time we reconnect */
 172    RotateDNS();
 173    if (Connect(Host,Port,"ftp",21,ServerFd,TimeOut,Owner) == false)
 174       return false;
 175
 176    // Login must be before getpeername otherwise dante won't work.
 177    Owner->Status(_("Logging in"));
 178    bool Res = Login();
 179
 180    // Get the remote server's address
 181    PeerAddrLen = sizeof(PeerAddr);
 182    if (getpeername(ServerFd,(sockaddr *)&PeerAddr,&PeerAddrLen) != 0)
 183       return _error->Errno("getpeername",_("Unable to determine the peer name"));
 184
 185    // Get the local machine's address
 186    ServerAddrLen = sizeof(ServerAddr);
 187    if (getsockname(ServerFd,(sockaddr *)&ServerAddr,&ServerAddrLen) != 0)
 188       return _error->Errno("getsockname",_("Unable to determine the local name"));
 189
 190    return Res;
 191 }
 192                                                                         /*}}}*/
 193 // FTPConn::Login - Login to the remote server                          /*{{{*/
 194 // ---------------------------------------------------------------------
 195 /* This performs both normal login and proxy login using a simples script
 196    stored in the config file. */
 197 bool FTPConn::Login()
 198 {
 199    unsigned int Tag;
 200    string Msg;
 201
 202    // Setup the variables needed for authentication
 203    string User = "anonymous";
 204    string Pass = "apt_get_ftp_2.1@debian.linux.user";
 205
 206    // Fill in the user/pass
 207    if (ServerName.User.empty() == false)
 208       User = ServerName.User;
 209    if (ServerName.Password.empty() == false)
 210       Pass = ServerName.Password;
 211
 212    // Perform simple login
 213    if (Proxy.empty() == true)
 214    {
 215       // Read the initial response
 216       if (ReadResp(Tag,Msg) == false)
 217          return false;
 218       if (Tag >= 400)
 219          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 220
 221       // Send the user
 222       if (WriteMsg(Tag,Msg,"USER %s",User.c_str()) == false)
 223          return false;
 224       if (Tag >= 400)
 225          return _error->Error(_("USER failed, server said: %s"),Msg.c_str());
 226
 227       if (Tag == 331) { // 331 User name okay, need password.
 228          // Send the Password
 229          if (WriteMsg(Tag,Msg,"PASS %s",Pass.c_str()) == false)
 230             return false;
 231          if (Tag >= 400)
 232             return _error->Error(_("PASS failed, server said: %s"),Msg.c_str());
 233       }
 234
 235       // Enter passive mode
 236       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 237          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 238       else
 239          TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 240    }
 241    else
 242    {
 243       // Read the initial response
 244       if (ReadResp(Tag,Msg) == false)
 245          return false;
 246       if (Tag >= 400)
 247          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 248
 249       // Perform proxy script execution
 250       Configuration::Item const *Opts = _config->Tree("Acquire::ftp::ProxyLogin");
 251       if (Opts == 0 || Opts->Child == 0)
 252          return _error->Error(_("A proxy server was specified but no login "
 253                               "script, Acquire::ftp::ProxyLogin is empty."));
 254       Opts = Opts->Child;
 255
 256       // Iterate over the entire login script
 257       for (; Opts != 0; Opts = Opts->Next)
 258       {
 259          if (Opts->Value.empty() == true)
 260             continue;
 261
 262          // Substitute the variables into the command
 263          string Tmp = Opts->Value;
 264          Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
 265          Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
 266          Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
 267          Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
 268          if (ServerName.Port != 0)
 269          {
 270             std::string SitePort;
 271             strprintf(SitePort, "%u", ServerName.Port);
 272             Tmp = SubstVar(Tmp,"$(SITE_PORT)", SitePort);
 273          }
 274          else
 275             Tmp = SubstVar(Tmp,"$(SITE_PORT)", "21");
 276          Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
 277
 278          // Send the command
 279          if (WriteMsg(Tag,Msg,"%s",Tmp.c_str()) == false)
 280             return false;
 281          if (Tag >= 400)
 282             return _error->Error(_("Login script command '%s' failed, server said: %s"),Tmp.c_str(),Msg.c_str());
 283       }
 284
 285       // Enter passive mode
 286       TryPassive = false;
 287       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 288          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 289       else
 290       {
 291          if (_config->Exists("Acquire::FTP::Proxy::Passive") == true)
 292             TryPassive = _config->FindB("Acquire::FTP::Proxy::Passive",true);
 293          else
 294             TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 295       }
 296    }
 297
 298    // Force the use of extended commands
 299    if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true)
 300       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true);
 301    else
 302       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false);
 303
 304    // Binary mode
 305    if (WriteMsg(Tag,Msg,"TYPE I") == false)
 306       return false;
 307    if (Tag >= 400)
 308       return _error->Error(_("TYPE failed, server said: %s"),Msg.c_str());
 309
 310    return true;
 311 }
 312                                                                         /*}}}*/
 313 // FTPConn::ReadLine - Read a line from the server                      /*{{{*/
 314 // ---------------------------------------------------------------------
 315 /* This performs a very simple buffered read. */
 316 bool FTPConn::ReadLine(string &Text)
 317 {
 318    if (ServerFd == -1)
 319       return false;
 320
 321    // Suck in a line
 322    while (Len < sizeof(Buffer))
 323    {
 324       // Scan the buffer for a new line
 325       for (unsigned int I = 0; I != Len; I++)
 326       {
 327          // Escape some special chars
 328          if (Buffer[I] == 0)
 329             Buffer[I] = '?';
 330
 331          // End of line?
 332          if (Buffer[I] != '\n')
 333             continue;
 334
 335          I++;
 336          Text = string(Buffer,I);
 337          memmove(Buffer,Buffer+I,Len - I);
 338          Len -= I;
 339          return true;
 340       }
 341
 342       // Wait for some data..
 343       if (WaitFd(ServerFd,false,TimeOut) == false)
 344       {
 345          Close();
 346          return _error->Error(_("Connection timeout"));
 347       }
 348
 349       // Suck it back
 350       int Res = read(ServerFd,Buffer + Len,sizeof(Buffer) - Len);
 351       if (Res == 0)
 352          _error->Error(_("Server closed the connection"));
 353       if (Res <= 0)
 354       {
 355          _error->Errno("read",_("Read error"));
 356          Close();
 357          return false;
 358       }
 359       Len += Res;
 360    }
 361
 362    return _error->Error(_("A response overflowed the buffer."));
 363 }
 364                                                                         /*}}}*/
 365 // FTPConn::ReadResp - Read a full response from the server             /*{{{*/
 366 // ---------------------------------------------------------------------
 367 /* This reads a reply code from the server, it handles both p */
 368 bool FTPConn::ReadResp(unsigned int &Ret,string &Text)
 369 {
 370    // Grab the first line of the response
 371    string Msg;
 372    if (ReadLine(Msg) == false)
 373        return false;
 374
 375    // Get the ID code
 376    char *End;
 377    Ret = strtol(Msg.c_str(),&End,10);
 378    if (End - Msg.c_str() != 3)
 379       return _error->Error(_("Protocol corruption"));
 380
 381    // All done ?
 382    Text = Msg.c_str()+4;
 383    if (*End == ' ')
 384    {
 385       if (Debug == true)
 386          cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 387       return true;
 388    }
 389
 390    if (*End != '-')
 391       return _error->Error(_("Protocol corruption"));
 392
 393    /* Okay, here we do the continued message trick. This is foolish, but
 394       proftpd follows the protocol as specified and wu-ftpd doesn't, so
 395       we filter. I wonder how many clients break if you use proftpd and
 396       put a '- in the 3rd spot in the message? */
 397    char Leader[4];
 398    strncpy(Leader,Msg.c_str(),3);
 399    Leader[3] = 0;
 400    while (ReadLine(Msg) == true)
 401    {
 402       // Short, it must be using RFC continuation..
 403       if (Msg.length() < 4)
 404       {
 405          Text += Msg;
 406          continue;
 407       }
 408
 409       // Oops, finished
 410       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == ' ')
 411       {
 412          Text += Msg.c_str()+4;
 413          break;
 414       }
 415
 416       // This message has the wu-ftpd style reply code prefixed
 417       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == '-')
 418       {
 419          Text += Msg.c_str()+4;
 420          continue;
 421       }
 422
 423       // Must be RFC style prefixing
 424       Text += Msg;
 425    }
 426
 427    if (Debug == true && _error->PendingError() == false)
 428       cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 429
 430    return !_error->PendingError();
 431 }
 432                                                                         /*}}}*/
 433 // FTPConn::WriteMsg - Send a message to the server                     /*{{{*/
 434 // ---------------------------------------------------------------------
 435 /* Simple printf like function.. */
 436 bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...)
 437 {
 438    va_list args;
 439    va_start(args,Fmt);
 440
 441    // sprintf the description
 442    char S[400];
 443    vsnprintf(S,sizeof(S) - 4,Fmt,args);
 444    strcat(S,"\r\n");
 445    va_end(args);
 446
 447    if (Debug == true)
 448       cerr << "-> '" << QuoteString(S,"") << "'" << endl;
 449
 450    // Send it off
 451    unsigned long Len = strlen(S);
 452    unsigned long Start = 0;
 453    while (Len != 0)
 454    {
 455       if (WaitFd(ServerFd,true,TimeOut) == false)
 456       {
 457          Close();
 458          return _error->Error(_("Connection timeout"));
 459       }
 460
 461       int Res = write(ServerFd,S + Start,Len);
 462       if (Res <= 0)
 463       {
 464          _error->Errno("write",_("Write error"));
 465          Close();
 466          return false;
 467       }
 468
 469       Len -= Res;
 470       Start += Res;
 471    }
 472
 473    return ReadResp(Ret,Text);
 474 }
 475                                                                         /*}}}*/
 476 // FTPConn::GoPasv - Enter Passive mode                                 /*{{{*/
 477 // ---------------------------------------------------------------------
 478 /* Try to enter passive mode, the return code does not indicate if passive
 479    mode could or could not be established, only if there was a fatal error.
 480    We have to enter passive mode every time we make a data connection :| */
 481 bool FTPConn::GoPasv()
 482 {
 483    /* The PASV command only works on IPv4 sockets, even though it could
 484       in theory suppory IPv6 via an all zeros reply */
 485    if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET ||
 486        ForceExtended == true)
 487       return ExtGoPasv();
 488
 489    if (PasvAddr != 0)
 490       freeaddrinfo(PasvAddr);
 491    PasvAddr = 0;
 492
 493    // Try to enable pasv mode
 494    unsigned int Tag;
 495    string Msg;
 496    if (WriteMsg(Tag,Msg,"PASV") == false)
 497       return false;
 498
 499    // Unsupported function
 500    string::size_type Pos = Msg.find('(');
 501    if (Tag >= 400 || Pos == string::npos)
 502       return true;
 503
 504    // Scan it
 505    unsigned a0,a1,a2,a3,p0,p1;
 506    if (sscanf(Msg.c_str() + Pos,"(%u,%u,%u,%u,%u,%u)",&a0,&a1,&a2,&a3,&p0,&p1) != 6)
 507       return true;
 508
 509    /* Some evil servers return 0 to mean their addr. We can actually speak
 510       to these servers natively using IPv6 */
 511    if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0)
 512    {
 513       // Get the IP in text form
 514       char Name[NI_MAXHOST];
 515       char Service[NI_MAXSERV];
 516       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 517                   Name,sizeof(Name),Service,sizeof(Service),
 518                   NI_NUMERICHOST|NI_NUMERICSERV);
 519
 520       struct addrinfo Hints;
 521       memset(&Hints,0,sizeof(Hints));
 522       Hints.ai_socktype = SOCK_STREAM;
 523       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 524       Hints.ai_flags |= AI_NUMERICHOST;
 525
 526       // Get a new passive address.
 527       char Port[100];
 528       snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 529       if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 530          return true;
 531       return true;
 532    }
 533
 534    struct addrinfo Hints;
 535    memset(&Hints,0,sizeof(Hints));
 536    Hints.ai_socktype = SOCK_STREAM;
 537    Hints.ai_family = AF_INET;
 538    Hints.ai_flags |= AI_NUMERICHOST;
 539
 540    // Get a new passive address.
 541    char Port[100];
 542    snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 543    char Name[100];
 544    snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3);
 545    if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 546       return true;
 547    return true;
 548 }
 549                                                                         /*}}}*/
 550 // FTPConn::ExtGoPasv - Enter Extended Passive mode                     /*{{{*/
 551 // ---------------------------------------------------------------------
 552 /* Try to enter extended passive mode. See GoPasv above and RFC 2428 */
 553 bool FTPConn::ExtGoPasv()
 554 {
 555    if (PasvAddr != 0)
 556       freeaddrinfo(PasvAddr);
 557    PasvAddr = 0;
 558
 559    // Try to enable pasv mode
 560    unsigned int Tag;
 561    string Msg;
 562    if (WriteMsg(Tag,Msg,"EPSV") == false)
 563       return false;
 564
 565    // Unsupported function
 566    string::size_type Pos = Msg.find('(');
 567    if (Tag >= 400 || Pos == string::npos)
 568       return true;
 569
 570    // Scan it
 571    string::const_iterator List[4];
 572    unsigned Count = 0;
 573    Pos++;
 574    for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); ++I)
 575    {
 576       if (*I != Msg[Pos])
 577          continue;
 578       if (Count >= 4)
 579          return true;
 580       List[Count++] = I;
 581    }
 582    if (Count != 4)
 583       return true;
 584
 585    // Break it up ..
 586    unsigned long Proto = 0;
 587    unsigned long Port = 0;
 588    string IP;
 589    IP = string(List[1]+1,List[2]);
 590    Port = atoi(string(List[2]+1,List[3]).c_str());
 591    if (IP.empty() == false)
 592       Proto = atoi(string(List[0]+1,List[1]).c_str());
 593
 594    if (Port == 0)
 595       return false;
 596
 597    // String version of the port
 598    char PStr[100];
 599    snprintf(PStr,sizeof(PStr),"%lu",Port);
 600
 601    // Get the IP in text form
 602    struct addrinfo Hints;
 603    memset(&Hints,0,sizeof(Hints));
 604    Hints.ai_socktype = SOCK_STREAM;
 605    Hints.ai_flags |= AI_NUMERICHOST;
 606
 607    /* The RFC defined case, connect to the old IP/protocol using the
 608       new port. */
 609    if (IP.empty() == true)
 610    {
 611       // Get the IP in text form
 612       char Name[NI_MAXHOST];
 613       char Service[NI_MAXSERV];
 614       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 615                   Name,sizeof(Name),Service,sizeof(Service),
 616                   NI_NUMERICHOST|NI_NUMERICSERV);
 617       IP = Name;
 618       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 619    }
 620    else
 621    {
 622       // Get the family..
 623       Hints.ai_family = 0;
 624       for (unsigned J = 0; AFMap[J].Family != 0; J++)
 625          if (AFMap[J].IETFFamily == Proto)
 626             Hints.ai_family = AFMap[J].Family;
 627       if (Hints.ai_family == 0)
 628          return true;
 629    }
 630
 631    // Get a new passive address.
 632    if (getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr) != 0)
 633       return true;
 634
 635    return true;
 636 }
 637                                                                         /*}}}*/
 638 // FTPConn::Size - Return the size of a file                            /*{{{*/
 639 // ---------------------------------------------------------------------
 640 /* Grab the file size from the server, 0 means no size or empty file */
 641 bool FTPConn::Size(const char *Path,unsigned long long &Size)
 642 {
 643    // Query the size
 644    unsigned int Tag;
 645    string Msg;
 646    Size = 0;
 647    if (WriteMsg(Tag,Msg,"SIZE %s",Path) == false)
 648       return false;
 649
 650    char *End;
 651    Size = strtoull(Msg.c_str(),&End,10);
 652    if (Tag >= 400 || End == Msg.c_str())
 653       Size = 0;
 654    return true;
 655 }
 656                                                                         /*}}}*/
 657 // FTPConn::ModTime - Return the modification time of the file          /*{{{*/
 658 // ---------------------------------------------------------------------
 659 /* Like Size no error is returned if the command is not supported. If the
 660    command fails then time is set to the current time of day to fool
 661    date checks. */
 662 bool FTPConn::ModTime(const char *Path, time_t &Time)
 663 {
 664    Time = time(&Time);
 665
 666    // Query the mod time
 667    unsigned int Tag;
 668    string Msg;
 669    if (WriteMsg(Tag,Msg,"MDTM %s",Path) == false)
 670       return false;
 671    if (Tag >= 400 || Msg.empty() == true || isdigit(Msg[0]) == 0)
 672       return true;
 673
 674    // Parse it
 675    return FTPMDTMStrToTime(Msg.c_str(), Time);
 676 }
 677                                                                         /*}}}*/
 678 // FTPConn::CreateDataFd - Get a data connection                        /*{{{*/
 679 // ---------------------------------------------------------------------
 680 /* Create the data connection. Call FinalizeDataFd after this though.. */
 681 bool FTPConn::CreateDataFd()
 682 {
 683    close(DataFd);
 684    DataFd = -1;
 685
 686    // Attempt to enter passive mode.
 687    if (TryPassive == true)
 688    {
 689       if (GoPasv() == false)
 690          return false;
 691
 692       // Oops, didn't work out, don't bother trying again.
 693       if (PasvAddr == 0)
 694          TryPassive = false;
 695    }
 696
 697    // Passive mode?
 698    if (PasvAddr != 0)
 699    {
 700       // Get a socket
 701       if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype,
 702                            PasvAddr->ai_protocol)) < 0)
 703          return _error->Errno("socket",_("Could not create a socket"));
 704
 705       // Connect to the server
 706       SetNonBlock(DataFd,true);
 707       if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 &&
 708           errno != EINPROGRESS)
 709          return _error->Errno("socket",_("Could not create a socket"));
 710
 711       /* This implements a timeout for connect by opening the connection
 712          nonblocking */
 713       if (WaitFd(DataFd,true,TimeOut) == false)
 714          return _error->Error(_("Could not connect data socket, connection timed out"));
 715       unsigned int Err;
 716       unsigned int Len = sizeof(Err);
 717       if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 718          return _error->Errno("getsockopt",_("Failed"));
 719       if (Err != 0)
 720          return _error->Error(_("Could not connect passive socket."));
 721
 722       return true;
 723    }
 724
 725    // Port mode :<
 726    close(DataListenFd);
 727    DataListenFd = -1;
 728
 729    // Get the information for a listening socket.
 730    struct addrinfo *BindAddr = NULL;
 731    struct addrinfo Hints;
 732    memset(&Hints,0,sizeof(Hints));
 733    Hints.ai_socktype = SOCK_STREAM;
 734    Hints.ai_flags |= AI_PASSIVE;
 735    Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family;
 736    if (getaddrinfo(0,"0",&Hints,&BindAddr) != 0 || BindAddr == NULL)
 737       return _error->Error(_("getaddrinfo was unable to get a listening socket"));
 738
 739    // Construct the socket
 740    if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype,
 741                               BindAddr->ai_protocol)) < 0)
 742    {
 743       freeaddrinfo(BindAddr);
 744       return _error->Errno("socket",_("Could not create a socket"));
 745    }
 746
 747    // Bind and listen
 748    if (bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0)
 749    {
 750       freeaddrinfo(BindAddr);
 751       return _error->Errno("bind",_("Could not bind a socket"));
 752    }
 753    freeaddrinfo(BindAddr);
 754    if (listen(DataListenFd,1) < 0)
 755       return _error->Errno("listen",_("Could not listen on the socket"));
 756    SetNonBlock(DataListenFd,true);
 757
 758    // Determine the name to send to the remote
 759    struct sockaddr_storage Addr;
 760    socklen_t AddrLen = sizeof(Addr);
 761    if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0)
 762       return _error->Errno("getsockname",_("Could not determine the socket's name"));
 763
 764
 765    // Reverse the address. We need the server address and the data port.
 766    char Name[NI_MAXHOST];
 767    char Service[NI_MAXSERV];
 768    char Service2[NI_MAXSERV];
 769    getnameinfo((struct sockaddr *)&Addr,AddrLen,
 770                Name,sizeof(Name),Service,sizeof(Service),
 771                NI_NUMERICHOST|NI_NUMERICSERV);
 772    getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen,
 773                Name,sizeof(Name),Service2,sizeof(Service2),
 774                NI_NUMERICHOST|NI_NUMERICSERV);
 775
 776    // Send off an IPv4 address in the old port format
 777    if (((struct sockaddr *)&Addr)->sa_family == AF_INET &&
 778        ForceExtended == false)
 779    {
 780       // Convert the dots in the quad into commas
 781       for (char *I = Name; *I != 0; I++)
 782          if (*I == '.')
 783             *I = ',';
 784       unsigned long Port = atoi(Service);
 785
 786       // Send the port command
 787       unsigned int Tag;
 788       string Msg;
 789       if (WriteMsg(Tag,Msg,"PORT %s,%d,%d",
 790                    Name,
 791                    (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false)
 792          return false;
 793       if (Tag >= 400)
 794          return _error->Error(_("Unable to send PORT command"));
 795       return true;
 796    }
 797
 798    // Construct an EPRT command
 799    unsigned Proto = 0;
 800    for (unsigned J = 0; AFMap[J].Family != 0; J++)
 801       if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family)
 802          Proto = AFMap[J].IETFFamily;
 803    if (Proto == 0)
 804       return _error->Error(_("Unknown address family %u (AF_*)"),
 805                            ((struct sockaddr *)&Addr)->sa_family);
 806
 807    // Send the EPRT command
 808    unsigned int Tag;
 809    string Msg;
 810    if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false)
 811       return false;
 812    if (Tag >= 400)
 813       return _error->Error(_("EPRT failed, server said: %s"),Msg.c_str());
 814    return true;
 815 }
 816                                                                         /*}}}*/
 817 // FTPConn::Finalize - Complete the Data connection                     /*{{{*/
 818 // ---------------------------------------------------------------------
 819 /* If the connection is in port mode this waits for the other end to hook
 820    up to us. */
 821 bool FTPConn::Finalize()
 822 {
 823    // Passive mode? Do nothing
 824    if (PasvAddr != 0)
 825       return true;
 826
 827    // Close any old socket..
 828    close(DataFd);
 829    DataFd = -1;
 830
 831    // Wait for someone to connect..
 832    if (WaitFd(DataListenFd,false,TimeOut) == false)
 833       return _error->Error(_("Data socket connect timed out"));
 834
 835    // Accept the connection
 836    struct sockaddr_in Addr;
 837    socklen_t Len = sizeof(Addr);
 838    DataFd = accept(DataListenFd,(struct sockaddr *)&Addr,&Len);
 839    if (DataFd < 0)
 840       return _error->Errno("accept",_("Unable to accept connection"));
 841
 842    close(DataListenFd);
 843    DataListenFd = -1;
 844
 845    return true;
 846 }
 847                                                                         /*}}}*/
 848 // FTPConn::Get - Get a file                                            /*{{{*/
 849 // ---------------------------------------------------------------------
 850 /* This opens a data connection, sends REST and RETR and then
 851    transfers the file over. */
 852 bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
 853                   Hashes &Hash,bool &Missing, unsigned long long MaximumSize,
 854                   pkgAcqMethod *Owner)
 855 {
 856    Missing = false;
 857    if (CreateDataFd() == false)
 858       return false;
 859
 860    unsigned int Tag;
 861    string Msg;
 862    if (Resume != 0)
 863    {
 864       if (WriteMsg(Tag,Msg,"REST %u",Resume) == false)
 865          return false;
 866       if (Tag >= 400)
 867          Resume = 0;
 868    }
 869
 870    if (To.Truncate(Resume) == false)
 871       return false;
 872
 873    if (To.Seek(0) == false)
 874       return false;
 875
 876    if (Resume != 0)
 877    {
 878       if (Hash.AddFD(To,Resume) == false)
 879       {
 880          _error->Errno("read",_("Problem hashing file"));
 881          return false;
 882       }
 883    }
 884
 885    // Send the get command
 886    if (WriteMsg(Tag,Msg,"RETR %s",Path) == false)
 887       return false;
 888
 889    if (Tag >= 400)
 890    {
 891       if (Tag == 550)
 892          Missing = true;
 893       return _error->Error(_("Unable to fetch file, server said '%s'"),Msg.c_str());
 894    }
 895
 896    // Finish off the data connection
 897    if (Finalize() == false)
 898       return false;
 899
 900    // Copy loop
 901    unsigned char Buffer[4096];
 902    while (1)
 903    {
 904       // Wait for some data..
 905       if (WaitFd(DataFd,false,TimeOut) == false)
 906       {
 907          Close();
 908          return _error->Error(_("Data socket timed out"));
 909       }
 910
 911       // Read the data..
 912       int Res = read(DataFd,Buffer,sizeof(Buffer));
 913       if (Res == 0)
 914          break;
 915       if (Res < 0)
 916       {
 917          if (errno == EAGAIN)
 918             continue;
 919          break;
 920       }
 921
 922       Hash.Add(Buffer,Res);
 923       if (To.Write(Buffer,Res) == false)
 924       {
 925          Close();
 926          return false;
 927       }
 928
 929       if (MaximumSize > 0 && To.Tell() > MaximumSize)
 930       {
 931          Owner->SetFailReason("MaximumSizeExceeded");
 932          return _error->Error("Writing more data than expected (%llu > %llu)",
 933                               To.Tell(), MaximumSize);
 934       }
 935    }
 936
 937    // All done
 938    close(DataFd);
 939    DataFd = -1;
 940
 941    // Read the closing message from the server
 942    if (ReadResp(Tag,Msg) == false)
 943       return false;
 944    if (Tag >= 400)
 945       return _error->Error(_("Data transfer failed, server said '%s'"),Msg.c_str());
 946    return true;
 947 }
 948                                                                         /*}}}*/
 949
 950 // FtpMethod::FtpMethod - Constructor                                   /*{{{*/
 951 // ---------------------------------------------------------------------
 952 /* */
 953 FtpMethod::FtpMethod() : pkgAcqMethod("1.0",SendConfig)
 954 {
 955    signal(SIGTERM,SigTerm);
 956    signal(SIGINT,SigTerm);
 957
 958    Server = 0;
 959    FailFd = -1;
 960 }
 961                                                                         /*}}}*/
 962 // FtpMethod::SigTerm - Handle a fatal signal                           /*{{{*/
 963 // ---------------------------------------------------------------------
 964 /* This closes and timestamps the open file. This is necessary to get
 965    resume behavoir on user abort */
 966 void FtpMethod::SigTerm(int)
 967 {
 968    if (FailFd == -1)
 969       _exit(100);
 970
 971    // Timestamp
 972    struct timeval times[2];
 973    times[0].tv_sec = FailTime;
 974    times[1].tv_sec = FailTime;
 975    times[0].tv_usec = times[1].tv_usec = 0;
 976    utimes(FailFile.c_str(), times);
 977
 978    close(FailFd);
 979
 980    _exit(100);
 981 }
 982                                                                         /*}}}*/
 983 // FtpMethod::Configuration - Handle a configuration message            /*{{{*/
 984 // ---------------------------------------------------------------------
 985 /* We stash the desired pipeline depth */
 986 bool FtpMethod::Configuration(string Message)
 987 {
 988    if (pkgAcqMethod::Configuration(Message) == false)
 989       return false;
 990
 991    TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut);
 992
 993    // no more active ftp, sorry
 994    DropPrivsOrDie();
 995
 996    return true;
 997 }
 998                                                                         /*}}}*/
 999 // FtpMethod::Fetch - Fetch a file                                      /*{{{*/
1000 // ---------------------------------------------------------------------
1001 /* Fetch a single file, called by the base class..  */
1002 bool FtpMethod::Fetch(FetchItem *Itm)
1003 {
1004    URI Get = Itm->Uri;
1005    const char *File = Get.Path.c_str();
1006    FetchResult Res;
1007    Res.Filename = Itm->DestFile;
1008    Res.IMSHit = false;
1009
1010    maybe_add_auth (Get, _config->FindFile("Dir::Etc::netrc"));
1011
1012    // Connect to the server
1013    if (Server == 0 || Server->Comp(Get) == false)
1014    {
1015       delete Server;
1016       Server = new FTPConn(Get);
1017    }
1018
1019    // Could not connect is a transient error..
1020    if (Server->Open(this) == false)
1021    {
1022       Server->Close();
1023       Fail(true);
1024       return true;
1025    }
1026
1027    // Get the files information
1028    Status(_("Query"));
1029    unsigned long long Size;
1030    if (Server->Size(File,Size) == false ||
1031        Server->ModTime(File,FailTime) == false)
1032    {
1033       Fail(true);
1034       return true;
1035    }
1036    Res.Size = Size;
1037
1038    // See if it is an IMS hit
1039    if (Itm->LastModified == FailTime)
1040    {
1041       Res.Size = 0;
1042       Res.IMSHit = true;
1043       URIDone(Res);
1044       return true;
1045    }
1046
1047    // See if the file exists
1048    struct stat Buf;
1049    if (stat(Itm->DestFile.c_str(),&Buf) == 0)
1050    {
1051       if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime)
1052       {
1053          Res.Size = Buf.st_size;
1054          Res.LastModified = Buf.st_mtime;
1055          Res.ResumePoint = Buf.st_size;
1056          URIDone(Res);
1057          return true;
1058       }
1059
1060       // Resume?
1061       if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size)
1062          Res.ResumePoint = Buf.st_size;
1063    }
1064
1065    // Open the file
1066    Hashes Hash(Itm->ExpectedHashes);
1067    {
1068       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
1069       if (_error->PendingError() == true)
1070          return false;
1071
1072       URIStart(Res);
1073
1074       FailFile = Itm->DestFile;
1075       FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1076       FailFd = Fd.Fd();
1077
1078       bool Missing;
1079       if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing,Itm->MaximumSize,this) == false)
1080       {
1081          Fd.Close();
1082
1083          // Timestamp
1084          struct timeval times[2];
1085          times[0].tv_sec = FailTime;
1086          times[1].tv_sec = FailTime;
1087          times[0].tv_usec = times[1].tv_usec = 0;
1088          utimes(FailFile.c_str(), times);
1089
1090          // If the file is missing we hard fail and delete the destfile
1091          // otherwise transient fail
1092          if (Missing == true) {
1093             unlink(FailFile.c_str());
1094             return false;
1095          }
1096          Fail(true);
1097          return true;
1098       }
1099
1100       Res.Size = Fd.Size();
1101
1102       // Timestamp
1103       struct timeval times[2];
1104       times[0].tv_sec = FailTime;
1105       times[1].tv_sec = FailTime;
1106       times[0].tv_usec = times[1].tv_usec = 0;
1107       utimes(Fd.Name().c_str(), times);
1108       FailFd = -1;
1109    }
1110
1111    Res.LastModified = FailTime;
1112    Res.TakeHashes(Hash);
1113
1114    URIDone(Res);
1115
1116    return true;
1117 }
1118                                                                         /*}}}*/
1119
1120 int main(int, const char *argv[])
1121 {
1122    setlocale(LC_ALL, "");
1123
1124    /* See if we should be come the http client - we do this for http
1125       proxy urls */
1126    if (getenv("ftp_proxy") != 0)
1127    {
1128       URI Proxy = string(getenv("ftp_proxy"));
1129
1130       // Run the HTTP method
1131       if (Proxy.Access == "http")
1132       {
1133          // Copy over the environment setting
1134          char S[300];
1135          snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy"));
1136          putenv(S);
1137          putenv((char *)"no_proxy=");
1138
1139          // Run the http method
1140          string Path = flNotFile(argv[0]) + "http";
1141          execl(Path.c_str(),Path.c_str(),(char *)NULL);
1142          cerr << _("Unable to invoke ") << Path << endl;
1143          exit(100);
1144       }
1145    }
1146
1147    FtpMethod Mth;
1148
1149    return Mth.Run();
1150 }