methods/ftp.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: ftp.cc,v 1.31.2.1 2004/01/16 18:58:50 mdz Exp $
   4 /* ######################################################################
   5
   6    FTP Aquire Method - This is the FTP aquire method for APT.
   7
   8    This is a very simple implementation that does not try to optimize
   9    at all. Commands are sent syncronously with the FTP server (as the
  10    rfc recommends, but it is not really necessary..) and no tricks are
  11    done to speed things along.
  12
  13    RFC 2428 describes the IPv6 FTP behavior
  14
  15    ##################################################################### */
  16                                                                         /*}}}*/
  17 // Include Files                                                        /*{{{*/
  18 #include <config.h>
  19
  20 #include <apt-pkg/fileutl.h>
  21 #include <apt-pkg/acquire-method.h>
  22 #include <apt-pkg/error.h>
  23 #include <apt-pkg/hashes.h>
  24 #include <apt-pkg/netrc.h>
  25 #include <apt-pkg/configuration.h>
  26
  27 #include <sys/stat.h>
  28 #include <sys/time.h>
  29 #include <utime.h>
  30 #include <unistd.h>
  31 #include <signal.h>
  32 #include <stdio.h>
  33 #include <errno.h>
  34 #include <stdarg.h>
  35 #include <iostream>
  36
  37 // Internet stuff
  38 #include <netinet/in.h>
  39 #include <sys/socket.h>
  40 #include <arpa/inet.h>
  41 #include <netdb.h>
  42
  43 #include "rfc2553emu.h"
  44 #include "connect.h"
  45 #include "ftp.h"
  46 #include <apti18n.h>
  47                                                                         /*}}}*/
  48
  49 using namespace std;
  50
  51 /* This table is for the EPRT and EPSV commands, it maps the OS address
  52    family to the IETF address families */
  53 struct AFMap
  54 {
  55    unsigned long Family;
  56    unsigned long IETFFamily;
  57 };
  58
  59 #ifndef AF_INET6
  60 struct AFMap AFMap[] = {{AF_INET,1},{}};
  61 #else
  62 struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{}};
  63 #endif
  64
  65 unsigned long TimeOut = 120;
  66 URI Proxy;
  67 string FtpMethod::FailFile;
  68 int FtpMethod::FailFd = -1;
  69 time_t FtpMethod::FailTime = 0;
  70
  71 // FTPConn::FTPConn - Constructor                                       /*{{{*/
  72 // ---------------------------------------------------------------------
  73 /* */
  74 FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1),
  75                             DataListenFd(-1), ServerName(Srv),
  76                             ForceExtended(false), TryPassive(true)
  77 {
  78    Debug = _config->FindB("Debug::Acquire::Ftp",false);
  79    PasvAddr = 0;
  80    Buffer[0] = '\0';
  81 }
  82                                                                         /*}}}*/
  83 // FTPConn::~FTPConn - Destructor                                       /*{{{*/
  84 // ---------------------------------------------------------------------
  85 /* */
  86 FTPConn::~FTPConn()
  87 {
  88    Close();
  89 }
  90                                                                         /*}}}*/
  91 // FTPConn::Close - Close down the connection                           /*{{{*/
  92 // ---------------------------------------------------------------------
  93 /* Just tear down the socket and data socket */
  94 void FTPConn::Close()
  95 {
  96    close(ServerFd);
  97    ServerFd = -1;
  98    close(DataFd);
  99    DataFd = -1;
 100    close(DataListenFd);
 101    DataListenFd = -1;
 102
 103    if (PasvAddr != 0)
 104       freeaddrinfo(PasvAddr);
 105    PasvAddr = 0;
 106 }
 107                                                                         /*}}}*/
 108 // FTPConn::Open - Open a new connection                                /*{{{*/
 109 // ---------------------------------------------------------------------
 110 /* Connect to the server using a non-blocking connection and perform a
 111    login. */
 112 bool FTPConn::Open(pkgAcqMethod *Owner)
 113 {
 114    // Use the already open connection if possible.
 115    if (ServerFd != -1)
 116       return true;
 117
 118    Close();
 119
 120    // Determine the proxy setting
 121    string SpecificProxy = _config->Find("Acquire::ftp::Proxy::" + ServerName.Host);
 122    if (!SpecificProxy.empty())
 123    {
 124            if (SpecificProxy == "DIRECT")
 125                    Proxy = "";
 126            else
 127                    Proxy = SpecificProxy;
 128    }
 129    else
 130    {
 131            string DefProxy = _config->Find("Acquire::ftp::Proxy");
 132            if (!DefProxy.empty())
 133            {
 134                    Proxy = DefProxy;
 135            }
 136            else
 137            {
 138                    char* result = getenv("ftp_proxy");
 139                    Proxy = result ? result : "";
 140            }
 141    }
 142
 143    // Parse no_proxy, a , separated list of domains
 144    if (getenv("no_proxy") != 0)
 145    {
 146       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 147          Proxy = "";
 148    }
 149
 150    // Determine what host and port to use based on the proxy settings
 151    int Port = 0;
 152    string Host;
 153    if (Proxy.empty() == true)
 154    {
 155       if (ServerName.Port != 0)
 156          Port = ServerName.Port;
 157       Host = ServerName.Host;
 158    }
 159    else
 160    {
 161       if (Proxy.Port != 0)
 162          Port = Proxy.Port;
 163       Host = Proxy.Host;
 164    }
 165
 166    /* Connect to the remote server. Since FTP is connection oriented we
 167       want to make sure we get a new server every time we reconnect */
 168    RotateDNS();
 169    if (Connect(Host,Port,"ftp",21,ServerFd,TimeOut,Owner) == false)
 170       return false;
 171
 172    // Login must be before getpeername otherwise dante won't work.
 173    Owner->Status(_("Logging in"));
 174    bool Res = Login();
 175
 176    // Get the remote server's address
 177    PeerAddrLen = sizeof(PeerAddr);
 178    if (getpeername(ServerFd,(sockaddr *)&PeerAddr,&PeerAddrLen) != 0)
 179       return _error->Errno("getpeername",_("Unable to determine the peer name"));
 180
 181    // Get the local machine's address
 182    ServerAddrLen = sizeof(ServerAddr);
 183    if (getsockname(ServerFd,(sockaddr *)&ServerAddr,&ServerAddrLen) != 0)
 184       return _error->Errno("getsockname",_("Unable to determine the local name"));
 185
 186    return Res;
 187 }
 188                                                                         /*}}}*/
 189 // FTPConn::Login - Login to the remote server                          /*{{{*/
 190 // ---------------------------------------------------------------------
 191 /* This performs both normal login and proxy login using a simples script
 192    stored in the config file. */
 193 bool FTPConn::Login()
 194 {
 195    unsigned int Tag;
 196    string Msg;
 197
 198    // Setup the variables needed for authentication
 199    string User = "anonymous";
 200    string Pass = "apt_get_ftp_2.1@debian.linux.user";
 201
 202    // Fill in the user/pass
 203    if (ServerName.User.empty() == false)
 204       User = ServerName.User;
 205    if (ServerName.Password.empty() == false)
 206       Pass = ServerName.Password;
 207
 208    // Perform simple login
 209    if (Proxy.empty() == true)
 210    {
 211       // Read the initial response
 212       if (ReadResp(Tag,Msg) == false)
 213          return false;
 214       if (Tag >= 400)
 215          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 216
 217       // Send the user
 218       if (WriteMsg(Tag,Msg,"USER %s",User.c_str()) == false)
 219          return false;
 220       if (Tag >= 400)
 221          return _error->Error(_("USER failed, server said: %s"),Msg.c_str());
 222
 223       if (Tag == 331) { // 331 User name okay, need password.
 224          // Send the Password
 225          if (WriteMsg(Tag,Msg,"PASS %s",Pass.c_str()) == false)
 226             return false;
 227          if (Tag >= 400)
 228             return _error->Error(_("PASS failed, server said: %s"),Msg.c_str());
 229       }
 230
 231       // Enter passive mode
 232       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 233          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 234       else
 235          TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 236    }
 237    else
 238    {
 239       // Read the initial response
 240       if (ReadResp(Tag,Msg) == false)
 241          return false;
 242       if (Tag >= 400)
 243          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 244
 245       // Perform proxy script execution
 246       Configuration::Item const *Opts = _config->Tree("Acquire::ftp::ProxyLogin");
 247       if (Opts == 0 || Opts->Child == 0)
 248          return _error->Error(_("A proxy server was specified but no login "
 249                               "script, Acquire::ftp::ProxyLogin is empty."));
 250       Opts = Opts->Child;
 251
 252       // Iterate over the entire login script
 253       for (; Opts != 0; Opts = Opts->Next)
 254       {
 255          if (Opts->Value.empty() == true)
 256             continue;
 257
 258          // Substitute the variables into the command
 259          char SitePort[20];
 260          if (ServerName.Port != 0)
 261             sprintf(SitePort,"%u",ServerName.Port);
 262          else
 263             strcpy(SitePort,"21");
 264          string Tmp = Opts->Value;
 265          Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
 266          Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
 267          Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
 268          Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
 269          Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort);
 270          Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
 271
 272          // Send the command
 273          if (WriteMsg(Tag,Msg,"%s",Tmp.c_str()) == false)
 274             return false;
 275          if (Tag >= 400)
 276             return _error->Error(_("Login script command '%s' failed, server said: %s"),Tmp.c_str(),Msg.c_str());
 277       }
 278
 279       // Enter passive mode
 280       TryPassive = false;
 281       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 282          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 283       else
 284       {
 285          if (_config->Exists("Acquire::FTP::Proxy::Passive") == true)
 286             TryPassive = _config->FindB("Acquire::FTP::Proxy::Passive",true);
 287          else
 288             TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 289       }
 290    }
 291
 292    // Force the use of extended commands
 293    if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true)
 294       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true);
 295    else
 296       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false);
 297
 298    // Binary mode
 299    if (WriteMsg(Tag,Msg,"TYPE I") == false)
 300       return false;
 301    if (Tag >= 400)
 302       return _error->Error(_("TYPE failed, server said: %s"),Msg.c_str());
 303
 304    return true;
 305 }
 306                                                                         /*}}}*/
 307 // FTPConn::ReadLine - Read a line from the server                      /*{{{*/
 308 // ---------------------------------------------------------------------
 309 /* This performs a very simple buffered read. */
 310 bool FTPConn::ReadLine(string &Text)
 311 {
 312    if (ServerFd == -1)
 313       return false;
 314
 315    // Suck in a line
 316    while (Len < sizeof(Buffer))
 317    {
 318       // Scan the buffer for a new line
 319       for (unsigned int I = 0; I != Len; I++)
 320       {
 321          // Escape some special chars
 322          if (Buffer[I] == 0)
 323             Buffer[I] = '?';
 324
 325          // End of line?
 326          if (Buffer[I] != '\n')
 327             continue;
 328
 329          I++;
 330          Text = string(Buffer,I);
 331          memmove(Buffer,Buffer+I,Len - I);
 332          Len -= I;
 333          return true;
 334       }
 335
 336       // Wait for some data..
 337       if (WaitFd(ServerFd,false,TimeOut) == false)
 338       {
 339          Close();
 340          return _error->Error(_("Connection timeout"));
 341       }
 342
 343       // Suck it back
 344       int Res = read(ServerFd,Buffer + Len,sizeof(Buffer) - Len);
 345       if (Res == 0)
 346          _error->Error(_("Server closed the connection"));
 347       if (Res <= 0)
 348       {
 349          _error->Errno("read",_("Read error"));
 350          Close();
 351          return false;
 352       }
 353       Len += Res;
 354    }
 355
 356    return _error->Error(_("A response overflowed the buffer."));
 357 }
 358                                                                         /*}}}*/
 359 // FTPConn::ReadResp - Read a full response from the server             /*{{{*/
 360 // ---------------------------------------------------------------------
 361 /* This reads a reply code from the server, it handles both p */
 362 bool FTPConn::ReadResp(unsigned int &Ret,string &Text)
 363 {
 364    // Grab the first line of the response
 365    string Msg;
 366    if (ReadLine(Msg) == false)
 367        return false;
 368
 369    // Get the ID code
 370    char *End;
 371    Ret = strtol(Msg.c_str(),&End,10);
 372    if (End - Msg.c_str() != 3)
 373       return _error->Error(_("Protocol corruption"));
 374
 375    // All done ?
 376    Text = Msg.c_str()+4;
 377    if (*End == ' ')
 378    {
 379       if (Debug == true)
 380          cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 381       return true;
 382    }
 383
 384    if (*End != '-')
 385       return _error->Error(_("Protocol corruption"));
 386
 387    /* Okay, here we do the continued message trick. This is foolish, but
 388       proftpd follows the protocol as specified and wu-ftpd doesn't, so
 389       we filter. I wonder how many clients break if you use proftpd and
 390       put a '- in the 3rd spot in the message? */
 391    char Leader[4];
 392    strncpy(Leader,Msg.c_str(),3);
 393    Leader[3] = 0;
 394    while (ReadLine(Msg) == true)
 395    {
 396       // Short, it must be using RFC continuation..
 397       if (Msg.length() < 4)
 398       {
 399          Text += Msg;
 400          continue;
 401       }
 402
 403       // Oops, finished
 404       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == ' ')
 405       {
 406          Text += Msg.c_str()+4;
 407          break;
 408       }
 409
 410       // This message has the wu-ftpd style reply code prefixed
 411       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == '-')
 412       {
 413          Text += Msg.c_str()+4;
 414          continue;
 415       }
 416
 417       // Must be RFC style prefixing
 418       Text += Msg;
 419    }
 420
 421    if (Debug == true && _error->PendingError() == false)
 422       cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 423
 424    return !_error->PendingError();
 425 }
 426                                                                         /*}}}*/
 427 // FTPConn::WriteMsg - Send a message to the server                     /*{{{*/
 428 // ---------------------------------------------------------------------
 429 /* Simple printf like function.. */
 430 bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...)
 431 {
 432    va_list args;
 433    va_start(args,Fmt);
 434
 435    // sprintf the description
 436    char S[400];
 437    vsnprintf(S,sizeof(S) - 4,Fmt,args);
 438    strcat(S,"\r\n");
 439
 440    if (Debug == true)
 441       cerr << "-> '" << QuoteString(S,"") << "'" << endl;
 442
 443    // Send it off
 444    unsigned long Len = strlen(S);
 445    unsigned long Start = 0;
 446    while (Len != 0)
 447    {
 448       if (WaitFd(ServerFd,true,TimeOut) == false)
 449       {
 450          Close();
 451          return _error->Error(_("Connection timeout"));
 452       }
 453
 454       int Res = write(ServerFd,S + Start,Len);
 455       if (Res <= 0)
 456       {
 457          _error->Errno("write",_("Write error"));
 458          Close();
 459          return false;
 460       }
 461
 462       Len -= Res;
 463       Start += Res;
 464    }
 465
 466    return ReadResp(Ret,Text);
 467 }
 468                                                                         /*}}}*/
 469 // FTPConn::GoPasv - Enter Passive mode                                 /*{{{*/
 470 // ---------------------------------------------------------------------
 471 /* Try to enter passive mode, the return code does not indicate if passive
 472    mode could or could not be established, only if there was a fatal error.
 473    We have to enter passive mode every time we make a data connection :| */
 474 bool FTPConn::GoPasv()
 475 {
 476    /* The PASV command only works on IPv4 sockets, even though it could
 477       in theory suppory IPv6 via an all zeros reply */
 478    if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET ||
 479        ForceExtended == true)
 480       return ExtGoPasv();
 481
 482    if (PasvAddr != 0)
 483       freeaddrinfo(PasvAddr);
 484    PasvAddr = 0;
 485
 486    // Try to enable pasv mode
 487    unsigned int Tag;
 488    string Msg;
 489    if (WriteMsg(Tag,Msg,"PASV") == false)
 490       return false;
 491
 492    // Unsupported function
 493    string::size_type Pos = Msg.find('(');
 494    if (Tag >= 400 || Pos == string::npos)
 495       return true;
 496
 497    // Scan it
 498    unsigned a0,a1,a2,a3,p0,p1;
 499    if (sscanf(Msg.c_str() + Pos,"(%u,%u,%u,%u,%u,%u)",&a0,&a1,&a2,&a3,&p0,&p1) != 6)
 500       return true;
 501
 502    /* Some evil servers return 0 to mean their addr. We can actually speak
 503       to these servers natively using IPv6 */
 504    if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0)
 505    {
 506       // Get the IP in text form
 507       char Name[NI_MAXHOST];
 508       char Service[NI_MAXSERV];
 509       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 510                   Name,sizeof(Name),Service,sizeof(Service),
 511                   NI_NUMERICHOST|NI_NUMERICSERV);
 512
 513       struct addrinfo Hints;
 514       memset(&Hints,0,sizeof(Hints));
 515       Hints.ai_socktype = SOCK_STREAM;
 516       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 517       Hints.ai_flags |= AI_NUMERICHOST;
 518
 519       // Get a new passive address.
 520       char Port[100];
 521       snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 522       if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 523          return true;
 524       return true;
 525    }
 526
 527    struct addrinfo Hints;
 528    memset(&Hints,0,sizeof(Hints));
 529    Hints.ai_socktype = SOCK_STREAM;
 530    Hints.ai_family = AF_INET;
 531    Hints.ai_flags |= AI_NUMERICHOST;
 532
 533    // Get a new passive address.
 534    char Port[100];
 535    snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 536    char Name[100];
 537    snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3);
 538    if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 539       return true;
 540    return true;
 541 }
 542                                                                         /*}}}*/
 543 // FTPConn::ExtGoPasv - Enter Extended Passive mode                     /*{{{*/
 544 // ---------------------------------------------------------------------
 545 /* Try to enter extended passive mode. See GoPasv above and RFC 2428 */
 546 bool FTPConn::ExtGoPasv()
 547 {
 548    if (PasvAddr != 0)
 549       freeaddrinfo(PasvAddr);
 550    PasvAddr = 0;
 551
 552    // Try to enable pasv mode
 553    unsigned int Tag;
 554    string Msg;
 555    if (WriteMsg(Tag,Msg,"EPSV") == false)
 556       return false;
 557
 558    // Unsupported function
 559    string::size_type Pos = Msg.find('(');
 560    if (Tag >= 400 || Pos == string::npos)
 561       return true;
 562
 563    // Scan it
 564    string::const_iterator List[4];
 565    unsigned Count = 0;
 566    Pos++;
 567    for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); ++I)
 568    {
 569       if (*I != Msg[Pos])
 570          continue;
 571       if (Count >= 4)
 572          return true;
 573       List[Count++] = I;
 574    }
 575    if (Count != 4)
 576       return true;
 577
 578    // Break it up ..
 579    unsigned long Proto = 0;
 580    unsigned long Port = 0;
 581    string IP;
 582    IP = string(List[1]+1,List[2]);
 583    Port = atoi(string(List[2]+1,List[3]).c_str());
 584    if (IP.empty() == false)
 585       Proto = atoi(string(List[0]+1,List[1]).c_str());
 586
 587    if (Port == 0)
 588       return false;
 589
 590    // String version of the port
 591    char PStr[100];
 592    snprintf(PStr,sizeof(PStr),"%lu",Port);
 593
 594    // Get the IP in text form
 595    struct addrinfo Hints;
 596    memset(&Hints,0,sizeof(Hints));
 597    Hints.ai_socktype = SOCK_STREAM;
 598    Hints.ai_flags |= AI_NUMERICHOST;
 599
 600    /* The RFC defined case, connect to the old IP/protocol using the
 601       new port. */
 602    if (IP.empty() == true)
 603    {
 604       // Get the IP in text form
 605       char Name[NI_MAXHOST];
 606       char Service[NI_MAXSERV];
 607       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 608                   Name,sizeof(Name),Service,sizeof(Service),
 609                   NI_NUMERICHOST|NI_NUMERICSERV);
 610       IP = Name;
 611       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 612    }
 613    else
 614    {
 615       // Get the family..
 616       Hints.ai_family = 0;
 617       for (unsigned J = 0; AFMap[J].Family != 0; J++)
 618          if (AFMap[J].IETFFamily == Proto)
 619             Hints.ai_family = AFMap[J].Family;
 620       if (Hints.ai_family == 0)
 621          return true;
 622    }
 623
 624    // Get a new passive address.
 625    if (getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr) != 0)
 626       return true;
 627
 628    return true;
 629 }
 630                                                                         /*}}}*/
 631 // FTPConn::Size - Return the size of a file                            /*{{{*/
 632 // ---------------------------------------------------------------------
 633 /* Grab the file size from the server, 0 means no size or empty file */
 634 bool FTPConn::Size(const char *Path,unsigned long long &Size)
 635 {
 636    // Query the size
 637    unsigned int Tag;
 638    string Msg;
 639    Size = 0;
 640    if (WriteMsg(Tag,Msg,"SIZE %s",Path) == false)
 641       return false;
 642
 643    char *End;
 644    Size = strtoull(Msg.c_str(),&End,10);
 645    if (Tag >= 400 || End == Msg.c_str())
 646       Size = 0;
 647    return true;
 648 }
 649                                                                         /*}}}*/
 650 // FTPConn::ModTime - Return the modification time of the file          /*{{{*/
 651 // ---------------------------------------------------------------------
 652 /* Like Size no error is returned if the command is not supported. If the
 653    command fails then time is set to the current time of day to fool
 654    date checks. */
 655 bool FTPConn::ModTime(const char *Path, time_t &Time)
 656 {
 657    Time = time(&Time);
 658
 659    // Query the mod time
 660    unsigned int Tag;
 661    string Msg;
 662    if (WriteMsg(Tag,Msg,"MDTM %s",Path) == false)
 663       return false;
 664    if (Tag >= 400 || Msg.empty() == true || isdigit(Msg[0]) == 0)
 665       return true;
 666
 667    // Parse it
 668    return FTPMDTMStrToTime(Msg.c_str(), Time);
 669 }
 670                                                                         /*}}}*/
 671 // FTPConn::CreateDataFd - Get a data connection                        /*{{{*/
 672 // ---------------------------------------------------------------------
 673 /* Create the data connection. Call FinalizeDataFd after this though.. */
 674 bool FTPConn::CreateDataFd()
 675 {
 676    close(DataFd);
 677    DataFd = -1;
 678
 679    // Attempt to enter passive mode.
 680    if (TryPassive == true)
 681    {
 682       if (GoPasv() == false)
 683          return false;
 684
 685       // Oops, didn't work out, don't bother trying again.
 686       if (PasvAddr == 0)
 687          TryPassive = false;
 688    }
 689
 690    // Passive mode?
 691    if (PasvAddr != 0)
 692    {
 693       // Get a socket
 694       if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype,
 695                            PasvAddr->ai_protocol)) < 0)
 696          return _error->Errno("socket",_("Could not create a socket"));
 697
 698       // Connect to the server
 699       SetNonBlock(DataFd,true);
 700       if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 &&
 701           errno != EINPROGRESS)
 702          return _error->Errno("socket",_("Could not create a socket"));
 703
 704       /* This implements a timeout for connect by opening the connection
 705          nonblocking */
 706       if (WaitFd(DataFd,true,TimeOut) == false)
 707          return _error->Error(_("Could not connect data socket, connection timed out"));
 708       unsigned int Err;
 709       unsigned int Len = sizeof(Err);
 710       if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 711          return _error->Errno("getsockopt",_("Failed"));
 712       if (Err != 0)
 713          return _error->Error(_("Could not connect passive socket."));
 714
 715       return true;
 716    }
 717
 718    // Port mode :<
 719    close(DataListenFd);
 720    DataListenFd = -1;
 721
 722    // Get the information for a listening socket.
 723    struct addrinfo *BindAddr = NULL;
 724    struct addrinfo Hints;
 725    memset(&Hints,0,sizeof(Hints));
 726    Hints.ai_socktype = SOCK_STREAM;
 727    Hints.ai_flags |= AI_PASSIVE;
 728    Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family;
 729    if (getaddrinfo(0,"0",&Hints,&BindAddr) != 0 || BindAddr == NULL)
 730       return _error->Error(_("getaddrinfo was unable to get a listening socket"));
 731
 732    // Construct the socket
 733    if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype,
 734                               BindAddr->ai_protocol)) < 0)
 735    {
 736       freeaddrinfo(BindAddr);
 737       return _error->Errno("socket",_("Could not create a socket"));
 738    }
 739
 740    // Bind and listen
 741    if (bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0)
 742    {
 743       freeaddrinfo(BindAddr);
 744       return _error->Errno("bind",_("Could not bind a socket"));
 745    }
 746    freeaddrinfo(BindAddr);
 747    if (listen(DataListenFd,1) < 0)
 748       return _error->Errno("listen",_("Could not listen on the socket"));
 749    SetNonBlock(DataListenFd,true);
 750
 751    // Determine the name to send to the remote
 752    struct sockaddr_storage Addr;
 753    socklen_t AddrLen = sizeof(Addr);
 754    if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0)
 755       return _error->Errno("getsockname",_("Could not determine the socket's name"));
 756
 757
 758    // Reverse the address. We need the server address and the data port.
 759    char Name[NI_MAXHOST];
 760    char Service[NI_MAXSERV];
 761    char Service2[NI_MAXSERV];
 762    getnameinfo((struct sockaddr *)&Addr,AddrLen,
 763                Name,sizeof(Name),Service,sizeof(Service),
 764                NI_NUMERICHOST|NI_NUMERICSERV);
 765    getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen,
 766                Name,sizeof(Name),Service2,sizeof(Service2),
 767                NI_NUMERICHOST|NI_NUMERICSERV);
 768
 769    // Send off an IPv4 address in the old port format
 770    if (((struct sockaddr *)&Addr)->sa_family == AF_INET &&
 771        ForceExtended == false)
 772    {
 773       // Convert the dots in the quad into commas
 774       for (char *I = Name; *I != 0; I++)
 775          if (*I == '.')
 776             *I = ',';
 777       unsigned long Port = atoi(Service);
 778
 779       // Send the port command
 780       unsigned int Tag;
 781       string Msg;
 782       if (WriteMsg(Tag,Msg,"PORT %s,%d,%d",
 783                    Name,
 784                    (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false)
 785          return false;
 786       if (Tag >= 400)
 787          return _error->Error(_("Unable to send PORT command"));
 788       return true;
 789    }
 790
 791    // Construct an EPRT command
 792    unsigned Proto = 0;
 793    for (unsigned J = 0; AFMap[J].Family != 0; J++)
 794       if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family)
 795          Proto = AFMap[J].IETFFamily;
 796    if (Proto == 0)
 797       return _error->Error(_("Unknown address family %u (AF_*)"),
 798                            ((struct sockaddr *)&Addr)->sa_family);
 799
 800    // Send the EPRT command
 801    unsigned int Tag;
 802    string Msg;
 803    if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false)
 804       return false;
 805    if (Tag >= 400)
 806       return _error->Error(_("EPRT failed, server said: %s"),Msg.c_str());
 807    return true;
 808 }
 809                                                                         /*}}}*/
 810 // FTPConn::Finalize - Complete the Data connection                     /*{{{*/
 811 // ---------------------------------------------------------------------
 812 /* If the connection is in port mode this waits for the other end to hook
 813    up to us. */
 814 bool FTPConn::Finalize()
 815 {
 816    // Passive mode? Do nothing
 817    if (PasvAddr != 0)
 818       return true;
 819
 820    // Close any old socket..
 821    close(DataFd);
 822    DataFd = -1;
 823
 824    // Wait for someone to connect..
 825    if (WaitFd(DataListenFd,false,TimeOut) == false)
 826       return _error->Error(_("Data socket connect timed out"));
 827
 828    // Accept the connection
 829    struct sockaddr_in Addr;
 830    socklen_t Len = sizeof(Addr);
 831    DataFd = accept(DataListenFd,(struct sockaddr *)&Addr,&Len);
 832    if (DataFd < 0)
 833       return _error->Errno("accept",_("Unable to accept connection"));
 834
 835    close(DataListenFd);
 836    DataListenFd = -1;
 837
 838    return true;
 839 }
 840                                                                         /*}}}*/
 841 // FTPConn::Get - Get a file                                            /*{{{*/
 842 // ---------------------------------------------------------------------
 843 /* This opens a data connection, sends REST and RETR and then
 844    transfers the file over. */
 845 bool FTPConn::Get(const char *Path,FileFd &To,unsigned long long Resume,
 846                   Hashes &Hash,bool &Missing)
 847 {
 848    Missing = false;
 849    if (CreateDataFd() == false)
 850       return false;
 851
 852    unsigned int Tag;
 853    string Msg;
 854    if (Resume != 0)
 855    {
 856       if (WriteMsg(Tag,Msg,"REST %u",Resume) == false)
 857          return false;
 858       if (Tag >= 400)
 859          Resume = 0;
 860    }
 861
 862    if (To.Truncate(Resume) == false)
 863       return false;
 864
 865    if (To.Seek(0) == false)
 866       return false;
 867
 868    if (Resume != 0)
 869    {
 870       if (Hash.AddFD(To,Resume) == false)
 871       {
 872          _error->Errno("read",_("Problem hashing file"));
 873          return false;
 874       }
 875    }
 876
 877    // Send the get command
 878    if (WriteMsg(Tag,Msg,"RETR %s",Path) == false)
 879       return false;
 880
 881    if (Tag >= 400)
 882    {
 883       if (Tag == 550)
 884          Missing = true;
 885       return _error->Error(_("Unable to fetch file, server said '%s'"),Msg.c_str());
 886    }
 887
 888    // Finish off the data connection
 889    if (Finalize() == false)
 890       return false;
 891
 892    // Copy loop
 893    unsigned char Buffer[4096];
 894    while (1)
 895    {
 896       // Wait for some data..
 897       if (WaitFd(DataFd,false,TimeOut) == false)
 898       {
 899          Close();
 900          return _error->Error(_("Data socket timed out"));
 901       }
 902
 903       // Read the data..
 904       int Res = read(DataFd,Buffer,sizeof(Buffer));
 905       if (Res == 0)
 906          break;
 907       if (Res < 0)
 908       {
 909          if (errno == EAGAIN)
 910             continue;
 911          break;
 912       }
 913
 914       Hash.Add(Buffer,Res);
 915       if (To.Write(Buffer,Res) == false)
 916       {
 917          Close();
 918          return false;
 919       }
 920    }
 921
 922    // All done
 923    close(DataFd);
 924    DataFd = -1;
 925
 926    // Read the closing message from the server
 927    if (ReadResp(Tag,Msg) == false)
 928       return false;
 929    if (Tag >= 400)
 930       return _error->Error(_("Data transfer failed, server said '%s'"),Msg.c_str());
 931    return true;
 932 }
 933                                                                         /*}}}*/
 934
 935 // FtpMethod::FtpMethod - Constructor                                   /*{{{*/
 936 // ---------------------------------------------------------------------
 937 /* */
 938 FtpMethod::FtpMethod() : pkgAcqMethod("1.0",SendConfig)
 939 {
 940    signal(SIGTERM,SigTerm);
 941    signal(SIGINT,SigTerm);
 942
 943    Server = 0;
 944    FailFd = -1;
 945 }
 946                                                                         /*}}}*/
 947 // FtpMethod::SigTerm - Handle a fatal signal                           /*{{{*/
 948 // ---------------------------------------------------------------------
 949 /* This closes and timestamps the open file. This is neccessary to get
 950    resume behavoir on user abort */
 951 void FtpMethod::SigTerm(int)
 952 {
 953    if (FailFd == -1)
 954       _exit(100);
 955    close(FailFd);
 956
 957    // Timestamp
 958    struct utimbuf UBuf;
 959    UBuf.actime = FailTime;
 960    UBuf.modtime = FailTime;
 961    utime(FailFile.c_str(),&UBuf);
 962
 963    _exit(100);
 964 }
 965                                                                         /*}}}*/
 966 // FtpMethod::Configuration - Handle a configuration message            /*{{{*/
 967 // ---------------------------------------------------------------------
 968 /* We stash the desired pipeline depth */
 969 bool FtpMethod::Configuration(string Message)
 970 {
 971    if (pkgAcqMethod::Configuration(Message) == false)
 972       return false;
 973
 974    TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut);
 975    return true;
 976 }
 977                                                                         /*}}}*/
 978 // FtpMethod::Fetch - Fetch a file                                      /*{{{*/
 979 // ---------------------------------------------------------------------
 980 /* Fetch a single file, called by the base class..  */
 981 bool FtpMethod::Fetch(FetchItem *Itm)
 982 {
 983    URI Get = Itm->Uri;
 984    const char *File = Get.Path.c_str();
 985    FetchResult Res;
 986    Res.Filename = Itm->DestFile;
 987    Res.IMSHit = false;
 988
 989    maybe_add_auth (Get, _config->FindFile("Dir::Etc::netrc"));
 990
 991    // Connect to the server
 992    if (Server == 0 || Server->Comp(Get) == false)
 993    {
 994       delete Server;
 995       Server = new FTPConn(Get);
 996    }
 997
 998    // Could not connect is a transient error..
 999    if (Server->Open(this) == false)
1000    {
1001       Server->Close();
1002       Fail(true);
1003       return true;
1004    }
1005
1006    // Get the files information
1007    Status(_("Query"));
1008    unsigned long long Size;
1009    if (Server->Size(File,Size) == false ||
1010        Server->ModTime(File,FailTime) == false)
1011    {
1012       Fail(true);
1013       return true;
1014    }
1015    Res.Size = Size;
1016
1017    // See if it is an IMS hit
1018    if (Itm->LastModified == FailTime)
1019    {
1020       Res.Size = 0;
1021       Res.IMSHit = true;
1022       URIDone(Res);
1023       return true;
1024    }
1025
1026    // See if the file exists
1027    struct stat Buf;
1028    if (stat(Itm->DestFile.c_str(),&Buf) == 0)
1029    {
1030       if (Size == (unsigned long long)Buf.st_size && FailTime == Buf.st_mtime)
1031       {
1032          Res.Size = Buf.st_size;
1033          Res.LastModified = Buf.st_mtime;
1034          Res.ResumePoint = Buf.st_size;
1035          URIDone(Res);
1036          return true;
1037       }
1038
1039       // Resume?
1040       if (FailTime == Buf.st_mtime && Size > (unsigned long long)Buf.st_size)
1041          Res.ResumePoint = Buf.st_size;
1042    }
1043
1044    // Open the file
1045    Hashes Hash;
1046    {
1047       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
1048       if (_error->PendingError() == true)
1049          return false;
1050
1051       URIStart(Res);
1052
1053       FailFile = Itm->DestFile;
1054       FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1055       FailFd = Fd.Fd();
1056
1057       bool Missing;
1058       if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing) == false)
1059       {
1060          Fd.Close();
1061
1062          // Timestamp
1063          struct utimbuf UBuf;
1064          UBuf.actime = FailTime;
1065          UBuf.modtime = FailTime;
1066          utime(FailFile.c_str(),&UBuf);
1067
1068          // If the file is missing we hard fail and delete the destfile
1069          // otherwise transient fail
1070          if (Missing == true) {
1071             unlink(FailFile.c_str());
1072             return false;
1073          }
1074          Fail(true);
1075          return true;
1076       }
1077
1078       Res.Size = Fd.Size();
1079    }
1080
1081    Res.LastModified = FailTime;
1082    Res.TakeHashes(Hash);
1083
1084    // Timestamp
1085    struct utimbuf UBuf;
1086    UBuf.actime = FailTime;
1087    UBuf.modtime = FailTime;
1088    utime(Queue->DestFile.c_str(),&UBuf);
1089    FailFd = -1;
1090
1091    URIDone(Res);
1092
1093    return true;
1094 }
1095                                                                         /*}}}*/
1096
1097 int main(int argc,const char *argv[])
1098 {
1099    setlocale(LC_ALL, "");
1100
1101    /* See if we should be come the http client - we do this for http
1102       proxy urls */
1103    if (getenv("ftp_proxy") != 0)
1104    {
1105       URI Proxy = string(getenv("ftp_proxy"));
1106
1107       // Run the HTTP method
1108       if (Proxy.Access == "http")
1109       {
1110          // Copy over the environment setting
1111          char S[300];
1112          snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy"));
1113          putenv(S);
1114          putenv((char *)"no_proxy=");
1115
1116          // Run the http method
1117          string Path = flNotFile(argv[0]) + "http";
1118          execl(Path.c_str(),Path.c_str(),(char *)NULL);
1119          cerr << _("Unable to invoke ") << Path << endl;
1120          exit(100);
1121       }
1122    }
1123
1124    FtpMethod Mth;
1125
1126    return Mth.Run();
1127 }