methods/ftp.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: ftp.cc,v 1.31.2.1 2004/01/16 18:58:50 mdz Exp $
   4 /* ######################################################################
   5
   6    FTP Aquire Method - This is the FTP aquire method for APT.
   7
   8    This is a very simple implementation that does not try to optimize
   9    at all. Commands are sent syncronously with the FTP server (as the
  10    rfc recommends, but it is not really necessary..) and no tricks are
  11    done to speed things along.
  12
  13    RFC 2428 describes the IPv6 FTP behavior
  14
  15    ##################################################################### */
  16                                                                         /*}}}*/
  17 // Include Files                                                        /*{{{*/
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/acquire-method.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/hashes.h>
  22 #include <apt-pkg/netrc.h>
  23
  24 #include <sys/stat.h>
  25 #include <sys/time.h>
  26 #include <utime.h>
  27 #include <unistd.h>
  28 #include <signal.h>
  29 #include <stdio.h>
  30 #include <errno.h>
  31 #include <stdarg.h>
  32 #include <iostream>
  33 #include <apti18n.h>
  34
  35 // Internet stuff
  36 #include <netinet/in.h>
  37 #include <sys/socket.h>
  38 #include <arpa/inet.h>
  39 #include <netdb.h>
  40
  41 #include "rfc2553emu.h"
  42 #include "connect.h"
  43 #include "ftp.h"
  44                                                                         /*}}}*/
  45
  46 using namespace std;
  47
  48 /* This table is for the EPRT and EPSV commands, it maps the OS address
  49    family to the IETF address families */
  50 struct AFMap
  51 {
  52    unsigned long Family;
  53    unsigned long IETFFamily;
  54 };
  55
  56 #ifndef AF_INET6
  57 struct AFMap AFMap[] = {{AF_INET,1},{}};
  58 #else
  59 struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{}};
  60 #endif
  61
  62 unsigned long TimeOut = 120;
  63 URI Proxy;
  64 string FtpMethod::FailFile;
  65 int FtpMethod::FailFd = -1;
  66 time_t FtpMethod::FailTime = 0;
  67
  68 // FTPConn::FTPConn - Constructor                                       /*{{{*/
  69 // ---------------------------------------------------------------------
  70 /* */
  71 FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1),
  72                             DataListenFd(-1), ServerName(Srv),
  73                             ForceExtended(false), TryPassive(true)
  74 {
  75    Debug = _config->FindB("Debug::Acquire::Ftp",false);
  76    PasvAddr = 0;
  77 }
  78                                                                         /*}}}*/
  79 // FTPConn::~FTPConn - Destructor                                       /*{{{*/
  80 // ---------------------------------------------------------------------
  81 /* */
  82 FTPConn::~FTPConn()
  83 {
  84    Close();
  85 }
  86                                                                         /*}}}*/
  87 // FTPConn::Close - Close down the connection                           /*{{{*/
  88 // ---------------------------------------------------------------------
  89 /* Just tear down the socket and data socket */
  90 void FTPConn::Close()
  91 {
  92    close(ServerFd);
  93    ServerFd = -1;
  94    close(DataFd);
  95    DataFd = -1;
  96    close(DataListenFd);
  97    DataListenFd = -1;
  98
  99    if (PasvAddr != 0)
 100       freeaddrinfo(PasvAddr);
 101    PasvAddr = 0;
 102 }
 103                                                                         /*}}}*/
 104 // FTPConn::Open - Open a new connection                                /*{{{*/
 105 // ---------------------------------------------------------------------
 106 /* Connect to the server using a non-blocking connection and perform a
 107    login. */
 108 bool FTPConn::Open(pkgAcqMethod *Owner)
 109 {
 110    // Use the already open connection if possible.
 111    if (ServerFd != -1)
 112       return true;
 113
 114    Close();
 115
 116    // Determine the proxy setting
 117    string SpecificProxy = _config->Find("Acquire::ftp::Proxy::" + ServerName.Host);
 118    if (!SpecificProxy.empty())
 119    {
 120            if (SpecificProxy == "DIRECT")
 121                    Proxy = "";
 122            else
 123                    Proxy = SpecificProxy;
 124    }
 125    else
 126    {
 127            string DefProxy = _config->Find("Acquire::ftp::Proxy");
 128            if (!DefProxy.empty())
 129            {
 130                    Proxy = DefProxy;
 131            }
 132            else
 133            {
 134                    char* result = getenv("ftp_proxy");
 135                    Proxy = result ? result : "";
 136            }
 137    }
 138
 139    // Parse no_proxy, a , separated list of domains
 140    if (getenv("no_proxy") != 0)
 141    {
 142       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 143          Proxy = "";
 144    }
 145
 146    // Determine what host and port to use based on the proxy settings
 147    int Port = 0;
 148    string Host;
 149    if (Proxy.empty() == true)
 150    {
 151       if (ServerName.Port != 0)
 152          Port = ServerName.Port;
 153       Host = ServerName.Host;
 154    }
 155    else
 156    {
 157       if (Proxy.Port != 0)
 158          Port = Proxy.Port;
 159       Host = Proxy.Host;
 160    }
 161
 162    /* Connect to the remote server. Since FTP is connection oriented we
 163       want to make sure we get a new server every time we reconnect */
 164    RotateDNS();
 165    if (Connect(Host,Port,"ftp",21,ServerFd,TimeOut,Owner) == false)
 166       return false;
 167
 168    // Login must be before getpeername otherwise dante won't work.
 169    Owner->Status(_("Logging in"));
 170    bool Res = Login();
 171
 172    // Get the remote server's address
 173    PeerAddrLen = sizeof(PeerAddr);
 174    if (getpeername(ServerFd,(sockaddr *)&PeerAddr,&PeerAddrLen) != 0)
 175       return _error->Errno("getpeername",_("Unable to determine the peer name"));
 176
 177    // Get the local machine's address
 178    ServerAddrLen = sizeof(ServerAddr);
 179    if (getsockname(ServerFd,(sockaddr *)&ServerAddr,&ServerAddrLen) != 0)
 180       return _error->Errno("getsockname",_("Unable to determine the local name"));
 181
 182    return Res;
 183 }
 184                                                                         /*}}}*/
 185 // FTPConn::Login - Login to the remote server                          /*{{{*/
 186 // ---------------------------------------------------------------------
 187 /* This performs both normal login and proxy login using a simples script
 188    stored in the config file. */
 189 bool FTPConn::Login()
 190 {
 191    unsigned int Tag;
 192    string Msg;
 193
 194    // Setup the variables needed for authentication
 195    string User = "anonymous";
 196    string Pass = "apt_get_ftp_2.1@debian.linux.user";
 197
 198    // Fill in the user/pass
 199    if (ServerName.User.empty() == false)
 200       User = ServerName.User;
 201    if (ServerName.Password.empty() == false)
 202       Pass = ServerName.Password;
 203
 204    // Perform simple login
 205    if (Proxy.empty() == true)
 206    {
 207       // Read the initial response
 208       if (ReadResp(Tag,Msg) == false)
 209          return false;
 210       if (Tag >= 400)
 211          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 212
 213       // Send the user
 214       if (WriteMsg(Tag,Msg,"USER %s",User.c_str()) == false)
 215          return false;
 216       if (Tag >= 400)
 217          return _error->Error(_("USER failed, server said: %s"),Msg.c_str());
 218
 219       if (Tag == 331) { // 331 User name okay, need password.
 220          // Send the Password
 221          if (WriteMsg(Tag,Msg,"PASS %s",Pass.c_str()) == false)
 222             return false;
 223          if (Tag >= 400)
 224             return _error->Error(_("PASS failed, server said: %s"),Msg.c_str());
 225       }
 226
 227       // Enter passive mode
 228       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 229          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 230       else
 231          TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 232    }
 233    else
 234    {
 235       // Read the initial response
 236       if (ReadResp(Tag,Msg) == false)
 237          return false;
 238       if (Tag >= 400)
 239          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 240
 241       // Perform proxy script execution
 242       Configuration::Item const *Opts = _config->Tree("Acquire::ftp::ProxyLogin");
 243       if (Opts == 0 || Opts->Child == 0)
 244          return _error->Error(_("A proxy server was specified but no login "
 245                               "script, Acquire::ftp::ProxyLogin is empty."));
 246       Opts = Opts->Child;
 247
 248       // Iterate over the entire login script
 249       for (; Opts != 0; Opts = Opts->Next)
 250       {
 251          if (Opts->Value.empty() == true)
 252             continue;
 253
 254          // Substitute the variables into the command
 255          char SitePort[20];
 256          if (ServerName.Port != 0)
 257             sprintf(SitePort,"%u",ServerName.Port);
 258          else
 259             strcpy(SitePort,"21");
 260          string Tmp = Opts->Value;
 261          Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
 262          Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
 263          Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
 264          Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
 265          Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort);
 266          Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
 267
 268          // Send the command
 269          if (WriteMsg(Tag,Msg,"%s",Tmp.c_str()) == false)
 270             return false;
 271          if (Tag >= 400)
 272             return _error->Error(_("Login script command '%s' failed, server said: %s"),Tmp.c_str(),Msg.c_str());
 273       }
 274
 275       // Enter passive mode
 276       TryPassive = false;
 277       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 278          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 279       else
 280       {
 281          if (_config->Exists("Acquire::FTP::Proxy::Passive") == true)
 282             TryPassive = _config->FindB("Acquire::FTP::Proxy::Passive",true);
 283          else
 284             TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 285       }
 286    }
 287
 288    // Force the use of extended commands
 289    if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true)
 290       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true);
 291    else
 292       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false);
 293
 294    // Binary mode
 295    if (WriteMsg(Tag,Msg,"TYPE I") == false)
 296       return false;
 297    if (Tag >= 400)
 298       return _error->Error(_("TYPE failed, server said: %s"),Msg.c_str());
 299
 300    return true;
 301 }
 302                                                                         /*}}}*/
 303 // FTPConn::ReadLine - Read a line from the server                      /*{{{*/
 304 // ---------------------------------------------------------------------
 305 /* This performs a very simple buffered read. */
 306 bool FTPConn::ReadLine(string &Text)
 307 {
 308    if (ServerFd == -1)
 309       return false;
 310
 311    // Suck in a line
 312    while (Len < sizeof(Buffer))
 313    {
 314       // Scan the buffer for a new line
 315       for (unsigned int I = 0; I != Len; I++)
 316       {
 317          // Escape some special chars
 318          if (Buffer[I] == 0)
 319             Buffer[I] = '?';
 320
 321          // End of line?
 322          if (Buffer[I] != '\n')
 323             continue;
 324
 325          I++;
 326          Text = string(Buffer,I);
 327          memmove(Buffer,Buffer+I,Len - I);
 328          Len -= I;
 329          return true;
 330       }
 331
 332       // Wait for some data..
 333       if (WaitFd(ServerFd,false,TimeOut) == false)
 334       {
 335          Close();
 336          return _error->Error(_("Connection timeout"));
 337       }
 338
 339       // Suck it back
 340       int Res = read(ServerFd,Buffer + Len,sizeof(Buffer) - Len);
 341       if (Res == 0)
 342          _error->Error(_("Server closed the connection"));
 343       if (Res <= 0)
 344       {
 345          _error->Errno("read",_("Read error"));
 346          Close();
 347          return false;
 348       }
 349       Len += Res;
 350    }
 351
 352    return _error->Error(_("A response overflowed the buffer."));
 353 }
 354                                                                         /*}}}*/
 355 // FTPConn::ReadResp - Read a full response from the server             /*{{{*/
 356 // ---------------------------------------------------------------------
 357 /* This reads a reply code from the server, it handles both p */
 358 bool FTPConn::ReadResp(unsigned int &Ret,string &Text)
 359 {
 360    // Grab the first line of the response
 361    string Msg;
 362    if (ReadLine(Msg) == false)
 363        return false;
 364
 365    // Get the ID code
 366    char *End;
 367    Ret = strtol(Msg.c_str(),&End,10);
 368    if (End - Msg.c_str() != 3)
 369       return _error->Error(_("Protocol corruption"));
 370
 371    // All done ?
 372    Text = Msg.c_str()+4;
 373    if (*End == ' ')
 374    {
 375       if (Debug == true)
 376          cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 377       return true;
 378    }
 379
 380    if (*End != '-')
 381       return _error->Error(_("Protocol corruption"));
 382
 383    /* Okay, here we do the continued message trick. This is foolish, but
 384       proftpd follows the protocol as specified and wu-ftpd doesn't, so
 385       we filter. I wonder how many clients break if you use proftpd and
 386       put a '- in the 3rd spot in the message? */
 387    char Leader[4];
 388    strncpy(Leader,Msg.c_str(),3);
 389    Leader[3] = 0;
 390    while (ReadLine(Msg) == true)
 391    {
 392       // Short, it must be using RFC continuation..
 393       if (Msg.length() < 4)
 394       {
 395          Text += Msg;
 396          continue;
 397       }
 398
 399       // Oops, finished
 400       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == ' ')
 401       {
 402          Text += Msg.c_str()+4;
 403          break;
 404       }
 405
 406       // This message has the wu-ftpd style reply code prefixed
 407       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == '-')
 408       {
 409          Text += Msg.c_str()+4;
 410          continue;
 411       }
 412
 413       // Must be RFC style prefixing
 414       Text += Msg;
 415    }
 416
 417    if (Debug == true && _error->PendingError() == false)
 418       cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 419
 420    return !_error->PendingError();
 421 }
 422                                                                         /*}}}*/
 423 // FTPConn::WriteMsg - Send a message to the server                     /*{{{*/
 424 // ---------------------------------------------------------------------
 425 /* Simple printf like function.. */
 426 bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...)
 427 {
 428    va_list args;
 429    va_start(args,Fmt);
 430
 431    // sprintf the description
 432    char S[400];
 433    vsnprintf(S,sizeof(S) - 4,Fmt,args);
 434    strcat(S,"\r\n");
 435
 436    if (Debug == true)
 437       cerr << "-> '" << QuoteString(S,"") << "'" << endl;
 438
 439    // Send it off
 440    unsigned long Len = strlen(S);
 441    unsigned long Start = 0;
 442    while (Len != 0)
 443    {
 444       if (WaitFd(ServerFd,true,TimeOut) == false)
 445       {
 446          Close();
 447          return _error->Error(_("Connection timeout"));
 448       }
 449
 450       int Res = write(ServerFd,S + Start,Len);
 451       if (Res <= 0)
 452       {
 453          _error->Errno("write",_("Write error"));
 454          Close();
 455          return false;
 456       }
 457
 458       Len -= Res;
 459       Start += Res;
 460    }
 461
 462    return ReadResp(Ret,Text);
 463 }
 464                                                                         /*}}}*/
 465 // FTPConn::GoPasv - Enter Passive mode                                 /*{{{*/
 466 // ---------------------------------------------------------------------
 467 /* Try to enter passive mode, the return code does not indicate if passive
 468    mode could or could not be established, only if there was a fatal error.
 469    We have to enter passive mode every time we make a data connection :| */
 470 bool FTPConn::GoPasv()
 471 {
 472    /* The PASV command only works on IPv4 sockets, even though it could
 473       in theory suppory IPv6 via an all zeros reply */
 474    if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET ||
 475        ForceExtended == true)
 476       return ExtGoPasv();
 477
 478    if (PasvAddr != 0)
 479       freeaddrinfo(PasvAddr);
 480    PasvAddr = 0;
 481
 482    // Try to enable pasv mode
 483    unsigned int Tag;
 484    string Msg;
 485    if (WriteMsg(Tag,Msg,"PASV") == false)
 486       return false;
 487
 488    // Unsupported function
 489    string::size_type Pos = Msg.find('(');
 490    if (Tag >= 400 || Pos == string::npos)
 491       return true;
 492
 493    // Scan it
 494    unsigned a0,a1,a2,a3,p0,p1;
 495    if (sscanf(Msg.c_str() + Pos,"(%u,%u,%u,%u,%u,%u)",&a0,&a1,&a2,&a3,&p0,&p1) != 6)
 496       return true;
 497
 498    /* Some evil servers return 0 to mean their addr. We can actually speak
 499       to these servers natively using IPv6 */
 500    if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0)
 501    {
 502       // Get the IP in text form
 503       char Name[NI_MAXHOST];
 504       char Service[NI_MAXSERV];
 505       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 506                   Name,sizeof(Name),Service,sizeof(Service),
 507                   NI_NUMERICHOST|NI_NUMERICSERV);
 508
 509       struct addrinfo Hints;
 510       memset(&Hints,0,sizeof(Hints));
 511       Hints.ai_socktype = SOCK_STREAM;
 512       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 513       Hints.ai_flags |= AI_NUMERICHOST;
 514
 515       // Get a new passive address.
 516       char Port[100];
 517       snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 518       if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 519          return true;
 520       return true;
 521    }
 522
 523    struct addrinfo Hints;
 524    memset(&Hints,0,sizeof(Hints));
 525    Hints.ai_socktype = SOCK_STREAM;
 526    Hints.ai_family = AF_INET;
 527    Hints.ai_flags |= AI_NUMERICHOST;
 528
 529    // Get a new passive address.
 530    char Port[100];
 531    snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 532    char Name[100];
 533    snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3);
 534    if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 535       return true;
 536    return true;
 537 }
 538                                                                         /*}}}*/
 539 // FTPConn::ExtGoPasv - Enter Extended Passive mode                     /*{{{*/
 540 // ---------------------------------------------------------------------
 541 /* Try to enter extended passive mode. See GoPasv above and RFC 2428 */
 542 bool FTPConn::ExtGoPasv()
 543 {
 544    if (PasvAddr != 0)
 545       freeaddrinfo(PasvAddr);
 546    PasvAddr = 0;
 547
 548    // Try to enable pasv mode
 549    unsigned int Tag;
 550    string Msg;
 551    if (WriteMsg(Tag,Msg,"EPSV") == false)
 552       return false;
 553
 554    // Unsupported function
 555    string::size_type Pos = Msg.find('(');
 556    if (Tag >= 400 || Pos == string::npos)
 557       return true;
 558
 559    // Scan it
 560    string::const_iterator List[4];
 561    unsigned Count = 0;
 562    Pos++;
 563    for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); ++I)
 564    {
 565       if (*I != Msg[Pos])
 566          continue;
 567       if (Count >= 4)
 568          return true;
 569       List[Count++] = I;
 570    }
 571    if (Count != 4)
 572       return true;
 573
 574    // Break it up ..
 575    unsigned long Proto = 0;
 576    unsigned long Port = 0;
 577    string IP;
 578    IP = string(List[1]+1,List[2]);
 579    Port = atoi(string(List[2]+1,List[3]).c_str());
 580    if (IP.empty() == false)
 581       Proto = atoi(string(List[0]+1,List[1]).c_str());
 582
 583    if (Port == 0)
 584       return false;
 585
 586    // String version of the port
 587    char PStr[100];
 588    snprintf(PStr,sizeof(PStr),"%lu",Port);
 589
 590    // Get the IP in text form
 591    struct addrinfo Hints;
 592    memset(&Hints,0,sizeof(Hints));
 593    Hints.ai_socktype = SOCK_STREAM;
 594    Hints.ai_flags |= AI_NUMERICHOST;
 595
 596    /* The RFC defined case, connect to the old IP/protocol using the
 597       new port. */
 598    if (IP.empty() == true)
 599    {
 600       // Get the IP in text form
 601       char Name[NI_MAXHOST];
 602       char Service[NI_MAXSERV];
 603       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 604                   Name,sizeof(Name),Service,sizeof(Service),
 605                   NI_NUMERICHOST|NI_NUMERICSERV);
 606       IP = Name;
 607       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 608    }
 609    else
 610    {
 611       // Get the family..
 612       Hints.ai_family = 0;
 613       for (unsigned J = 0; AFMap[J].Family != 0; J++)
 614          if (AFMap[J].IETFFamily == Proto)
 615             Hints.ai_family = AFMap[J].Family;
 616       if (Hints.ai_family == 0)
 617          return true;
 618    }
 619
 620    // Get a new passive address.
 621    int Res;
 622    if ((Res = getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr)) != 0)
 623       return true;
 624
 625    return true;
 626 }
 627                                                                         /*}}}*/
 628 // FTPConn::Size - Return the size of a file                            /*{{{*/
 629 // ---------------------------------------------------------------------
 630 /* Grab the file size from the server, 0 means no size or empty file */
 631 bool FTPConn::Size(const char *Path,unsigned long &Size)
 632 {
 633    // Query the size
 634    unsigned int Tag;
 635    string Msg;
 636    Size = 0;
 637    if (WriteMsg(Tag,Msg,"SIZE %s",Path) == false)
 638       return false;
 639
 640    char *End;
 641    Size = strtol(Msg.c_str(),&End,10);
 642    if (Tag >= 400 || End == Msg.c_str())
 643       Size = 0;
 644    return true;
 645 }
 646                                                                         /*}}}*/
 647 // FTPConn::ModTime - Return the modification time of the file          /*{{{*/
 648 // ---------------------------------------------------------------------
 649 /* Like Size no error is returned if the command is not supported. If the
 650    command fails then time is set to the current time of day to fool
 651    date checks. */
 652 bool FTPConn::ModTime(const char *Path, time_t &Time)
 653 {
 654    Time = time(&Time);
 655
 656    // Query the mod time
 657    unsigned int Tag;
 658    string Msg;
 659    if (WriteMsg(Tag,Msg,"MDTM %s",Path) == false)
 660       return false;
 661    if (Tag >= 400 || Msg.empty() == true || isdigit(Msg[0]) == 0)
 662       return true;
 663
 664    // Parse it
 665    return FTPMDTMStrToTime(Msg.c_str(), Time);
 666 }
 667                                                                         /*}}}*/
 668 // FTPConn::CreateDataFd - Get a data connection                        /*{{{*/
 669 // ---------------------------------------------------------------------
 670 /* Create the data connection. Call FinalizeDataFd after this though.. */
 671 bool FTPConn::CreateDataFd()
 672 {
 673    close(DataFd);
 674    DataFd = -1;
 675
 676    // Attempt to enter passive mode.
 677    if (TryPassive == true)
 678    {
 679       if (GoPasv() == false)
 680          return false;
 681
 682       // Oops, didn't work out, don't bother trying again.
 683       if (PasvAddr == 0)
 684          TryPassive = false;
 685    }
 686
 687    // Passive mode?
 688    if (PasvAddr != 0)
 689    {
 690       // Get a socket
 691       if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype,
 692                            PasvAddr->ai_protocol)) < 0)
 693          return _error->Errno("socket",_("Could not create a socket"));
 694
 695       // Connect to the server
 696       SetNonBlock(DataFd,true);
 697       if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 &&
 698           errno != EINPROGRESS)
 699          return _error->Errno("socket",_("Could not create a socket"));
 700
 701       /* This implements a timeout for connect by opening the connection
 702          nonblocking */
 703       if (WaitFd(DataFd,true,TimeOut) == false)
 704          return _error->Error(_("Could not connect data socket, connection timed out"));
 705       unsigned int Err;
 706       unsigned int Len = sizeof(Err);
 707       if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 708          return _error->Errno("getsockopt",_("Failed"));
 709       if (Err != 0)
 710          return _error->Error(_("Could not connect passive socket."));
 711
 712       return true;
 713    }
 714
 715    // Port mode :<
 716    close(DataListenFd);
 717    DataListenFd = -1;
 718
 719    // Get the information for a listening socket.
 720    struct addrinfo *BindAddr = 0;
 721    struct addrinfo Hints;
 722    memset(&Hints,0,sizeof(Hints));
 723    Hints.ai_socktype = SOCK_STREAM;
 724    Hints.ai_flags |= AI_PASSIVE;
 725    Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family;
 726    int Res;
 727    if ((Res = getaddrinfo(0,"0",&Hints,&BindAddr)) != 0)
 728       return _error->Error(_("getaddrinfo was unable to get a listening socket"));
 729
 730    // Construct the socket
 731    if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype,
 732                               BindAddr->ai_protocol)) < 0)
 733    {
 734       freeaddrinfo(BindAddr);
 735       return _error->Errno("socket",_("Could not create a socket"));
 736    }
 737
 738    // Bind and listen
 739    if (bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0)
 740    {
 741       freeaddrinfo(BindAddr);
 742       return _error->Errno("bind",_("Could not bind a socket"));
 743    }
 744    freeaddrinfo(BindAddr);
 745    if (listen(DataListenFd,1) < 0)
 746       return _error->Errno("listen",_("Could not listen on the socket"));
 747    SetNonBlock(DataListenFd,true);
 748
 749    // Determine the name to send to the remote
 750    struct sockaddr_storage Addr;
 751    socklen_t AddrLen = sizeof(Addr);
 752    if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0)
 753       return _error->Errno("getsockname",_("Could not determine the socket's name"));
 754
 755
 756    // Reverse the address. We need the server address and the data port.
 757    char Name[NI_MAXHOST];
 758    char Service[NI_MAXSERV];
 759    char Service2[NI_MAXSERV];
 760    getnameinfo((struct sockaddr *)&Addr,AddrLen,
 761                Name,sizeof(Name),Service,sizeof(Service),
 762                NI_NUMERICHOST|NI_NUMERICSERV);
 763    getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen,
 764                Name,sizeof(Name),Service2,sizeof(Service2),
 765                NI_NUMERICHOST|NI_NUMERICSERV);
 766
 767    // Send off an IPv4 address in the old port format
 768    if (((struct sockaddr *)&Addr)->sa_family == AF_INET &&
 769        ForceExtended == false)
 770    {
 771       // Convert the dots in the quad into commas
 772       for (char *I = Name; *I != 0; I++)
 773          if (*I == '.')
 774             *I = ',';
 775       unsigned long Port = atoi(Service);
 776
 777       // Send the port command
 778       unsigned int Tag;
 779       string Msg;
 780       if (WriteMsg(Tag,Msg,"PORT %s,%d,%d",
 781                    Name,
 782                    (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false)
 783          return false;
 784       if (Tag >= 400)
 785          return _error->Error(_("Unable to send PORT command"));
 786       return true;
 787    }
 788
 789    // Construct an EPRT command
 790    unsigned Proto = 0;
 791    for (unsigned J = 0; AFMap[J].Family != 0; J++)
 792       if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family)
 793          Proto = AFMap[J].IETFFamily;
 794    if (Proto == 0)
 795       return _error->Error(_("Unknown address family %u (AF_*)"),
 796                            ((struct sockaddr *)&Addr)->sa_family);
 797
 798    // Send the EPRT command
 799    unsigned int Tag;
 800    string Msg;
 801    if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false)
 802       return false;
 803    if (Tag >= 400)
 804       return _error->Error(_("EPRT failed, server said: %s"),Msg.c_str());
 805    return true;
 806 }
 807                                                                         /*}}}*/
 808 // FTPConn::Finalize - Complete the Data connection                     /*{{{*/
 809 // ---------------------------------------------------------------------
 810 /* If the connection is in port mode this waits for the other end to hook
 811    up to us. */
 812 bool FTPConn::Finalize()
 813 {
 814    // Passive mode? Do nothing
 815    if (PasvAddr != 0)
 816       return true;
 817
 818    // Close any old socket..
 819    close(DataFd);
 820    DataFd = -1;
 821
 822    // Wait for someone to connect..
 823    if (WaitFd(DataListenFd,false,TimeOut) == false)
 824       return _error->Error(_("Data socket connect timed out"));
 825
 826    // Accept the connection
 827    struct sockaddr_in Addr;
 828    socklen_t Len = sizeof(Addr);
 829    DataFd = accept(DataListenFd,(struct sockaddr *)&Addr,&Len);
 830    if (DataFd < 0)
 831       return _error->Errno("accept",_("Unable to accept connection"));
 832
 833    close(DataListenFd);
 834    DataListenFd = -1;
 835
 836    return true;
 837 }
 838                                                                         /*}}}*/
 839 // FTPConn::Get - Get a file                                            /*{{{*/
 840 // ---------------------------------------------------------------------
 841 /* This opens a data connection, sends REST and RETR and then
 842    transfers the file over. */
 843 bool FTPConn::Get(const char *Path,FileFd &To,unsigned long Resume,
 844                   Hashes &Hash,bool &Missing)
 845 {
 846    Missing = false;
 847    if (CreateDataFd() == false)
 848       return false;
 849
 850    unsigned int Tag;
 851    string Msg;
 852    if (Resume != 0)
 853    {
 854       if (WriteMsg(Tag,Msg,"REST %u",Resume) == false)
 855          return false;
 856       if (Tag >= 400)
 857          Resume = 0;
 858    }
 859
 860    if (To.Truncate(Resume) == false)
 861       return false;
 862
 863    if (To.Seek(0) == false)
 864       return false;
 865
 866    if (Resume != 0)
 867    {
 868       if (Hash.AddFD(To.Fd(),Resume) == false)
 869       {
 870          _error->Errno("read",_("Problem hashing file"));
 871          return false;
 872       }
 873    }
 874
 875    // Send the get command
 876    if (WriteMsg(Tag,Msg,"RETR %s",Path) == false)
 877       return false;
 878
 879    if (Tag >= 400)
 880    {
 881       if (Tag == 550)
 882          Missing = true;
 883       return _error->Error(_("Unable to fetch file, server said '%s'"),Msg.c_str());
 884    }
 885
 886    // Finish off the data connection
 887    if (Finalize() == false)
 888       return false;
 889
 890    // Copy loop
 891    unsigned char Buffer[4096];
 892    while (1)
 893    {
 894       // Wait for some data..
 895       if (WaitFd(DataFd,false,TimeOut) == false)
 896       {
 897          Close();
 898          return _error->Error(_("Data socket timed out"));
 899       }
 900
 901       // Read the data..
 902       int Res = read(DataFd,Buffer,sizeof(Buffer));
 903       if (Res == 0)
 904          break;
 905       if (Res < 0)
 906       {
 907          if (errno == EAGAIN)
 908             continue;
 909          break;
 910       }
 911
 912       Hash.Add(Buffer,Res);
 913       if (To.Write(Buffer,Res) == false)
 914       {
 915          Close();
 916          return false;
 917       }
 918    }
 919
 920    // All done
 921    close(DataFd);
 922    DataFd = -1;
 923
 924    // Read the closing message from the server
 925    if (ReadResp(Tag,Msg) == false)
 926       return false;
 927    if (Tag >= 400)
 928       return _error->Error(_("Data transfer failed, server said '%s'"),Msg.c_str());
 929    return true;
 930 }
 931                                                                         /*}}}*/
 932
 933 // FtpMethod::FtpMethod - Constructor                                   /*{{{*/
 934 // ---------------------------------------------------------------------
 935 /* */
 936 FtpMethod::FtpMethod() : pkgAcqMethod("1.0",SendConfig)
 937 {
 938    signal(SIGTERM,SigTerm);
 939    signal(SIGINT,SigTerm);
 940
 941    Server = 0;
 942    FailFd = -1;
 943 }
 944                                                                         /*}}}*/
 945 // FtpMethod::SigTerm - Handle a fatal signal                           /*{{{*/
 946 // ---------------------------------------------------------------------
 947 /* This closes and timestamps the open file. This is neccessary to get
 948    resume behavoir on user abort */
 949 void FtpMethod::SigTerm(int)
 950 {
 951    if (FailFd == -1)
 952       _exit(100);
 953    close(FailFd);
 954
 955    // Timestamp
 956    struct utimbuf UBuf;
 957    UBuf.actime = FailTime;
 958    UBuf.modtime = FailTime;
 959    utime(FailFile.c_str(),&UBuf);
 960
 961    _exit(100);
 962 }
 963                                                                         /*}}}*/
 964 // FtpMethod::Configuration - Handle a configuration message            /*{{{*/
 965 // ---------------------------------------------------------------------
 966 /* We stash the desired pipeline depth */
 967 bool FtpMethod::Configuration(string Message)
 968 {
 969    if (pkgAcqMethod::Configuration(Message) == false)
 970       return false;
 971
 972    TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut);
 973    return true;
 974 }
 975                                                                         /*}}}*/
 976 // FtpMethod::Fetch - Fetch a file                                      /*{{{*/
 977 // ---------------------------------------------------------------------
 978 /* Fetch a single file, called by the base class..  */
 979 bool FtpMethod::Fetch(FetchItem *Itm)
 980 {
 981    URI Get = Itm->Uri;
 982    const char *File = Get.Path.c_str();
 983    FetchResult Res;
 984    Res.Filename = Itm->DestFile;
 985    Res.IMSHit = false;
 986
 987    maybe_add_auth (Get, _config->FindFile("Dir::Etc::netrc"));
 988
 989    // Connect to the server
 990    if (Server == 0 || Server->Comp(Get) == false)
 991    {
 992       delete Server;
 993       Server = new FTPConn(Get);
 994    }
 995
 996    // Could not connect is a transient error..
 997    if (Server->Open(this) == false)
 998    {
 999       Server->Close();
1000       Fail(true);
1001       return true;
1002    }
1003
1004    // Get the files information
1005    Status(_("Query"));
1006    unsigned long Size;
1007    if (Server->Size(File,Size) == false ||
1008        Server->ModTime(File,FailTime) == false)
1009    {
1010       Fail(true);
1011       return true;
1012    }
1013    Res.Size = Size;
1014
1015    // See if it is an IMS hit
1016    if (Itm->LastModified == FailTime)
1017    {
1018       Res.Size = 0;
1019       Res.IMSHit = true;
1020       URIDone(Res);
1021       return true;
1022    }
1023
1024    // See if the file exists
1025    struct stat Buf;
1026    if (stat(Itm->DestFile.c_str(),&Buf) == 0)
1027    {
1028       if (Size == (unsigned)Buf.st_size && FailTime == Buf.st_mtime)
1029       {
1030          Res.Size = Buf.st_size;
1031          Res.LastModified = Buf.st_mtime;
1032          Res.ResumePoint = Buf.st_size;
1033          URIDone(Res);
1034          return true;
1035       }
1036
1037       // Resume?
1038       if (FailTime == Buf.st_mtime && Size > (unsigned)Buf.st_size)
1039          Res.ResumePoint = Buf.st_size;
1040    }
1041
1042    // Open the file
1043    Hashes Hash;
1044    {
1045       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
1046       if (_error->PendingError() == true)
1047          return false;
1048
1049       URIStart(Res);
1050
1051       FailFile = Itm->DestFile;
1052       FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1053       FailFd = Fd.Fd();
1054
1055       bool Missing;
1056       if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing) == false)
1057       {
1058          Fd.Close();
1059
1060          // Timestamp
1061          struct utimbuf UBuf;
1062          UBuf.actime = FailTime;
1063          UBuf.modtime = FailTime;
1064          utime(FailFile.c_str(),&UBuf);
1065
1066          // If the file is missing we hard fail and delete the destfile
1067          // otherwise transient fail
1068          if (Missing == true) {
1069             unlink(FailFile.c_str());
1070             return false;
1071          }
1072          Fail(true);
1073          return true;
1074       }
1075
1076       Res.Size = Fd.Size();
1077    }
1078
1079    Res.LastModified = FailTime;
1080    Res.TakeHashes(Hash);
1081
1082    // Timestamp
1083    struct utimbuf UBuf;
1084    UBuf.actime = FailTime;
1085    UBuf.modtime = FailTime;
1086    utime(Queue->DestFile.c_str(),&UBuf);
1087    FailFd = -1;
1088
1089    URIDone(Res);
1090
1091    return true;
1092 }
1093                                                                         /*}}}*/
1094
1095 int main(int argc,const char *argv[])
1096 {
1097    setlocale(LC_ALL, "");
1098
1099    /* See if we should be come the http client - we do this for http
1100       proxy urls */
1101    if (getenv("ftp_proxy") != 0)
1102    {
1103       URI Proxy = string(getenv("ftp_proxy"));
1104
1105       // Run the HTTP method
1106       if (Proxy.Access == "http")
1107       {
1108          // Copy over the environment setting
1109          char S[300];
1110          snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy"));
1111          putenv(S);
1112          putenv((char *)"no_proxy=");
1113
1114          // Run the http method
1115          string Path = flNotFile(argv[0]) + "http";
1116          execl(Path.c_str(),Path.c_str(),(char *)NULL);
1117          cerr << _("Unable to invoke ") << Path << endl;
1118          exit(100);
1119       }
1120    }
1121
1122    FtpMethod Mth;
1123
1124    return Mth.Run();
1125 }