methods/ftp.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: ftp.cc,v 1.31.2.1 2004/01/16 18:58:50 mdz Exp $
   4 /* ######################################################################
   5
   6    FTP Aquire Method - This is the FTP aquire method for APT.
   7
   8    This is a very simple implementation that does not try to optimize
   9    at all. Commands are sent syncronously with the FTP server (as the
  10    rfc recommends, but it is not really necessary..) and no tricks are
  11    done to speed things along.
  12
  13    RFC 2428 describes the IPv6 FTP behavior
  14
  15    ##################################################################### */
  16                                                                         /*}}}*/
  17 // Include Files                                                        /*{{{*/
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/acquire-method.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/hashes.h>
  22
  23 #include <sys/stat.h>
  24 #include <sys/time.h>
  25 #include <utime.h>
  26 #include <unistd.h>
  27 #include <signal.h>
  28 #include <stdio.h>
  29 #include <errno.h>
  30 #include <stdarg.h>
  31 #include <iostream>
  32 #include <apti18n.h>
  33
  34 // Internet stuff
  35 #include <netinet/in.h>
  36 #include <sys/socket.h>
  37 #include <arpa/inet.h>
  38 #include <netdb.h>
  39
  40 #include "rfc2553emu.h"
  41 #include "connect.h"
  42 #include "ftp.h"
  43                                                                         /*}}}*/
  44
  45 using namespace std;
  46
  47 /* This table is for the EPRT and EPSV commands, it maps the OS address
  48    family to the IETF address families */
  49 struct AFMap
  50 {
  51    unsigned long Family;
  52    unsigned long IETFFamily;
  53 };
  54
  55 #ifndef AF_INET6
  56 struct AFMap AFMap[] = {{AF_INET,1},{}};
  57 #else
  58 struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{}};
  59 #endif
  60
  61 unsigned long TimeOut = 120;
  62 URI Proxy;
  63 string FtpMethod::FailFile;
  64 int FtpMethod::FailFd = -1;
  65 time_t FtpMethod::FailTime = 0;
  66
  67 // FTPConn::FTPConn - Constructor                                       /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* */
  70 FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1),
  71                             DataListenFd(-1), ServerName(Srv)
  72 {
  73    Debug = _config->FindB("Debug::Acquire::Ftp",false);
  74    PasvAddr = 0;
  75 }
  76                                                                         /*}}}*/
  77 // FTPConn::~FTPConn - Destructor                                       /*{{{*/
  78 // ---------------------------------------------------------------------
  79 /* */
  80 FTPConn::~FTPConn()
  81 {
  82    Close();
  83 }
  84                                                                         /*}}}*/
  85 // FTPConn::Close - Close down the connection                           /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* Just tear down the socket and data socket */
  88 void FTPConn::Close()
  89 {
  90    close(ServerFd);
  91    ServerFd = -1;
  92    close(DataFd);
  93    DataFd = -1;
  94    close(DataListenFd);
  95    DataListenFd = -1;
  96
  97    if (PasvAddr != 0)
  98       freeaddrinfo(PasvAddr);
  99    PasvAddr = 0;
 100 }
 101                                                                         /*}}}*/
 102 // FTPConn::Open - Open a new connection                                /*{{{*/
 103 // ---------------------------------------------------------------------
 104 /* Connect to the server using a non-blocking connection and perform a
 105    login. */
 106 bool FTPConn::Open(pkgAcqMethod *Owner)
 107 {
 108    // Use the already open connection if possible.
 109    if (ServerFd != -1)
 110       return true;
 111
 112    Close();
 113
 114    // Determine the proxy setting
 115    string SpecificProxy = _config->Find("Acquire::ftp::Proxy::" + ServerName.Host);
 116    if (!SpecificProxy.empty())
 117    {
 118            if (SpecificProxy == "DIRECT")
 119                    Proxy = "";
 120            else
 121                    Proxy = SpecificProxy;
 122    }
 123    else
 124    {
 125            string DefProxy = _config->Find("Acquire::ftp::Proxy");
 126            if (!DefProxy.empty())
 127            {
 128                    Proxy = DefProxy;
 129            }
 130            else
 131            {
 132                    char* result = getenv("ftp_proxy");
 133                    Proxy = result ? result : "";
 134            }
 135    }
 136
 137    // Parse no_proxy, a , separated list of domains
 138    if (getenv("no_proxy") != 0)
 139    {
 140       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 141          Proxy = "";
 142    }
 143
 144    // Determine what host and port to use based on the proxy settings
 145    int Port = 0;
 146    string Host;
 147    if (Proxy.empty() == true)
 148    {
 149       if (ServerName.Port != 0)
 150          Port = ServerName.Port;
 151       Host = ServerName.Host;
 152    }
 153    else
 154    {
 155       if (Proxy.Port != 0)
 156          Port = Proxy.Port;
 157       Host = Proxy.Host;
 158    }
 159
 160    /* Connect to the remote server. Since FTP is connection oriented we
 161       want to make sure we get a new server every time we reconnect */
 162    RotateDNS();
 163    if (Connect(Host,Port,"ftp",21,ServerFd,TimeOut,Owner) == false)
 164       return false;
 165
 166    // Login must be before getpeername otherwise dante won't work.
 167    Owner->Status(_("Logging in"));
 168    bool Res = Login();
 169
 170    // Get the remote server's address
 171    PeerAddrLen = sizeof(PeerAddr);
 172    if (getpeername(ServerFd,(sockaddr *)&PeerAddr,&PeerAddrLen) != 0)
 173       return _error->Errno("getpeername",_("Unable to determine the peer name"));
 174
 175    // Get the local machine's address
 176    ServerAddrLen = sizeof(ServerAddr);
 177    if (getsockname(ServerFd,(sockaddr *)&ServerAddr,&ServerAddrLen) != 0)
 178       return _error->Errno("getsockname",_("Unable to determine the local name"));
 179
 180    return Res;
 181 }
 182                                                                         /*}}}*/
 183 // FTPConn::Login - Login to the remote server                          /*{{{*/
 184 // ---------------------------------------------------------------------
 185 /* This performs both normal login and proxy login using a simples script
 186    stored in the config file. */
 187 bool FTPConn::Login()
 188 {
 189    unsigned int Tag;
 190    string Msg;
 191
 192    // Setup the variables needed for authentication
 193    string User = "anonymous";
 194    string Pass = "apt_get_ftp_2.1@debian.linux.user";
 195
 196    // Fill in the user/pass
 197    if (ServerName.User.empty() == false)
 198       User = ServerName.User;
 199    if (ServerName.Password.empty() == false)
 200       Pass = ServerName.Password;
 201
 202    // Perform simple login
 203    if (Proxy.empty() == true)
 204    {
 205       // Read the initial response
 206       if (ReadResp(Tag,Msg) == false)
 207          return false;
 208       if (Tag >= 400)
 209          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 210
 211       // Send the user
 212       if (WriteMsg(Tag,Msg,"USER %s",User.c_str()) == false)
 213          return false;
 214       if (Tag >= 400)
 215          return _error->Error(_("USER failed, server said: %s"),Msg.c_str());
 216
 217       if (Tag == 331) { // 331 User name okay, need password.
 218          // Send the Password
 219          if (WriteMsg(Tag,Msg,"PASS %s",Pass.c_str()) == false)
 220             return false;
 221          if (Tag >= 400)
 222             return _error->Error(_("PASS failed, server said: %s"),Msg.c_str());
 223       }
 224
 225       // Enter passive mode
 226       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 227          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 228       else
 229          TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 230    }
 231    else
 232    {
 233       // Read the initial response
 234       if (ReadResp(Tag,Msg) == false)
 235          return false;
 236       if (Tag >= 400)
 237          return _error->Error(_("The server refused the connection and said: %s"),Msg.c_str());
 238
 239       // Perform proxy script execution
 240       Configuration::Item const *Opts = _config->Tree("Acquire::ftp::ProxyLogin");
 241       if (Opts == 0 || Opts->Child == 0)
 242          return _error->Error(_("A proxy server was specified but no login "
 243                               "script, Acquire::ftp::ProxyLogin is empty."));
 244       Opts = Opts->Child;
 245
 246       // Iterate over the entire login script
 247       for (; Opts != 0; Opts = Opts->Next)
 248       {
 249          if (Opts->Value.empty() == true)
 250             continue;
 251
 252          // Substitute the variables into the command
 253          char SitePort[20];
 254          if (ServerName.Port != 0)
 255             sprintf(SitePort,"%u",ServerName.Port);
 256          else
 257             strcpy(SitePort,"21");
 258          string Tmp = Opts->Value;
 259          Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
 260          Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
 261          Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
 262          Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
 263          Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort);
 264          Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
 265
 266          // Send the command
 267          if (WriteMsg(Tag,Msg,"%s",Tmp.c_str()) == false)
 268             return false;
 269          if (Tag >= 400)
 270             return _error->Error(_("Login script command '%s' failed, server said: %s"),Tmp.c_str(),Msg.c_str());
 271       }
 272
 273       // Enter passive mode
 274       TryPassive = false;
 275       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 276          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 277       else
 278       {
 279          if (_config->Exists("Acquire::FTP::Proxy::Passive") == true)
 280             TryPassive = _config->FindB("Acquire::FTP::Proxy::Passive",true);
 281          else
 282             TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 283       }
 284    }
 285
 286    // Force the use of extended commands
 287    if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true)
 288       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true);
 289    else
 290       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false);
 291
 292    // Binary mode
 293    if (WriteMsg(Tag,Msg,"TYPE I") == false)
 294       return false;
 295    if (Tag >= 400)
 296       return _error->Error(_("TYPE failed, server said: %s"),Msg.c_str());
 297
 298    return true;
 299 }
 300                                                                         /*}}}*/
 301 // FTPConn::ReadLine - Read a line from the server                      /*{{{*/
 302 // ---------------------------------------------------------------------
 303 /* This performs a very simple buffered read. */
 304 bool FTPConn::ReadLine(string &Text)
 305 {
 306    if (ServerFd == -1)
 307       return false;
 308
 309    // Suck in a line
 310    while (Len < sizeof(Buffer))
 311    {
 312       // Scan the buffer for a new line
 313       for (unsigned int I = 0; I != Len; I++)
 314       {
 315          // Escape some special chars
 316          if (Buffer[I] == 0)
 317             Buffer[I] = '?';
 318
 319          // End of line?
 320          if (Buffer[I] != '\n')
 321             continue;
 322
 323          I++;
 324          Text = string(Buffer,I);
 325          memmove(Buffer,Buffer+I,Len - I);
 326          Len -= I;
 327          return true;
 328       }
 329
 330       // Wait for some data..
 331       if (WaitFd(ServerFd,false,TimeOut) == false)
 332       {
 333          Close();
 334          return _error->Error(_("Connection timeout"));
 335       }
 336
 337       // Suck it back
 338       int Res = read(ServerFd,Buffer + Len,sizeof(Buffer) - Len);
 339       if (Res == 0)
 340          _error->Error(_("Server closed the connection"));
 341       if (Res <= 0)
 342       {
 343          _error->Errno("read",_("Read error"));
 344          Close();
 345          return false;
 346       }
 347       Len += Res;
 348    }
 349
 350    return _error->Error(_("A response overflowed the buffer."));
 351 }
 352                                                                         /*}}}*/
 353 // FTPConn::ReadResp - Read a full response from the server             /*{{{*/
 354 // ---------------------------------------------------------------------
 355 /* This reads a reply code from the server, it handles both p */
 356 bool FTPConn::ReadResp(unsigned int &Ret,string &Text)
 357 {
 358    // Grab the first line of the response
 359    string Msg;
 360    if (ReadLine(Msg) == false)
 361        return false;
 362
 363    // Get the ID code
 364    char *End;
 365    Ret = strtol(Msg.c_str(),&End,10);
 366    if (End - Msg.c_str() != 3)
 367       return _error->Error(_("Protocol corruption"));
 368
 369    // All done ?
 370    Text = Msg.c_str()+4;
 371    if (*End == ' ')
 372    {
 373       if (Debug == true)
 374          cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 375       return true;
 376    }
 377
 378    if (*End != '-')
 379       return _error->Error(_("Protocol corruption"));
 380
 381    /* Okay, here we do the continued message trick. This is foolish, but
 382       proftpd follows the protocol as specified and wu-ftpd doesn't, so
 383       we filter. I wonder how many clients break if you use proftpd and
 384       put a '- in the 3rd spot in the message? */
 385    char Leader[4];
 386    strncpy(Leader,Msg.c_str(),3);
 387    Leader[3] = 0;
 388    while (ReadLine(Msg) == true)
 389    {
 390       // Short, it must be using RFC continuation..
 391       if (Msg.length() < 4)
 392       {
 393          Text += Msg;
 394          continue;
 395       }
 396
 397       // Oops, finished
 398       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == ' ')
 399       {
 400          Text += Msg.c_str()+4;
 401          break;
 402       }
 403
 404       // This message has the wu-ftpd style reply code prefixed
 405       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == '-')
 406       {
 407          Text += Msg.c_str()+4;
 408          continue;
 409       }
 410
 411       // Must be RFC style prefixing
 412       Text += Msg;
 413    }
 414
 415    if (Debug == true && _error->PendingError() == false)
 416       cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 417
 418    return !_error->PendingError();
 419 }
 420                                                                         /*}}}*/
 421 // FTPConn::WriteMsg - Send a message to the server                     /*{{{*/
 422 // ---------------------------------------------------------------------
 423 /* Simple printf like function.. */
 424 bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...)
 425 {
 426    va_list args;
 427    va_start(args,Fmt);
 428
 429    // sprintf the description
 430    char S[400];
 431    vsnprintf(S,sizeof(S) - 4,Fmt,args);
 432    strcat(S,"\r\n");
 433
 434    if (Debug == true)
 435       cerr << "-> '" << QuoteString(S,"") << "'" << endl;
 436
 437    // Send it off
 438    unsigned long Len = strlen(S);
 439    unsigned long Start = 0;
 440    while (Len != 0)
 441    {
 442       if (WaitFd(ServerFd,true,TimeOut) == false)
 443       {
 444          Close();
 445          return _error->Error(_("Connection timeout"));
 446       }
 447
 448       int Res = write(ServerFd,S + Start,Len);
 449       if (Res <= 0)
 450       {
 451          _error->Errno("write",_("Write error"));
 452          Close();
 453          return false;
 454       }
 455
 456       Len -= Res;
 457       Start += Res;
 458    }
 459
 460    return ReadResp(Ret,Text);
 461 }
 462                                                                         /*}}}*/
 463 // FTPConn::GoPasv - Enter Passive mode                                 /*{{{*/
 464 // ---------------------------------------------------------------------
 465 /* Try to enter passive mode, the return code does not indicate if passive
 466    mode could or could not be established, only if there was a fatal error.
 467    We have to enter passive mode every time we make a data connection :| */
 468 bool FTPConn::GoPasv()
 469 {
 470    /* The PASV command only works on IPv4 sockets, even though it could
 471       in theory suppory IPv6 via an all zeros reply */
 472    if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET ||
 473        ForceExtended == true)
 474       return ExtGoPasv();
 475
 476    if (PasvAddr != 0)
 477       freeaddrinfo(PasvAddr);
 478    PasvAddr = 0;
 479
 480    // Try to enable pasv mode
 481    unsigned int Tag;
 482    string Msg;
 483    if (WriteMsg(Tag,Msg,"PASV") == false)
 484       return false;
 485
 486    // Unsupported function
 487    string::size_type Pos = Msg.find('(');
 488    if (Tag >= 400 || Pos == string::npos)
 489       return true;
 490
 491    // Scan it
 492    unsigned a0,a1,a2,a3,p0,p1;
 493    if (sscanf(Msg.c_str() + Pos,"(%u,%u,%u,%u,%u,%u)",&a0,&a1,&a2,&a3,&p0,&p1) != 6)
 494       return true;
 495
 496    /* Some evil servers return 0 to mean their addr. We can actually speak
 497       to these servers natively using IPv6 */
 498    if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0)
 499    {
 500       // Get the IP in text form
 501       char Name[NI_MAXHOST];
 502       char Service[NI_MAXSERV];
 503       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 504                   Name,sizeof(Name),Service,sizeof(Service),
 505                   NI_NUMERICHOST|NI_NUMERICSERV);
 506
 507       struct addrinfo Hints;
 508       memset(&Hints,0,sizeof(Hints));
 509       Hints.ai_socktype = SOCK_STREAM;
 510       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 511       Hints.ai_flags |= AI_NUMERICHOST;
 512
 513       // Get a new passive address.
 514       char Port[100];
 515       snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 516       if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 517          return true;
 518       return true;
 519    }
 520
 521    struct addrinfo Hints;
 522    memset(&Hints,0,sizeof(Hints));
 523    Hints.ai_socktype = SOCK_STREAM;
 524    Hints.ai_family = AF_INET;
 525    Hints.ai_flags |= AI_NUMERICHOST;
 526
 527    // Get a new passive address.
 528    char Port[100];
 529    snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 530    char Name[100];
 531    snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3);
 532    if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 533       return true;
 534    return true;
 535 }
 536                                                                         /*}}}*/
 537 // FTPConn::ExtGoPasv - Enter Extended Passive mode                     /*{{{*/
 538 // ---------------------------------------------------------------------
 539 /* Try to enter extended passive mode. See GoPasv above and RFC 2428 */
 540 bool FTPConn::ExtGoPasv()
 541 {
 542    if (PasvAddr != 0)
 543       freeaddrinfo(PasvAddr);
 544    PasvAddr = 0;
 545
 546    // Try to enable pasv mode
 547    unsigned int Tag;
 548    string Msg;
 549    if (WriteMsg(Tag,Msg,"EPSV") == false)
 550       return false;
 551
 552    // Unsupported function
 553    string::size_type Pos = Msg.find('(');
 554    if (Tag >= 400 || Pos == string::npos)
 555       return true;
 556
 557    // Scan it
 558    string::const_iterator List[4];
 559    unsigned Count = 0;
 560    Pos++;
 561    for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); I++)
 562    {
 563       if (*I != Msg[Pos])
 564          continue;
 565       if (Count >= 4)
 566          return true;
 567       List[Count++] = I;
 568    }
 569    if (Count != 4)
 570       return true;
 571
 572    // Break it up ..
 573    unsigned long Proto = 0;
 574    unsigned long Port = 0;
 575    string IP;
 576    IP = string(List[1]+1,List[2]);
 577    Port = atoi(string(List[2]+1,List[3]).c_str());
 578    if (IP.empty() == false)
 579       Proto = atoi(string(List[0]+1,List[1]).c_str());
 580
 581    if (Port == 0)
 582       return false;
 583
 584    // String version of the port
 585    char PStr[100];
 586    snprintf(PStr,sizeof(PStr),"%lu",Port);
 587
 588    // Get the IP in text form
 589    struct addrinfo Hints;
 590    memset(&Hints,0,sizeof(Hints));
 591    Hints.ai_socktype = SOCK_STREAM;
 592    Hints.ai_flags |= AI_NUMERICHOST;
 593
 594    /* The RFC defined case, connect to the old IP/protocol using the
 595       new port. */
 596    if (IP.empty() == true)
 597    {
 598       // Get the IP in text form
 599       char Name[NI_MAXHOST];
 600       char Service[NI_MAXSERV];
 601       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 602                   Name,sizeof(Name),Service,sizeof(Service),
 603                   NI_NUMERICHOST|NI_NUMERICSERV);
 604       IP = Name;
 605       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 606    }
 607    else
 608    {
 609       // Get the family..
 610       Hints.ai_family = 0;
 611       for (unsigned J = 0; AFMap[J].Family != 0; J++)
 612          if (AFMap[J].IETFFamily == Proto)
 613             Hints.ai_family = AFMap[J].Family;
 614       if (Hints.ai_family == 0)
 615          return true;
 616    }
 617
 618    // Get a new passive address.
 619    int Res;
 620    if ((Res = getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr)) != 0)
 621       return true;
 622
 623    return true;
 624 }
 625                                                                         /*}}}*/
 626 // FTPConn::Size - Return the size of a file                            /*{{{*/
 627 // ---------------------------------------------------------------------
 628 /* Grab the file size from the server, 0 means no size or empty file */
 629 bool FTPConn::Size(const char *Path,unsigned long &Size)
 630 {
 631    // Query the size
 632    unsigned int Tag;
 633    string Msg;
 634    Size = 0;
 635    if (WriteMsg(Tag,Msg,"SIZE %s",Path) == false)
 636       return false;
 637
 638    char *End;
 639    Size = strtol(Msg.c_str(),&End,10);
 640    if (Tag >= 400 || End == Msg.c_str())
 641       Size = 0;
 642    return true;
 643 }
 644                                                                         /*}}}*/
 645 // FTPConn::ModTime - Return the modification time of the file          /*{{{*/
 646 // ---------------------------------------------------------------------
 647 /* Like Size no error is returned if the command is not supported. If the
 648    command fails then time is set to the current time of day to fool
 649    date checks. */
 650 bool FTPConn::ModTime(const char *Path, time_t &Time)
 651 {
 652    Time = time(&Time);
 653
 654    // Query the mod time
 655    unsigned int Tag;
 656    string Msg;
 657    if (WriteMsg(Tag,Msg,"MDTM %s",Path) == false)
 658       return false;
 659    if (Tag >= 400 || Msg.empty() == true || isdigit(Msg[0]) == 0)
 660       return true;
 661
 662    // Parse it
 663    StrToTime(Msg,Time);
 664    return true;
 665 }
 666                                                                         /*}}}*/
 667 // FTPConn::CreateDataFd - Get a data connection                        /*{{{*/
 668 // ---------------------------------------------------------------------
 669 /* Create the data connection. Call FinalizeDataFd after this though.. */
 670 bool FTPConn::CreateDataFd()
 671 {
 672    close(DataFd);
 673    DataFd = -1;
 674
 675    // Attempt to enter passive mode.
 676    if (TryPassive == true)
 677    {
 678       if (GoPasv() == false)
 679          return false;
 680
 681       // Oops, didn't work out, don't bother trying again.
 682       if (PasvAddr == 0)
 683          TryPassive = false;
 684    }
 685
 686    // Passive mode?
 687    if (PasvAddr != 0)
 688    {
 689       // Get a socket
 690       if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype,
 691                            PasvAddr->ai_protocol)) < 0)
 692          return _error->Errno("socket",_("Could not create a socket"));
 693
 694       // Connect to the server
 695       SetNonBlock(DataFd,true);
 696       if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 &&
 697           errno != EINPROGRESS)
 698          return _error->Errno("socket",_("Could not create a socket"));
 699
 700       /* This implements a timeout for connect by opening the connection
 701          nonblocking */
 702       if (WaitFd(DataFd,true,TimeOut) == false)
 703          return _error->Error(_("Could not connect data socket, connection timed out"));
 704       unsigned int Err;
 705       unsigned int Len = sizeof(Err);
 706       if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 707          return _error->Errno("getsockopt",_("Failed"));
 708       if (Err != 0)
 709          return _error->Error(_("Could not connect passive socket."));
 710
 711       return true;
 712    }
 713
 714    // Port mode :<
 715    close(DataListenFd);
 716    DataListenFd = -1;
 717
 718    // Get the information for a listening socket.
 719    struct addrinfo *BindAddr = 0;
 720    struct addrinfo Hints;
 721    memset(&Hints,0,sizeof(Hints));
 722    Hints.ai_socktype = SOCK_STREAM;
 723    Hints.ai_flags |= AI_PASSIVE;
 724    Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family;
 725    int Res;
 726    if ((Res = getaddrinfo(0,"0",&Hints,&BindAddr)) != 0)
 727       return _error->Error(_("getaddrinfo was unable to get a listening socket"));
 728
 729    // Construct the socket
 730    if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype,
 731                               BindAddr->ai_protocol)) < 0)
 732    {
 733       freeaddrinfo(BindAddr);
 734       return _error->Errno("socket",_("Could not create a socket"));
 735    }
 736
 737    // Bind and listen
 738    if (bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0)
 739    {
 740       freeaddrinfo(BindAddr);
 741       return _error->Errno("bind",_("Could not bind a socket"));
 742    }
 743    freeaddrinfo(BindAddr);
 744    if (listen(DataListenFd,1) < 0)
 745       return _error->Errno("listen",_("Could not listen on the socket"));
 746    SetNonBlock(DataListenFd,true);
 747
 748    // Determine the name to send to the remote
 749    struct sockaddr_storage Addr;
 750    socklen_t AddrLen = sizeof(Addr);
 751    if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0)
 752       return _error->Errno("getsockname",_("Could not determine the socket's name"));
 753
 754
 755    // Reverse the address. We need the server address and the data port.
 756    char Name[NI_MAXHOST];
 757    char Service[NI_MAXSERV];
 758    char Service2[NI_MAXSERV];
 759    getnameinfo((struct sockaddr *)&Addr,AddrLen,
 760                Name,sizeof(Name),Service,sizeof(Service),
 761                NI_NUMERICHOST|NI_NUMERICSERV);
 762    getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen,
 763                Name,sizeof(Name),Service2,sizeof(Service2),
 764                NI_NUMERICHOST|NI_NUMERICSERV);
 765
 766    // Send off an IPv4 address in the old port format
 767    if (((struct sockaddr *)&Addr)->sa_family == AF_INET &&
 768        ForceExtended == false)
 769    {
 770       // Convert the dots in the quad into commas
 771       for (char *I = Name; *I != 0; I++)
 772          if (*I == '.')
 773             *I = ',';
 774       unsigned long Port = atoi(Service);
 775
 776       // Send the port command
 777       unsigned int Tag;
 778       string Msg;
 779       if (WriteMsg(Tag,Msg,"PORT %s,%d,%d",
 780                    Name,
 781                    (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false)
 782          return false;
 783       if (Tag >= 400)
 784          return _error->Error(_("Unable to send PORT command"));
 785       return true;
 786    }
 787
 788    // Construct an EPRT command
 789    unsigned Proto = 0;
 790    for (unsigned J = 0; AFMap[J].Family != 0; J++)
 791       if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family)
 792          Proto = AFMap[J].IETFFamily;
 793    if (Proto == 0)
 794       return _error->Error(_("Unknown address family %u (AF_*)"),
 795                            ((struct sockaddr *)&Addr)->sa_family);
 796
 797    // Send the EPRT command
 798    unsigned int Tag;
 799    string Msg;
 800    if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false)
 801       return false;
 802    if (Tag >= 400)
 803       return _error->Error(_("EPRT failed, server said: %s"),Msg.c_str());
 804    return true;
 805 }
 806                                                                         /*}}}*/
 807 // FTPConn::Finalize - Complete the Data connection                     /*{{{*/
 808 // ---------------------------------------------------------------------
 809 /* If the connection is in port mode this waits for the other end to hook
 810    up to us. */
 811 bool FTPConn::Finalize()
 812 {
 813    // Passive mode? Do nothing
 814    if (PasvAddr != 0)
 815       return true;
 816
 817    // Close any old socket..
 818    close(DataFd);
 819    DataFd = -1;
 820
 821    // Wait for someone to connect..
 822    if (WaitFd(DataListenFd,false,TimeOut) == false)
 823       return _error->Error(_("Data socket connect timed out"));
 824
 825    // Accept the connection
 826    struct sockaddr_in Addr;
 827    socklen_t Len = sizeof(Addr);
 828    DataFd = accept(DataListenFd,(struct sockaddr *)&Addr,&Len);
 829    if (DataFd < 0)
 830       return _error->Errno("accept",_("Unable to accept connection"));
 831
 832    close(DataListenFd);
 833    DataListenFd = -1;
 834
 835    return true;
 836 }
 837                                                                         /*}}}*/
 838 // FTPConn::Get - Get a file                                            /*{{{*/
 839 // ---------------------------------------------------------------------
 840 /* This opens a data connection, sends REST and RETR and then
 841    transfers the file over. */
 842 bool FTPConn::Get(const char *Path,FileFd &To,unsigned long Resume,
 843                   Hashes &Hash,bool &Missing)
 844 {
 845    Missing = false;
 846    if (CreateDataFd() == false)
 847       return false;
 848
 849    unsigned int Tag;
 850    string Msg;
 851    if (Resume != 0)
 852    {
 853       if (WriteMsg(Tag,Msg,"REST %u",Resume) == false)
 854          return false;
 855       if (Tag >= 400)
 856          Resume = 0;
 857    }
 858
 859    if (To.Truncate(Resume) == false)
 860       return false;
 861
 862    if (To.Seek(0) == false)
 863       return false;
 864
 865    if (Resume != 0)
 866    {
 867       if (Hash.AddFD(To.Fd(),Resume) == false)
 868       {
 869          _error->Errno("read",_("Problem hashing file"));
 870          return false;
 871       }
 872    }
 873
 874    // Send the get command
 875    if (WriteMsg(Tag,Msg,"RETR %s",Path) == false)
 876       return false;
 877
 878    if (Tag >= 400)
 879    {
 880       if (Tag == 550)
 881          Missing = true;
 882       return _error->Error(_("Unable to fetch file, server said '%s'"),Msg.c_str());
 883    }
 884
 885    // Finish off the data connection
 886    if (Finalize() == false)
 887       return false;
 888
 889    // Copy loop
 890    unsigned char Buffer[4096];
 891    while (1)
 892    {
 893       // Wait for some data..
 894       if (WaitFd(DataFd,false,TimeOut) == false)
 895       {
 896          Close();
 897          return _error->Error(_("Data socket timed out"));
 898       }
 899
 900       // Read the data..
 901       int Res = read(DataFd,Buffer,sizeof(Buffer));
 902       if (Res == 0)
 903          break;
 904       if (Res < 0)
 905       {
 906          if (errno == EAGAIN)
 907             continue;
 908          break;
 909       }
 910
 911       Hash.Add(Buffer,Res);
 912       if (To.Write(Buffer,Res) == false)
 913       {
 914          Close();
 915          return false;
 916       }
 917    }
 918
 919    // All done
 920    close(DataFd);
 921    DataFd = -1;
 922
 923    // Read the closing message from the server
 924    if (ReadResp(Tag,Msg) == false)
 925       return false;
 926    if (Tag >= 400)
 927       return _error->Error(_("Data transfer failed, server said '%s'"),Msg.c_str());
 928    return true;
 929 }
 930                                                                         /*}}}*/
 931
 932 // FtpMethod::FtpMethod - Constructor                                   /*{{{*/
 933 // ---------------------------------------------------------------------
 934 /* */
 935 FtpMethod::FtpMethod() : pkgAcqMethod("1.0",SendConfig)
 936 {
 937    signal(SIGTERM,SigTerm);
 938    signal(SIGINT,SigTerm);
 939
 940    Server = 0;
 941    FailFd = -1;
 942 }
 943                                                                         /*}}}*/
 944 // FtpMethod::SigTerm - Handle a fatal signal                           /*{{{*/
 945 // ---------------------------------------------------------------------
 946 /* This closes and timestamps the open file. This is neccessary to get
 947    resume behavoir on user abort */
 948 void FtpMethod::SigTerm(int)
 949 {
 950    if (FailFd == -1)
 951       _exit(100);
 952    close(FailFd);
 953
 954    // Timestamp
 955    struct utimbuf UBuf;
 956    UBuf.actime = FailTime;
 957    UBuf.modtime = FailTime;
 958    utime(FailFile.c_str(),&UBuf);
 959
 960    _exit(100);
 961 }
 962                                                                         /*}}}*/
 963 // FtpMethod::Configuration - Handle a configuration message            /*{{{*/
 964 // ---------------------------------------------------------------------
 965 /* We stash the desired pipeline depth */
 966 bool FtpMethod::Configuration(string Message)
 967 {
 968    if (pkgAcqMethod::Configuration(Message) == false)
 969       return false;
 970
 971    TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut);
 972    return true;
 973 }
 974                                                                         /*}}}*/
 975 // FtpMethod::Fetch - Fetch a file                                      /*{{{*/
 976 // ---------------------------------------------------------------------
 977 /* Fetch a single file, called by the base class..  */
 978 bool FtpMethod::Fetch(FetchItem *Itm)
 979 {
 980    URI Get = Itm->Uri;
 981    const char *File = Get.Path.c_str();
 982    FetchResult Res;
 983    Res.Filename = Itm->DestFile;
 984    Res.IMSHit = false;
 985
 986    // Connect to the server
 987    if (Server == 0 || Server->Comp(Get) == false)
 988    {
 989       delete Server;
 990       Server = new FTPConn(Get);
 991    }
 992
 993    // Could not connect is a transient error..
 994    if (Server->Open(this) == false)
 995    {
 996       Server->Close();
 997       Fail(true);
 998       return true;
 999    }
1000
1001    // Get the files information
1002    Status(_("Query"));
1003    unsigned long Size;
1004    if (Server->Size(File,Size) == false ||
1005        Server->ModTime(File,FailTime) == false)
1006    {
1007       Fail(true);
1008       return true;
1009    }
1010    Res.Size = Size;
1011
1012    // See if it is an IMS hit
1013    if (Itm->LastModified == FailTime)
1014    {
1015       Res.Size = 0;
1016       Res.IMSHit = true;
1017       URIDone(Res);
1018       return true;
1019    }
1020
1021    // See if the file exists
1022    struct stat Buf;
1023    if (stat(Itm->DestFile.c_str(),&Buf) == 0)
1024    {
1025       if (Size == (unsigned)Buf.st_size && FailTime == Buf.st_mtime)
1026       {
1027          Res.Size = Buf.st_size;
1028          Res.LastModified = Buf.st_mtime;
1029          Res.ResumePoint = Buf.st_size;
1030          URIDone(Res);
1031          return true;
1032       }
1033
1034       // Resume?
1035       if (FailTime == Buf.st_mtime && Size > (unsigned)Buf.st_size)
1036          Res.ResumePoint = Buf.st_size;
1037    }
1038
1039    // Open the file
1040    Hashes Hash;
1041    {
1042       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
1043       if (_error->PendingError() == true)
1044          return false;
1045
1046       URIStart(Res);
1047
1048       FailFile = Itm->DestFile;
1049       FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1050       FailFd = Fd.Fd();
1051
1052       bool Missing;
1053       if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing) == false)
1054       {
1055          Fd.Close();
1056
1057          // Timestamp
1058          struct utimbuf UBuf;
1059          UBuf.actime = FailTime;
1060          UBuf.modtime = FailTime;
1061          utime(FailFile.c_str(),&UBuf);
1062
1063          // If the file is missing we hard fail and delete the destfile
1064          // otherwise transient fail
1065          if (Missing == true) {
1066             unlink(FailFile.c_str());
1067             return false;
1068          }
1069          Fail(true);
1070          return true;
1071       }
1072
1073       Res.Size = Fd.Size();
1074    }
1075
1076    Res.LastModified = FailTime;
1077    Res.TakeHashes(Hash);
1078
1079    // Timestamp
1080    struct utimbuf UBuf;
1081    UBuf.actime = FailTime;
1082    UBuf.modtime = FailTime;
1083    utime(Queue->DestFile.c_str(),&UBuf);
1084    FailFd = -1;
1085
1086    URIDone(Res);
1087
1088    return true;
1089 }
1090                                                                         /*}}}*/
1091
1092 int main(int argc,const char *argv[])
1093 {
1094    setlocale(LC_ALL, "");
1095
1096    /* See if we should be come the http client - we do this for http
1097       proxy urls */
1098    if (getenv("ftp_proxy") != 0)
1099    {
1100       URI Proxy = string(getenv("ftp_proxy"));
1101
1102       // Run the HTTP method
1103       if (Proxy.Access == "http")
1104       {
1105          // Copy over the environment setting
1106          char S[300];
1107          snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy"));
1108          putenv(S);
1109          putenv((char *)"no_proxy=");
1110
1111          // Run the http method
1112          string Path = flNotFile(argv[0]) + "http";
1113          execl(Path.c_str(),Path.c_str(),(char *)NULL);
1114          cerr << _("Unable to invoke ") << Path << endl;
1115          exit(100);
1116       }
1117    }
1118
1119    FtpMethod Mth;
1120
1121    return Mth.Run();
1122 }