methods/ftp.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: ftp.cc,v 1.31.2.1 2004/01/16 18:58:50 mdz Exp $
   4 /* ######################################################################
   5
   6    FTP Aquire Method - This is the FTP aquire method for APT.
   7
   8    This is a very simple implementation that does not try to optimize
   9    at all. Commands are sent syncronously with the FTP server (as the
  10    rfc recommends, but it is not really necessary..) and no tricks are
  11    done to speed things along.
  12
  13    RFC 2428 describes the IPv6 FTP behavior
  14
  15    ##################################################################### */
  16                                                                         /*}}}*/
  17 // Include Files                                                        /*{{{*/
  18 #include <apt-pkg/fileutl.h>
  19 #include <apt-pkg/acquire-method.h>
  20 #include <apt-pkg/error.h>
  21 #include <apt-pkg/hashes.h>
  22
  23 #include <sys/stat.h>
  24 #include <sys/time.h>
  25 #include <utime.h>
  26 #include <unistd.h>
  27 #include <signal.h>
  28 #include <stdio.h>
  29 #include <errno.h>
  30 #include <stdarg.h>
  31 #include <iostream>
  32 #include <apti18n.h>
  33
  34 // Internet stuff
  35 #include <netinet/in.h>
  36 #include <sys/socket.h>
  37 #include <arpa/inet.h>
  38 #include <netdb.h>
  39
  40 #include "rfc2553emu.h"
  41 #include "connect.h"
  42 #include "ftp.h"
  43                                                                         /*}}}*/
  44
  45 using namespace std;
  46
  47 /* This table is for the EPRT and EPSV commands, it maps the OS address
  48    family to the IETF address families */
  49 struct AFMap
  50 {
  51    unsigned long Family;
  52    unsigned long IETFFamily;
  53 };
  54
  55 #ifndef AF_INET6
  56 struct AFMap AFMap[] = {{AF_INET,1},{}};
  57 #else
  58 struct AFMap AFMap[] = {{AF_INET,1},{AF_INET6,2},{}};
  59 #endif
  60
  61 unsigned long TimeOut = 120;
  62 URI Proxy;
  63 string FtpMethod::FailFile;
  64 int FtpMethod::FailFd = -1;
  65 time_t FtpMethod::FailTime = 0;
  66
  67 // FTPConn::FTPConn - Constructor                                       /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* */
  70 FTPConn::FTPConn(URI Srv) : Len(0), ServerFd(-1), DataFd(-1),
  71                             DataListenFd(-1), ServerName(Srv)
  72 {
  73    Debug = _config->FindB("Debug::Acquire::Ftp",false);
  74    PasvAddr = 0;
  75 }
  76                                                                         /*}}}*/
  77 // FTPConn::~FTPConn - Destructor                                       /*{{{*/
  78 // ---------------------------------------------------------------------
  79 /* */
  80 FTPConn::~FTPConn()
  81 {
  82    Close();
  83 }
  84                                                                         /*}}}*/
  85 // FTPConn::Close - Close down the connection                           /*{{{*/
  86 // ---------------------------------------------------------------------
  87 /* Just tear down the socket and data socket */
  88 void FTPConn::Close()
  89 {
  90    close(ServerFd);
  91    ServerFd = -1;
  92    close(DataFd);
  93    DataFd = -1;
  94    close(DataListenFd);
  95    DataListenFd = -1;
  96
  97    if (PasvAddr != 0)
  98       freeaddrinfo(PasvAddr);
  99    PasvAddr = 0;
 100 }
 101                                                                         /*}}}*/
 102 // FTPConn::Open - Open a new connection                                /*{{{*/
 103 // ---------------------------------------------------------------------
 104 /* Connect to the server using a non-blocking connection and perform a
 105    login. */
 106 bool FTPConn::Open(pkgAcqMethod *Owner)
 107 {
 108    // Use the already open connection if possible.
 109    if (ServerFd != -1)
 110       return true;
 111
 112    Close();
 113
 114    // Determine the proxy setting
 115    if (getenv("ftp_proxy") == 0)
 116    {
 117       string DefProxy = _config->Find("Acquire::ftp::Proxy");
 118       string SpecificProxy = _config->Find("Acquire::ftp::Proxy::" + ServerName.Host);
 119       if (SpecificProxy.empty() == false)
 120       {
 121          if (SpecificProxy == "DIRECT")
 122             Proxy = "";
 123          else
 124             Proxy = SpecificProxy;
 125       }
 126       else
 127          Proxy = DefProxy;
 128    }
 129    else
 130       Proxy = getenv("ftp_proxy");
 131
 132    // Parse no_proxy, a , separated list of domains
 133    if (getenv("no_proxy") != 0)
 134    {
 135       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 136          Proxy = "";
 137    }
 138
 139    // Determine what host and port to use based on the proxy settings
 140    int Port = 0;
 141    string Host;
 142    if (Proxy.empty() == true)
 143    {
 144       if (ServerName.Port != 0)
 145          Port = ServerName.Port;
 146       Host = ServerName.Host;
 147    }
 148    else
 149    {
 150       if (Proxy.Port != 0)
 151          Port = Proxy.Port;
 152       Host = Proxy.Host;
 153    }
 154
 155    /* Connect to the remote server. Since FTP is connection oriented we
 156       want to make sure we get a new server every time we reconnect */
 157    RotateDNS();
 158    if (Connect(Host,Port,"ftp",21,ServerFd,TimeOut,Owner) == false)
 159       return false;
 160
 161    // Login must be before getpeername otherwise dante won't work.
 162    Owner->Status(_("Logging in"));
 163    bool Res = Login();
 164
 165    // Get the remote server's address
 166    PeerAddrLen = sizeof(PeerAddr);
 167    if (getpeername(ServerFd,(sockaddr *)&PeerAddr,&PeerAddrLen) != 0)
 168       return _error->Errno("getpeername",_("Unable to determine the peer name"));
 169
 170    // Get the local machine's address
 171    ServerAddrLen = sizeof(ServerAddr);
 172    if (getsockname(ServerFd,(sockaddr *)&ServerAddr,&ServerAddrLen) != 0)
 173       return _error->Errno("getsockname",_("Unable to determine the local name"));
 174
 175    return Res;
 176 }
 177                                                                         /*}}}*/
 178 // FTPConn::Login - Login to the remote server                          /*{{{*/
 179 // ---------------------------------------------------------------------
 180 /* This performs both normal login and proxy login using a simples script
 181    stored in the config file. */
 182 bool FTPConn::Login()
 183 {
 184    unsigned int Tag;
 185    string Msg;
 186
 187    // Setup the variables needed for authentication
 188    string User = "anonymous";
 189    string Pass = "apt_get_ftp_2.1@debian.linux.user";
 190
 191    // Fill in the user/pass
 192    if (ServerName.User.empty() == false)
 193       User = ServerName.User;
 194    if (ServerName.Password.empty() == false)
 195       Pass = ServerName.Password;
 196
 197    // Perform simple login
 198    if (Proxy.empty() == true)
 199    {
 200       // Read the initial response
 201       if (ReadResp(Tag,Msg) == false)
 202          return false;
 203       if (Tag >= 400)
 204          return _error->Error(_("Server refused our connection and said: %s"),Msg.c_str());
 205
 206       // Send the user
 207       if (WriteMsg(Tag,Msg,"USER %s",User.c_str()) == false)
 208          return false;
 209       if (Tag >= 400)
 210          return _error->Error(_("USER failed, server said: %s"),Msg.c_str());
 211
 212       if (Tag == 331) { // 331 User name okay, need password.
 213          // Send the Password
 214          if (WriteMsg(Tag,Msg,"PASS %s",Pass.c_str()) == false)
 215             return false;
 216          if (Tag >= 400)
 217             return _error->Error(_("PASS failed, server said: %s"),Msg.c_str());
 218       }
 219
 220       // Enter passive mode
 221       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 222          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 223       else
 224          TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 225    }
 226    else
 227    {
 228       // Read the initial response
 229       if (ReadResp(Tag,Msg) == false)
 230          return false;
 231       if (Tag >= 400)
 232          return _error->Error(_("Server refused our connection and said: %s"),Msg.c_str());
 233
 234       // Perform proxy script execution
 235       Configuration::Item const *Opts = _config->Tree("Acquire::ftp::ProxyLogin");
 236       if (Opts == 0 || Opts->Child == 0)
 237          return _error->Error(_("A proxy server was specified but no login "
 238                               "script, Acquire::ftp::ProxyLogin is empty."));
 239       Opts = Opts->Child;
 240
 241       // Iterate over the entire login script
 242       for (; Opts != 0; Opts = Opts->Next)
 243       {
 244          if (Opts->Value.empty() == true)
 245             continue;
 246
 247          // Substitute the variables into the command
 248          char SitePort[20];
 249          if (ServerName.Port != 0)
 250             sprintf(SitePort,"%u",ServerName.Port);
 251          else
 252             strcpy(SitePort,"21");
 253          string Tmp = Opts->Value;
 254          Tmp = SubstVar(Tmp,"$(PROXY_USER)",Proxy.User);
 255          Tmp = SubstVar(Tmp,"$(PROXY_PASS)",Proxy.Password);
 256          Tmp = SubstVar(Tmp,"$(SITE_USER)",User);
 257          Tmp = SubstVar(Tmp,"$(SITE_PASS)",Pass);
 258          Tmp = SubstVar(Tmp,"$(SITE_PORT)",SitePort);
 259          Tmp = SubstVar(Tmp,"$(SITE)",ServerName.Host);
 260
 261          // Send the command
 262          if (WriteMsg(Tag,Msg,"%s",Tmp.c_str()) == false)
 263             return false;
 264          if (Tag >= 400)
 265             return _error->Error(_("Login script command '%s' failed, server said: %s"),Tmp.c_str(),Msg.c_str());
 266       }
 267
 268       // Enter passive mode
 269       TryPassive = false;
 270       if (_config->Exists("Acquire::FTP::Passive::" + ServerName.Host) == true)
 271          TryPassive = _config->FindB("Acquire::FTP::Passive::" + ServerName.Host,true);
 272       else
 273       {
 274          if (_config->Exists("Acquire::FTP::Proxy::Passive") == true)
 275             TryPassive = _config->FindB("Acquire::FTP::Proxy::Passive",true);
 276          else
 277             TryPassive = _config->FindB("Acquire::FTP::Passive",true);
 278       }
 279    }
 280
 281    // Force the use of extended commands
 282    if (_config->Exists("Acquire::FTP::ForceExtended::" + ServerName.Host) == true)
 283       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended::" + ServerName.Host,true);
 284    else
 285       ForceExtended = _config->FindB("Acquire::FTP::ForceExtended",false);
 286
 287    // Binary mode
 288    if (WriteMsg(Tag,Msg,"TYPE I") == false)
 289       return false;
 290    if (Tag >= 400)
 291       return _error->Error(_("TYPE failed, server said: %s"),Msg.c_str());
 292
 293    return true;
 294 }
 295                                                                         /*}}}*/
 296 // FTPConn::ReadLine - Read a line from the server                      /*{{{*/
 297 // ---------------------------------------------------------------------
 298 /* This performs a very simple buffered read. */
 299 bool FTPConn::ReadLine(string &Text)
 300 {
 301    if (ServerFd == -1)
 302       return false;
 303
 304    // Suck in a line
 305    while (Len < sizeof(Buffer))
 306    {
 307       // Scan the buffer for a new line
 308       for (unsigned int I = 0; I != Len; I++)
 309       {
 310          // Escape some special chars
 311          if (Buffer[I] == 0)
 312             Buffer[I] = '?';
 313
 314          // End of line?
 315          if (Buffer[I] != '\n')
 316             continue;
 317
 318          I++;
 319          Text = string(Buffer,I);
 320          memmove(Buffer,Buffer+I,Len - I);
 321          Len -= I;
 322          return true;
 323       }
 324
 325       // Wait for some data..
 326       if (WaitFd(ServerFd,false,TimeOut) == false)
 327       {
 328          Close();
 329          return _error->Error(_("Connection timeout"));
 330       }
 331
 332       // Suck it back
 333       int Res = read(ServerFd,Buffer + Len,sizeof(Buffer) - Len);
 334       if (Res == 0)
 335          _error->Error(_("Server closed the connection"));
 336       if (Res <= 0)
 337       {
 338          _error->Errno("read",_("Read error"));
 339          Close();
 340          return false;
 341       }
 342       Len += Res;
 343    }
 344
 345    return _error->Error(_("A response overflowed the buffer."));
 346 }
 347                                                                         /*}}}*/
 348 // FTPConn::ReadResp - Read a full response from the server             /*{{{*/
 349 // ---------------------------------------------------------------------
 350 /* This reads a reply code from the server, it handles both p */
 351 bool FTPConn::ReadResp(unsigned int &Ret,string &Text)
 352 {
 353    // Grab the first line of the response
 354    string Msg;
 355    if (ReadLine(Msg) == false)
 356        return false;
 357
 358    // Get the ID code
 359    char *End;
 360    Ret = strtol(Msg.c_str(),&End,10);
 361    if (End - Msg.c_str() != 3)
 362       return _error->Error(_("Protocol corruption"));
 363
 364    // All done ?
 365    Text = Msg.c_str()+4;
 366    if (*End == ' ')
 367    {
 368       if (Debug == true)
 369          cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 370       return true;
 371    }
 372
 373    if (*End != '-')
 374       return _error->Error(_("Protocol corruption"));
 375
 376    /* Okay, here we do the continued message trick. This is foolish, but
 377       proftpd follows the protocol as specified and wu-ftpd doesn't, so
 378       we filter. I wonder how many clients break if you use proftpd and
 379       put a '- in the 3rd spot in the message? */
 380    char Leader[4];
 381    strncpy(Leader,Msg.c_str(),3);
 382    Leader[3] = 0;
 383    while (ReadLine(Msg) == true)
 384    {
 385       // Short, it must be using RFC continuation..
 386       if (Msg.length() < 4)
 387       {
 388          Text += Msg;
 389          continue;
 390       }
 391
 392       // Oops, finished
 393       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == ' ')
 394       {
 395          Text += Msg.c_str()+4;
 396          break;
 397       }
 398
 399       // This message has the wu-ftpd style reply code prefixed
 400       if (strncmp(Msg.c_str(),Leader,3) == 0 && Msg[3] == '-')
 401       {
 402          Text += Msg.c_str()+4;
 403          continue;
 404       }
 405
 406       // Must be RFC style prefixing
 407       Text += Msg;
 408    }
 409
 410    if (Debug == true && _error->PendingError() == false)
 411       cerr << "<- '" << QuoteString(Text,"") << "'" << endl;
 412
 413    return !_error->PendingError();
 414 }
 415                                                                         /*}}}*/
 416 // FTPConn::WriteMsg - Send a message to the server                     /*{{{*/
 417 // ---------------------------------------------------------------------
 418 /* Simple printf like function.. */
 419 bool FTPConn::WriteMsg(unsigned int &Ret,string &Text,const char *Fmt,...)
 420 {
 421    va_list args;
 422    va_start(args,Fmt);
 423
 424    // sprintf the description
 425    char S[400];
 426    vsnprintf(S,sizeof(S) - 4,Fmt,args);
 427    strcat(S,"\r\n");
 428
 429    if (Debug == true)
 430       cerr << "-> '" << QuoteString(S,"") << "'" << endl;
 431
 432    // Send it off
 433    unsigned long Len = strlen(S);
 434    unsigned long Start = 0;
 435    while (Len != 0)
 436    {
 437       if (WaitFd(ServerFd,true,TimeOut) == false)
 438       {
 439          Close();
 440          return _error->Error(_("Connection timeout"));
 441       }
 442
 443       int Res = write(ServerFd,S + Start,Len);
 444       if (Res <= 0)
 445       {
 446          _error->Errno("write",_("Write Error"));
 447          Close();
 448          return false;
 449       }
 450
 451       Len -= Res;
 452       Start += Res;
 453    }
 454
 455    return ReadResp(Ret,Text);
 456 }
 457                                                                         /*}}}*/
 458 // FTPConn::GoPasv - Enter Passive mode                                 /*{{{*/
 459 // ---------------------------------------------------------------------
 460 /* Try to enter passive mode, the return code does not indicate if passive
 461    mode could or could not be established, only if there was a fatal error.
 462    We have to enter passive mode every time we make a data connection :| */
 463 bool FTPConn::GoPasv()
 464 {
 465    /* The PASV command only works on IPv4 sockets, even though it could
 466       in theory suppory IPv6 via an all zeros reply */
 467    if (((struct sockaddr *)&PeerAddr)->sa_family != AF_INET ||
 468        ForceExtended == true)
 469       return ExtGoPasv();
 470
 471    if (PasvAddr != 0)
 472       freeaddrinfo(PasvAddr);
 473    PasvAddr = 0;
 474
 475    // Try to enable pasv mode
 476    unsigned int Tag;
 477    string Msg;
 478    if (WriteMsg(Tag,Msg,"PASV") == false)
 479       return false;
 480
 481    // Unsupported function
 482    string::size_type Pos = Msg.find('(');
 483    if (Tag >= 400 || Pos == string::npos)
 484       return true;
 485
 486    // Scan it
 487    unsigned a0,a1,a2,a3,p0,p1;
 488    if (sscanf(Msg.c_str() + Pos,"(%u,%u,%u,%u,%u,%u)",&a0,&a1,&a2,&a3,&p0,&p1) != 6)
 489       return true;
 490
 491    /* Some evil servers return 0 to mean their addr. We can actually speak
 492       to these servers natively using IPv6 */
 493    if (a0 == 0 && a1 == 0 && a2 == 0 && a3 == 0)
 494    {
 495       // Get the IP in text form
 496       char Name[NI_MAXHOST];
 497       char Service[NI_MAXSERV];
 498       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 499                   Name,sizeof(Name),Service,sizeof(Service),
 500                   NI_NUMERICHOST|NI_NUMERICSERV);
 501
 502       struct addrinfo Hints;
 503       memset(&Hints,0,sizeof(Hints));
 504       Hints.ai_socktype = SOCK_STREAM;
 505       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 506       Hints.ai_flags |= AI_NUMERICHOST;
 507
 508       // Get a new passive address.
 509       char Port[100];
 510       snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 511       if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 512          return true;
 513       return true;
 514    }
 515
 516    struct addrinfo Hints;
 517    memset(&Hints,0,sizeof(Hints));
 518    Hints.ai_socktype = SOCK_STREAM;
 519    Hints.ai_family = AF_INET;
 520    Hints.ai_flags |= AI_NUMERICHOST;
 521
 522    // Get a new passive address.
 523    char Port[100];
 524    snprintf(Port,sizeof(Port),"%u",(p0 << 8) + p1);
 525    char Name[100];
 526    snprintf(Name,sizeof(Name),"%u.%u.%u.%u",a0,a1,a2,a3);
 527    if (getaddrinfo(Name,Port,&Hints,&PasvAddr) != 0)
 528       return true;
 529    return true;
 530 }
 531                                                                         /*}}}*/
 532 // FTPConn::ExtGoPasv - Enter Extended Passive mode                     /*{{{*/
 533 // ---------------------------------------------------------------------
 534 /* Try to enter extended passive mode. See GoPasv above and RFC 2428 */
 535 bool FTPConn::ExtGoPasv()
 536 {
 537    if (PasvAddr != 0)
 538       freeaddrinfo(PasvAddr);
 539    PasvAddr = 0;
 540
 541    // Try to enable pasv mode
 542    unsigned int Tag;
 543    string Msg;
 544    if (WriteMsg(Tag,Msg,"EPSV") == false)
 545       return false;
 546
 547    // Unsupported function
 548    string::size_type Pos = Msg.find('(');
 549    if (Tag >= 400 || Pos == string::npos)
 550       return true;
 551
 552    // Scan it
 553    string::const_iterator List[4];
 554    unsigned Count = 0;
 555    Pos++;
 556    for (string::const_iterator I = Msg.begin() + Pos; I < Msg.end(); I++)
 557    {
 558       if (*I != Msg[Pos])
 559          continue;
 560       if (Count >= 4)
 561          return true;
 562       List[Count++] = I;
 563    }
 564    if (Count != 4)
 565       return true;
 566
 567    // Break it up ..
 568    unsigned long Proto = 0;
 569    unsigned long Port = 0;
 570    string IP;
 571    IP = string(List[1]+1,List[2]);
 572    Port = atoi(string(List[2]+1,List[3]).c_str());
 573    if (IP.empty() == false)
 574       Proto = atoi(string(List[0]+1,List[1]).c_str());
 575
 576    if (Port == 0)
 577       return false;
 578
 579    // String version of the port
 580    char PStr[100];
 581    snprintf(PStr,sizeof(PStr),"%lu",Port);
 582
 583    // Get the IP in text form
 584    struct addrinfo Hints;
 585    memset(&Hints,0,sizeof(Hints));
 586    Hints.ai_socktype = SOCK_STREAM;
 587    Hints.ai_flags |= AI_NUMERICHOST;
 588
 589    /* The RFC defined case, connect to the old IP/protocol using the
 590       new port. */
 591    if (IP.empty() == true)
 592    {
 593       // Get the IP in text form
 594       char Name[NI_MAXHOST];
 595       char Service[NI_MAXSERV];
 596       getnameinfo((struct sockaddr *)&PeerAddr,PeerAddrLen,
 597                   Name,sizeof(Name),Service,sizeof(Service),
 598                   NI_NUMERICHOST|NI_NUMERICSERV);
 599       IP = Name;
 600       Hints.ai_family = ((struct sockaddr *)&PeerAddr)->sa_family;
 601    }
 602    else
 603    {
 604       // Get the family..
 605       Hints.ai_family = 0;
 606       for (unsigned J = 0; AFMap[J].Family != 0; J++)
 607          if (AFMap[J].IETFFamily == Proto)
 608             Hints.ai_family = AFMap[J].Family;
 609       if (Hints.ai_family == 0)
 610          return true;
 611    }
 612
 613    // Get a new passive address.
 614    int Res;
 615    if ((Res = getaddrinfo(IP.c_str(),PStr,&Hints,&PasvAddr)) != 0)
 616       return true;
 617
 618    return true;
 619 }
 620                                                                         /*}}}*/
 621 // FTPConn::Size - Return the size of a file                            /*{{{*/
 622 // ---------------------------------------------------------------------
 623 /* Grab the file size from the server, 0 means no size or empty file */
 624 bool FTPConn::Size(const char *Path,unsigned long &Size)
 625 {
 626    // Query the size
 627    unsigned int Tag;
 628    string Msg;
 629    Size = 0;
 630    if (WriteMsg(Tag,Msg,"SIZE %s",Path) == false)
 631       return false;
 632
 633    char *End;
 634    Size = strtol(Msg.c_str(),&End,10);
 635    if (Tag >= 400 || End == Msg.c_str())
 636       Size = 0;
 637    return true;
 638 }
 639                                                                         /*}}}*/
 640 // FTPConn::ModTime - Return the modification time of the file          /*{{{*/
 641 // ---------------------------------------------------------------------
 642 /* Like Size no error is returned if the command is not supported. If the
 643    command fails then time is set to the current time of day to fool
 644    date checks. */
 645 bool FTPConn::ModTime(const char *Path, time_t &Time)
 646 {
 647    Time = time(&Time);
 648
 649    // Query the mod time
 650    unsigned int Tag;
 651    string Msg;
 652    if (WriteMsg(Tag,Msg,"MDTM %s",Path) == false)
 653       return false;
 654    if (Tag >= 400 || Msg.empty() == true || isdigit(Msg[0]) == 0)
 655       return true;
 656
 657    // Parse it
 658    StrToTime(Msg,Time);
 659    return true;
 660 }
 661                                                                         /*}}}*/
 662 // FTPConn::CreateDataFd - Get a data connection                        /*{{{*/
 663 // ---------------------------------------------------------------------
 664 /* Create the data connection. Call FinalizeDataFd after this though.. */
 665 bool FTPConn::CreateDataFd()
 666 {
 667    close(DataFd);
 668    DataFd = -1;
 669
 670    // Attempt to enter passive mode.
 671    if (TryPassive == true)
 672    {
 673       if (GoPasv() == false)
 674          return false;
 675
 676       // Oops, didn't work out, don't bother trying again.
 677       if (PasvAddr == 0)
 678          TryPassive = false;
 679    }
 680
 681    // Passive mode?
 682    if (PasvAddr != 0)
 683    {
 684       // Get a socket
 685       if ((DataFd = socket(PasvAddr->ai_family,PasvAddr->ai_socktype,
 686                            PasvAddr->ai_protocol)) < 0)
 687          return _error->Errno("socket",_("Could not create a socket"));
 688
 689       // Connect to the server
 690       SetNonBlock(DataFd,true);
 691       if (connect(DataFd,PasvAddr->ai_addr,PasvAddr->ai_addrlen) < 0 &&
 692           errno != EINPROGRESS)
 693          return _error->Errno("socket",_("Could not create a socket"));
 694
 695       /* This implements a timeout for connect by opening the connection
 696          nonblocking */
 697       if (WaitFd(DataFd,true,TimeOut) == false)
 698          return _error->Error(_("Could not connect data socket, connection timed out"));
 699       unsigned int Err;
 700       unsigned int Len = sizeof(Err);
 701       if (getsockopt(DataFd,SOL_SOCKET,SO_ERROR,&Err,&Len) != 0)
 702          return _error->Errno("getsockopt",_("Failed"));
 703       if (Err != 0)
 704          return _error->Error(_("Could not connect passive socket."));
 705
 706       return true;
 707    }
 708
 709    // Port mode :<
 710    close(DataListenFd);
 711    DataListenFd = -1;
 712
 713    // Get the information for a listening socket.
 714    struct addrinfo *BindAddr = 0;
 715    struct addrinfo Hints;
 716    memset(&Hints,0,sizeof(Hints));
 717    Hints.ai_socktype = SOCK_STREAM;
 718    Hints.ai_flags |= AI_PASSIVE;
 719    Hints.ai_family = ((struct sockaddr *)&ServerAddr)->sa_family;
 720    int Res;
 721    if ((Res = getaddrinfo(0,"0",&Hints,&BindAddr)) != 0)
 722       return _error->Error(_("getaddrinfo was unable to get a listening socket"));
 723
 724    // Construct the socket
 725    if ((DataListenFd = socket(BindAddr->ai_family,BindAddr->ai_socktype,
 726                               BindAddr->ai_protocol)) < 0)
 727    {
 728       freeaddrinfo(BindAddr);
 729       return _error->Errno("socket",_("Could not create a socket"));
 730    }
 731
 732    // Bind and listen
 733    if (bind(DataListenFd,BindAddr->ai_addr,BindAddr->ai_addrlen) < 0)
 734    {
 735       freeaddrinfo(BindAddr);
 736       return _error->Errno("bind",_("Could not bind a socket"));
 737    }
 738    freeaddrinfo(BindAddr);
 739    if (listen(DataListenFd,1) < 0)
 740       return _error->Errno("listen",_("Could not listen on the socket"));
 741    SetNonBlock(DataListenFd,true);
 742
 743    // Determine the name to send to the remote
 744    struct sockaddr_storage Addr;
 745    socklen_t AddrLen = sizeof(Addr);
 746    if (getsockname(DataListenFd,(sockaddr *)&Addr,&AddrLen) < 0)
 747       return _error->Errno("getsockname",_("Could not determine the socket's name"));
 748
 749
 750    // Reverse the address. We need the server address and the data port.
 751    char Name[NI_MAXHOST];
 752    char Service[NI_MAXSERV];
 753    char Service2[NI_MAXSERV];
 754    getnameinfo((struct sockaddr *)&Addr,AddrLen,
 755                Name,sizeof(Name),Service,sizeof(Service),
 756                NI_NUMERICHOST|NI_NUMERICSERV);
 757    getnameinfo((struct sockaddr *)&ServerAddr,ServerAddrLen,
 758                Name,sizeof(Name),Service2,sizeof(Service2),
 759                NI_NUMERICHOST|NI_NUMERICSERV);
 760
 761    // Send off an IPv4 address in the old port format
 762    if (((struct sockaddr *)&Addr)->sa_family == AF_INET &&
 763        ForceExtended == false)
 764    {
 765       // Convert the dots in the quad into commas
 766       for (char *I = Name; *I != 0; I++)
 767          if (*I == '.')
 768             *I = ',';
 769       unsigned long Port = atoi(Service);
 770
 771       // Send the port command
 772       unsigned int Tag;
 773       string Msg;
 774       if (WriteMsg(Tag,Msg,"PORT %s,%d,%d",
 775                    Name,
 776                    (int)(Port >> 8) & 0xff, (int)(Port & 0xff)) == false)
 777          return false;
 778       if (Tag >= 400)
 779          return _error->Error(_("Unable to send PORT command"));
 780       return true;
 781    }
 782
 783    // Construct an EPRT command
 784    unsigned Proto = 0;
 785    for (unsigned J = 0; AFMap[J].Family != 0; J++)
 786       if (AFMap[J].Family == ((struct sockaddr *)&Addr)->sa_family)
 787          Proto = AFMap[J].IETFFamily;
 788    if (Proto == 0)
 789       return _error->Error(_("Unknown address family %u (AF_*)"),
 790                            ((struct sockaddr *)&Addr)->sa_family);
 791
 792    // Send the EPRT command
 793    unsigned int Tag;
 794    string Msg;
 795    if (WriteMsg(Tag,Msg,"EPRT |%u|%s|%s|",Proto,Name,Service) == false)
 796       return false;
 797    if (Tag >= 400)
 798       return _error->Error(_("EPRT failed, server said: %s"),Msg.c_str());
 799    return true;
 800 }
 801                                                                         /*}}}*/
 802 // FTPConn::Finalize - Complete the Data connection                     /*{{{*/
 803 // ---------------------------------------------------------------------
 804 /* If the connection is in port mode this waits for the other end to hook
 805    up to us. */
 806 bool FTPConn::Finalize()
 807 {
 808    // Passive mode? Do nothing
 809    if (PasvAddr != 0)
 810       return true;
 811
 812    // Close any old socket..
 813    close(DataFd);
 814    DataFd = -1;
 815
 816    // Wait for someone to connect..
 817    if (WaitFd(DataListenFd,false,TimeOut) == false)
 818       return _error->Error(_("Data socket connect timed out"));
 819
 820    // Accept the connection
 821    struct sockaddr_in Addr;
 822    socklen_t Len = sizeof(Addr);
 823    DataFd = accept(DataListenFd,(struct sockaddr *)&Addr,&Len);
 824    if (DataFd < 0)
 825       return _error->Errno("accept",_("Unable to accept connection"));
 826
 827    close(DataListenFd);
 828    DataListenFd = -1;
 829
 830    return true;
 831 }
 832                                                                         /*}}}*/
 833 // FTPConn::Get - Get a file                                            /*{{{*/
 834 // ---------------------------------------------------------------------
 835 /* This opens a data connection, sends REST and RETR and then
 836    transfers the file over. */
 837 bool FTPConn::Get(const char *Path,FileFd &To,unsigned long Resume,
 838                   Hashes &Hash,bool &Missing)
 839 {
 840    Missing = false;
 841    if (CreateDataFd() == false)
 842       return false;
 843
 844    unsigned int Tag;
 845    string Msg;
 846    if (Resume != 0)
 847    {
 848       if (WriteMsg(Tag,Msg,"REST %u",Resume) == false)
 849          return false;
 850       if (Tag >= 400)
 851          Resume = 0;
 852    }
 853
 854    if (To.Truncate(Resume) == false)
 855       return false;
 856
 857    if (To.Seek(0) == false)
 858       return false;
 859
 860    if (Resume != 0)
 861    {
 862       if (Hash.AddFD(To.Fd(),Resume) == false)
 863       {
 864          _error->Errno("read",_("Problem hashing file"));
 865          return false;
 866       }
 867    }
 868
 869    // Send the get command
 870    if (WriteMsg(Tag,Msg,"RETR %s",Path) == false)
 871       return false;
 872
 873    if (Tag >= 400)
 874    {
 875       if (Tag == 550)
 876          Missing = true;
 877       return _error->Error(_("Unable to fetch file, server said '%s'"),Msg.c_str());
 878    }
 879
 880    // Finish off the data connection
 881    if (Finalize() == false)
 882       return false;
 883
 884    // Copy loop
 885    unsigned char Buffer[4096];
 886    while (1)
 887    {
 888       // Wait for some data..
 889       if (WaitFd(DataFd,false,TimeOut) == false)
 890       {
 891          Close();
 892          return _error->Error(_("Data socket timed out"));
 893       }
 894
 895       // Read the data..
 896       int Res = read(DataFd,Buffer,sizeof(Buffer));
 897       if (Res == 0)
 898          break;
 899       if (Res < 0)
 900       {
 901          if (errno == EAGAIN)
 902             continue;
 903          break;
 904       }
 905
 906       Hash.Add(Buffer,Res);
 907       if (To.Write(Buffer,Res) == false)
 908       {
 909          Close();
 910          return false;
 911       }
 912    }
 913
 914    // All done
 915    close(DataFd);
 916    DataFd = -1;
 917
 918    // Read the closing message from the server
 919    if (ReadResp(Tag,Msg) == false)
 920       return false;
 921    if (Tag >= 400)
 922       return _error->Error(_("Data transfer failed, server said '%s'"),Msg.c_str());
 923    return true;
 924 }
 925                                                                         /*}}}*/
 926
 927 // FtpMethod::FtpMethod - Constructor                                   /*{{{*/
 928 // ---------------------------------------------------------------------
 929 /* */
 930 FtpMethod::FtpMethod() : pkgAcqMethod("1.0",SendConfig)
 931 {
 932    signal(SIGTERM,SigTerm);
 933    signal(SIGINT,SigTerm);
 934
 935    Server = 0;
 936    FailFd = -1;
 937 }
 938                                                                         /*}}}*/
 939 // FtpMethod::SigTerm - Handle a fatal signal                           /*{{{*/
 940 // ---------------------------------------------------------------------
 941 /* This closes and timestamps the open file. This is neccessary to get
 942    resume behavoir on user abort */
 943 void FtpMethod::SigTerm(int)
 944 {
 945    if (FailFd == -1)
 946       _exit(100);
 947    close(FailFd);
 948
 949    // Timestamp
 950    struct utimbuf UBuf;
 951    UBuf.actime = FailTime;
 952    UBuf.modtime = FailTime;
 953    utime(FailFile.c_str(),&UBuf);
 954
 955    _exit(100);
 956 }
 957                                                                         /*}}}*/
 958 // FtpMethod::Configuration - Handle a configuration message            /*{{{*/
 959 // ---------------------------------------------------------------------
 960 /* We stash the desired pipeline depth */
 961 bool FtpMethod::Configuration(string Message)
 962 {
 963    if (pkgAcqMethod::Configuration(Message) == false)
 964       return false;
 965
 966    TimeOut = _config->FindI("Acquire::Ftp::Timeout",TimeOut);
 967    return true;
 968 }
 969                                                                         /*}}}*/
 970 // FtpMethod::Fetch - Fetch a file                                      /*{{{*/
 971 // ---------------------------------------------------------------------
 972 /* Fetch a single file, called by the base class..  */
 973 bool FtpMethod::Fetch(FetchItem *Itm)
 974 {
 975    URI Get = Itm->Uri;
 976    const char *File = Get.Path.c_str();
 977    FetchResult Res;
 978    Res.Filename = Itm->DestFile;
 979    Res.IMSHit = false;
 980
 981    // Connect to the server
 982    if (Server == 0 || Server->Comp(Get) == false)
 983    {
 984       delete Server;
 985       Server = new FTPConn(Get);
 986    }
 987
 988    // Could not connect is a transient error..
 989    if (Server->Open(this) == false)
 990    {
 991       Server->Close();
 992       Fail(true);
 993       return true;
 994    }
 995
 996    // Get the files information
 997    Status(_("Query"));
 998    unsigned long Size;
 999    if (Server->Size(File,Size) == false ||
1000        Server->ModTime(File,FailTime) == false)
1001    {
1002       Fail(true);
1003       return true;
1004    }
1005    Res.Size = Size;
1006
1007    // See if it is an IMS hit
1008    if (Itm->LastModified == FailTime)
1009    {
1010       Res.Size = 0;
1011       Res.IMSHit = true;
1012       URIDone(Res);
1013       return true;
1014    }
1015
1016    // See if the file exists
1017    struct stat Buf;
1018    if (stat(Itm->DestFile.c_str(),&Buf) == 0)
1019    {
1020       if (Size == (unsigned)Buf.st_size && FailTime == Buf.st_mtime)
1021       {
1022          Res.Size = Buf.st_size;
1023          Res.LastModified = Buf.st_mtime;
1024          Res.ResumePoint = Buf.st_size;
1025          URIDone(Res);
1026          return true;
1027       }
1028
1029       // Resume?
1030       if (FailTime == Buf.st_mtime && Size > (unsigned)Buf.st_size)
1031          Res.ResumePoint = Buf.st_size;
1032    }
1033
1034    // Open the file
1035    Hashes Hash;
1036    {
1037       FileFd Fd(Itm->DestFile,FileFd::WriteAny);
1038       if (_error->PendingError() == true)
1039          return false;
1040
1041       URIStart(Res);
1042
1043       FailFile = Itm->DestFile;
1044       FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1045       FailFd = Fd.Fd();
1046
1047       bool Missing;
1048       if (Server->Get(File,Fd,Res.ResumePoint,Hash,Missing) == false)
1049       {
1050          Fd.Close();
1051
1052          // Timestamp
1053          struct utimbuf UBuf;
1054          UBuf.actime = FailTime;
1055          UBuf.modtime = FailTime;
1056          utime(FailFile.c_str(),&UBuf);
1057
1058          // If the file is missing we hard fail otherwise transient fail
1059          if (Missing == true)
1060             return false;
1061          Fail(true);
1062          return true;
1063       }
1064
1065       Res.Size = Fd.Size();
1066    }
1067
1068    Res.LastModified = FailTime;
1069    Res.TakeHashes(Hash);
1070
1071    // Timestamp
1072    struct utimbuf UBuf;
1073    UBuf.actime = FailTime;
1074    UBuf.modtime = FailTime;
1075    utime(Queue->DestFile.c_str(),&UBuf);
1076    FailFd = -1;
1077
1078    URIDone(Res);
1079
1080    return true;
1081 }
1082                                                                         /*}}}*/
1083
1084 int main(int argc,const char *argv[])
1085 {
1086    setlocale(LC_ALL, "");
1087
1088    /* See if we should be come the http client - we do this for http
1089       proxy urls */
1090    if (getenv("ftp_proxy") != 0)
1091    {
1092       URI Proxy = string(getenv("ftp_proxy"));
1093
1094       // Run the HTTP method
1095       if (Proxy.Access == "http")
1096       {
1097          // Copy over the environment setting
1098          char S[300];
1099          snprintf(S,sizeof(S),"http_proxy=%s",getenv("ftp_proxy"));
1100          putenv(S);
1101          putenv("no_proxy=");
1102
1103          // Run the http method
1104          string Path = flNotFile(argv[0]) + "http";
1105          execl(Path.c_str(),Path.c_str(),0);
1106          cerr << _("Unable to invoke ") << Path << endl;
1107          exit(100);
1108       }
1109    }
1110
1111    FtpMethod Mth;
1112
1113    return Mth.Run();
1114 }