methods/http.cc

   1 extern "C" {
   2     #include <mach-o/nlist.h>
   3 }
   4
   5 // -*- mode: cpp; mode: fold -*-
   6 // Description                                                          /*{{{*/
   7 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   8 /* ######################################################################
   9
  10    HTTP Aquire Method - This is the HTTP aquire method for APT.
  11
  12    It uses HTTP/1.1 and many of the fancy options there-in, such as
  13    pipelining, range, if-range and so on.
  14
  15    It is based on a doubly buffered select loop. A groupe of requests are
  16    fed into a single output buffer that is constantly fed out the
  17    socket. This provides ideal pipelining as in many cases all of the
  18    requests will fit into a single packet. The input socket is buffered
  19    the same way and fed into the fd for the file (may be a pipe in future).
  20
  21    This double buffering provides fairly substantial transfer rates,
  22    compared to wget the http method is about 4% faster. Most importantly,
  23    when HTTP is compared with FTP as a protocol the speed difference is
  24    huge. In tests over the internet from two sites to llug (via ATM) this
  25    program got 230k/s sustained http transfer rates. FTP on the other
  26    hand topped out at 170k/s. That combined with the time to setup the
  27    FTP connection makes HTTP a vastly superior protocol.
  28
  29    ##################################################################### */
  30                                                                         /*}}}*/
  31 // Include Files                                                        /*{{{*/
  32 #include <apt-pkg/fileutl.h>
  33 #include <apt-pkg/acquire-method.h>
  34 #include <apt-pkg/error.h>
  35 #include <apt-pkg/hashes.h>
  36
  37 #include <sys/stat.h>
  38 #include <sys/time.h>
  39 #include <utime.h>
  40 #include <unistd.h>
  41 #include <signal.h>
  42 #include <stdio.h>
  43 #include <errno.h>
  44 #include <string.h>
  45 #include <iostream>
  46 #include <apti18n.h>
  47
  48 // Internet stuff
  49 #include <netdb.h>
  50
  51 #include <CoreFoundation/CoreFoundation.h>
  52 #include <CoreServices/CoreServices.h>
  53
  54 #include "connect.h"
  55 #include "rfc2553emu.h"
  56 #include "http.h"
  57
  58                                                                         /*}}}*/
  59 using namespace std;
  60
  61 string HttpMethod::FailFile;
  62 int HttpMethod::FailFd = -1;
  63 time_t HttpMethod::FailTime = 0;
  64 unsigned long PipelineDepth = 10;
  65 unsigned long TimeOut = 120;
  66 bool Debug = false;
  67
  68 unsigned long CircleBuf::BwReadLimit=0;
  69 unsigned long CircleBuf::BwTickReadData=0;
  70 struct timeval CircleBuf::BwReadTick={0,0};
  71 const unsigned int CircleBuf::BW_HZ=10;
  72
  73 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  74 // ---------------------------------------------------------------------
  75 /* */
  76 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
  77 {
  78    Buf = new unsigned char[Size];
  79    Reset();
  80
  81    CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
  82 }
  83                                                                         /*}}}*/
  84 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  85 // ---------------------------------------------------------------------
  86 /* */
  87 void CircleBuf::Reset()
  88 {
  89    InP = 0;
  90    OutP = 0;
  91    StrPos = 0;
  92    MaxGet = (unsigned int)-1;
  93    OutQueue = string();
  94    if (Hash != 0)
  95    {
  96       delete Hash;
  97       Hash = new Hashes;
  98    }
  99 };
 100                                                                         /*}}}*/
 101 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
 102 // ---------------------------------------------------------------------
 103 /* This fills up the buffer with as much data as is in the FD, assuming it
 104    is non-blocking.. */
 105 bool CircleBuf::Read(int Fd)
 106 {
 107    unsigned long BwReadMax;
 108
 109    while (1)
 110    {
 111       // Woops, buffer is full
 112       if (InP - OutP == Size)
 113          return true;
 114
 115       // what's left to read in this tick
 116       BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 117
 118       if(CircleBuf::BwReadLimit) {
 119          struct timeval now;
 120          gettimeofday(&now,0);
 121
 122          unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 123             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 124          if(d > 1000000/BW_HZ) {
 125             CircleBuf::BwReadTick = now;
 126             CircleBuf::BwTickReadData = 0;
 127          }
 128
 129          if(CircleBuf::BwTickReadData >= BwReadMax) {
 130             usleep(1000000/BW_HZ);
 131             return true;
 132          }
 133       }
 134
 135       // Write the buffer segment
 136       int Res;
 137       if(CircleBuf::BwReadLimit) {
 138          Res = read(Fd,Buf + (InP%Size),
 139                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 140       } else
 141          Res = read(Fd,Buf + (InP%Size),LeftRead());
 142
 143       if(Res > 0 && BwReadLimit > 0)
 144          CircleBuf::BwTickReadData += Res;
 145
 146       if (Res == 0)
 147          return false;
 148       if (Res < 0)
 149       {
 150          if (errno == EAGAIN)
 151             return true;
 152          return false;
 153       }
 154
 155       if (InP == 0)
 156          gettimeofday(&Start,0);
 157       InP += Res;
 158    }
 159 }
 160                                                                         /*}}}*/
 161 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 162 // ---------------------------------------------------------------------
 163 /* This will hold the string in and fill the buffer with it as it empties */
 164 bool CircleBuf::Read(string Data)
 165 {
 166    OutQueue += Data;
 167    FillOut();
 168    return true;
 169 }
 170                                                                         /*}}}*/
 171 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 172 // ---------------------------------------------------------------------
 173 /* */
 174 void CircleBuf::FillOut()
 175 {
 176    if (OutQueue.empty() == true)
 177       return;
 178    while (1)
 179    {
 180       // Woops, buffer is full
 181       if (InP - OutP == Size)
 182          return;
 183
 184       // Write the buffer segment
 185       unsigned long Sz = LeftRead();
 186       if (OutQueue.length() - StrPos < Sz)
 187          Sz = OutQueue.length() - StrPos;
 188       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 189
 190       // Advance
 191       StrPos += Sz;
 192       InP += Sz;
 193       if (OutQueue.length() == StrPos)
 194       {
 195          StrPos = 0;
 196          OutQueue = "";
 197          return;
 198       }
 199    }
 200 }
 201                                                                         /*}}}*/
 202 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 203 // ---------------------------------------------------------------------
 204 /* This empties the buffer into the FD. */
 205 bool CircleBuf::Write(int Fd)
 206 {
 207    while (1)
 208    {
 209       FillOut();
 210
 211       // Woops, buffer is empty
 212       if (OutP == InP)
 213          return true;
 214
 215       if (OutP == MaxGet)
 216          return true;
 217
 218       // Write the buffer segment
 219       int Res;
 220       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 221
 222       if (Res == 0)
 223          return false;
 224       if (Res < 0)
 225       {
 226          if (errno == EAGAIN)
 227             return true;
 228
 229          return false;
 230       }
 231
 232       if (Hash != 0)
 233          Hash->Add(Buf + (OutP%Size),Res);
 234
 235       OutP += Res;
 236    }
 237 }
 238                                                                         /*}}}*/
 239 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 240 // ---------------------------------------------------------------------
 241 /* This copies till the first empty line */
 242 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 243 {
 244    // We cheat and assume it is unneeded to have more than one buffer load
 245    for (unsigned long I = OutP; I < InP; I++)
 246    {
 247       if (Buf[I%Size] != '\n')
 248          continue;
 249       ++I;
 250
 251       if (Single == false)
 252       {
 253          if (I < InP  && Buf[I%Size] == '\r')
 254             ++I;
 255          if (I >= InP || Buf[I%Size] != '\n')
 256             continue;
 257          ++I;
 258       }
 259
 260       Data = "";
 261       while (OutP < I)
 262       {
 263          unsigned long Sz = LeftWrite();
 264          if (Sz == 0)
 265             return false;
 266          if (I - OutP < Sz)
 267             Sz = I - OutP;
 268          Data += string((char *)(Buf + (OutP%Size)),Sz);
 269          OutP += Sz;
 270       }
 271       return true;
 272    }
 273    return false;
 274 }
 275                                                                         /*}}}*/
 276 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 277 // ---------------------------------------------------------------------
 278 /* */
 279 void CircleBuf::Stats()
 280 {
 281    if (InP == 0)
 282       return;
 283
 284    struct timeval Stop;
 285    gettimeofday(&Stop,0);
 286 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 287              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 288    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 289 }
 290                                                                         /*}}}*/
 291
 292 // ServerState::ServerState - Constructor                               /*{{{*/
 293 // ---------------------------------------------------------------------
 294 /* */
 295 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 296                         In(64*1024), Out(4*1024),
 297                         ServerName(Srv)
 298 {
 299    Reset();
 300 }
 301                                                                         /*}}}*/
 302 // ServerState::Open - Open a connection to the server                  /*{{{*/
 303 // ---------------------------------------------------------------------
 304 /* This opens a connection to the server. */
 305 bool ServerState::Open()
 306 {
 307    // Use the already open connection if possible.
 308    if (ServerFd != -1)
 309       return true;
 310
 311    Close();
 312    In.Reset();
 313    Out.Reset();
 314    Persistent = true;
 315
 316    // Determine the proxy setting
 317    if (getenv("http_proxy") == 0)
 318    {
 319       string DefProxy = _config->Find("Acquire::http::Proxy");
 320       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 321       if (SpecificProxy.empty() == false)
 322       {
 323          if (SpecificProxy == "DIRECT")
 324             Proxy = "";
 325          else
 326             Proxy = SpecificProxy;
 327       }
 328       else
 329          Proxy = DefProxy;
 330    }
 331    else
 332       Proxy = getenv("http_proxy");
 333
 334    // Parse no_proxy, a , separated list of domains
 335    if (getenv("no_proxy") != 0)
 336    {
 337       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 338          Proxy = "";
 339    }
 340
 341    // Determine what host and port to use based on the proxy settings
 342    int Port = 0;
 343    string Host;
 344    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 345    {
 346       if (ServerName.Port != 0)
 347          Port = ServerName.Port;
 348       Host = ServerName.Host;
 349    }
 350    else
 351    {
 352       if (Proxy.Port != 0)
 353          Port = Proxy.Port;
 354       Host = Proxy.Host;
 355    }
 356
 357    // Connect to the remote server
 358    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 359       return false;
 360
 361    return true;
 362 }
 363                                                                         /*}}}*/
 364 // ServerState::Close - Close a connection to the server                /*{{{*/
 365 // ---------------------------------------------------------------------
 366 /* */
 367 bool ServerState::Close()
 368 {
 369    close(ServerFd);
 370    ServerFd = -1;
 371    return true;
 372 }
 373                                                                         /*}}}*/
 374 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 375 // ---------------------------------------------------------------------
 376 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 377    parse error occured */
 378 int ServerState::RunHeaders()
 379 {
 380    State = Header;
 381
 382    Owner->Status(_("Waiting for headers"));
 383
 384    Major = 0;
 385    Minor = 0;
 386    Result = 0;
 387    Size = 0;
 388    StartPos = 0;
 389    Encoding = Closes;
 390    HaveContent = false;
 391    time(&Date);
 392
 393    do
 394    {
 395       string Data;
 396       if (In.WriteTillEl(Data) == false)
 397          continue;
 398
 399       if (Debug == true)
 400          clog << Data;
 401
 402       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 403       {
 404          string::const_iterator J = I;
 405          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 406          if (HeaderLine(string(I,J)) == false)
 407             return 2;
 408          I = J;
 409       }
 410
 411       // 100 Continue is a Nop...
 412       if (Result == 100)
 413          continue;
 414
 415       // Tidy up the connection persistance state.
 416       if (Encoding == Closes && HaveContent == true)
 417          Persistent = false;
 418
 419       return 0;
 420    }
 421    while (Owner->Go(false,this) == true);
 422
 423    return 1;
 424 }
 425                                                                         /*}}}*/
 426 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 427 // ---------------------------------------------------------------------
 428 /* */
 429 bool ServerState::RunData()
 430 {
 431    State = Data;
 432
 433    // Chunked transfer encoding is fun..
 434    if (Encoding == Chunked)
 435    {
 436       while (1)
 437       {
 438          // Grab the block size
 439          bool Last = true;
 440          string Data;
 441          In.Limit(-1);
 442          do
 443          {
 444             if (In.WriteTillEl(Data,true) == true)
 445                break;
 446          }
 447          while ((Last = Owner->Go(false,this)) == true);
 448
 449          if (Last == false)
 450             return false;
 451
 452          // See if we are done
 453          unsigned long Len = strtol(Data.c_str(),0,16);
 454          if (Len == 0)
 455          {
 456             In.Limit(-1);
 457
 458             // We have to remove the entity trailer
 459             Last = true;
 460             do
 461             {
 462                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 463                   break;
 464             }
 465             while ((Last = Owner->Go(false,this)) == true);
 466             if (Last == false)
 467                return false;
 468             return !_error->PendingError();
 469          }
 470
 471          // Transfer the block
 472          In.Limit(Len);
 473          while (Owner->Go(true,this) == true)
 474             if (In.IsLimit() == true)
 475                break;
 476
 477          // Error
 478          if (In.IsLimit() == false)
 479             return false;
 480
 481          // The server sends an extra new line before the next block specifier..
 482          In.Limit(-1);
 483          Last = true;
 484          do
 485          {
 486             if (In.WriteTillEl(Data,true) == true)
 487                break;
 488          }
 489          while ((Last = Owner->Go(false,this)) == true);
 490          if (Last == false)
 491             return false;
 492       }
 493    }
 494    else
 495    {
 496       /* Closes encoding is used when the server did not specify a size, the
 497          loss of the connection means we are done */
 498       if (Encoding == Closes)
 499          In.Limit(-1);
 500       else
 501          In.Limit(Size - StartPos);
 502
 503       // Just transfer the whole block.
 504       do
 505       {
 506          if (In.IsLimit() == false)
 507             continue;
 508
 509          In.Limit(-1);
 510          return !_error->PendingError();
 511       }
 512       while (Owner->Go(true,this) == true);
 513    }
 514
 515    return Owner->Flush(this) && !_error->PendingError();
 516 }
 517                                                                         /*}}}*/
 518 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 519 // ---------------------------------------------------------------------
 520 /* */
 521 bool ServerState::HeaderLine(string Line)
 522 {
 523    if (Line.empty() == true)
 524       return true;
 525
 526    // The http server might be trying to do something evil.
 527    if (Line.length() >= MAXLEN)
 528       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 529
 530    string::size_type Pos = Line.find(' ');
 531    if (Pos == string::npos || Pos+1 > Line.length())
 532    {
 533       // Blah, some servers use "connection:closes", evil.
 534       Pos = Line.find(':');
 535       if (Pos == string::npos || Pos + 2 > Line.length())
 536          return _error->Error(_("Bad header line"));
 537       Pos++;
 538    }
 539
 540    // Parse off any trailing spaces between the : and the next word.
 541    string::size_type Pos2 = Pos;
 542    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 543       Pos2++;
 544
 545    string Tag = string(Line,0,Pos);
 546    string Val = string(Line,Pos2);
 547
 548    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 549    {
 550       // Evil servers return no version
 551       if (Line[4] == '/')
 552       {
 553          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 554                     &Result,Code) != 4)
 555             return _error->Error(_("The HTTP server sent an invalid reply header"));
 556       }
 557       else
 558       {
 559          Major = 0;
 560          Minor = 9;
 561          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 562             return _error->Error(_("The HTTP server sent an invalid reply header"));
 563       }
 564
 565       /* Check the HTTP response header to get the default persistance
 566          state. */
 567       if (Major < 1)
 568          Persistent = false;
 569       else
 570       {
 571          if (Major == 1 && Minor <= 0)
 572             Persistent = false;
 573          else
 574             Persistent = true;
 575       }
 576
 577       return true;
 578    }
 579
 580    if (stringcasecmp(Tag,"Content-Length:") == 0)
 581    {
 582       if (Encoding == Closes)
 583          Encoding = Stream;
 584       HaveContent = true;
 585
 586       // The length is already set from the Content-Range header
 587       if (StartPos != 0)
 588          return true;
 589
 590       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 591          return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
 592       return true;
 593    }
 594
 595    if (stringcasecmp(Tag,"Content-Type:") == 0)
 596    {
 597       HaveContent = true;
 598       return true;
 599    }
 600
 601    if (stringcasecmp(Tag,"Content-Range:") == 0)
 602    {
 603       HaveContent = true;
 604
 605       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 606          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 607       if ((unsigned)StartPos > Size)
 608          return _error->Error(_("This HTTP server has broken range support"));
 609       return true;
 610    }
 611
 612    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 613    {
 614       HaveContent = true;
 615       if (stringcasecmp(Val,"chunked") == 0)
 616          Encoding = Chunked;
 617       return true;
 618    }
 619
 620    if (stringcasecmp(Tag,"Connection:") == 0)
 621    {
 622       if (stringcasecmp(Val,"close") == 0)
 623          Persistent = false;
 624       if (stringcasecmp(Val,"keep-alive") == 0)
 625          Persistent = true;
 626       return true;
 627    }
 628
 629    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 630    {
 631       if (StrToTime(Val,Date) == false)
 632          return _error->Error(_("Unknown date format"));
 633       return true;
 634    }
 635
 636    return true;
 637 }
 638                                                                         /*}}}*/
 639
 640 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 641 // ---------------------------------------------------------------------
 642 /* This places the http request in the outbound buffer */
 643 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 644 {
 645    URI Uri = Itm->Uri;
 646
 647    // The HTTP server expects a hostname with a trailing :port
 648    char Buf[1000];
 649    string ProperHost = Uri.Host;
 650    if (Uri.Port != 0)
 651    {
 652       sprintf(Buf,":%u",Uri.Port);
 653       ProperHost += Buf;
 654    }
 655
 656    // Just in case.
 657    if (Itm->Uri.length() >= sizeof(Buf))
 658        abort();
 659
 660    /* Build the request. We include a keep-alive header only for non-proxy
 661       requests. This is to tweak old http/1.0 servers that do support keep-alive
 662       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 663       will glitch HTTP/1.0 proxies because they do not filter it out and
 664       pass it on, HTTP/1.1 says the connection should default to keep alive
 665       and we expect the proxy to do this */
 666    if (Proxy.empty() == true || Proxy.Host.empty())
 667       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 668               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 669    else
 670    {
 671       /* Generate a cache control header if necessary. We place a max
 672          cache age on index files, optionally set a no-cache directive
 673          and a no-store directive for archives. */
 674       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 675               Itm->Uri.c_str(),ProperHost.c_str());
 676       // only generate a cache control header if we actually want to
 677       // use a cache
 678       if (_config->FindB("Acquire::http::No-Cache",false) == false)
 679       {
 680          if (Itm->IndexFile == true)
 681             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 682                     _config->FindI("Acquire::http::Max-Age",0));
 683          else
 684          {
 685             if (_config->FindB("Acquire::http::No-Store",false) == true)
 686                strcat(Buf,"Cache-Control: no-store\r\n");
 687          }
 688       }
 689    }
 690    // generate a no-cache header if needed
 691    if (_config->FindB("Acquire::http::No-Cache",false) == true)
 692       strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 693
 694
 695    string Req = Buf;
 696
 697    // Check for a partial file
 698    struct stat SBuf;
 699    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 700    {
 701       // In this case we send an if-range query with a range header
 702       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 703               TimeRFC1123(SBuf.st_mtime).c_str());
 704       Req += Buf;
 705    }
 706    else
 707    {
 708       if (Itm->LastModified != 0)
 709       {
 710          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 711          Req += Buf;
 712       }
 713    }
 714
 715    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 716       Req += string("Proxy-Authorization: Basic ") +
 717           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 718
 719    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 720       Req += string("Authorization: Basic ") +
 721           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 722
 723    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 724
 725    if (Debug == true)
 726       cerr << Req << endl;
 727
 728    Out.Read(Req);
 729 }
 730                                                                         /*}}}*/
 731 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 732 // ---------------------------------------------------------------------
 733 /* This runs the select loop over the server FDs, Output file FDs and
 734    stdin. */
 735 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 736 {
 737    // Server has closed the connection
 738    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 739                                ToFile == false))
 740       return false;
 741
 742    fd_set rfds,wfds;
 743    FD_ZERO(&rfds);
 744    FD_ZERO(&wfds);
 745
 746    /* Add the server. We only send more requests if the connection will
 747       be persisting */
 748    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 749        && Srv->Persistent == true)
 750       FD_SET(Srv->ServerFd,&wfds);
 751    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 752       FD_SET(Srv->ServerFd,&rfds);
 753
 754    // Add the file
 755    int FileFD = -1;
 756    if (File != 0)
 757       FileFD = File->Fd();
 758
 759    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 760       FD_SET(FileFD,&wfds);
 761
 762    // Add stdin
 763    FD_SET(STDIN_FILENO,&rfds);
 764
 765    // Figure out the max fd
 766    int MaxFd = FileFD;
 767    if (MaxFd < Srv->ServerFd)
 768       MaxFd = Srv->ServerFd;
 769
 770    // Select
 771    struct timeval tv;
 772    tv.tv_sec = TimeOut;
 773    tv.tv_usec = 0;
 774    int Res = 0;
 775    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 776    {
 777       if (errno == EINTR)
 778          return true;
 779       return _error->Errno("select",_("Select failed"));
 780    }
 781
 782    if (Res == 0)
 783    {
 784       _error->Error(_("Connection timed out"));
 785       return ServerDie(Srv);
 786    }
 787
 788    // Handle server IO
 789    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 790    {
 791       errno = 0;
 792       if (Srv->In.Read(Srv->ServerFd) == false)
 793          return ServerDie(Srv);
 794    }
 795
 796    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 797    {
 798       errno = 0;
 799       if (Srv->Out.Write(Srv->ServerFd) == false)
 800          return ServerDie(Srv);
 801    }
 802
 803    // Send data to the file
 804    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 805    {
 806       if (Srv->In.Write(FileFD) == false)
 807          return _error->Errno("write",_("Error writing to output file"));
 808    }
 809
 810    // Handle commands from APT
 811    if (FD_ISSET(STDIN_FILENO,&rfds))
 812    {
 813       if (Run(true) != -1)
 814          exit(100);
 815    }
 816
 817    return true;
 818 }
 819                                                                         /*}}}*/
 820 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 821 // ---------------------------------------------------------------------
 822 /* This takes the current input buffer from the Server FD and writes it
 823    into the file */
 824 bool HttpMethod::Flush(ServerState *Srv)
 825 {
 826    if (File != 0)
 827    {
 828       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 829       // can't be set
 830       if (File->Name() != "/dev/null")
 831          SetNonBlock(File->Fd(),false);
 832       if (Srv->In.WriteSpace() == false)
 833          return true;
 834
 835       while (Srv->In.WriteSpace() == true)
 836       {
 837          if (Srv->In.Write(File->Fd()) == false)
 838             return _error->Errno("write",_("Error writing to file"));
 839          if (Srv->In.IsLimit() == true)
 840             return true;
 841       }
 842
 843       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 844          return true;
 845    }
 846    return false;
 847 }
 848                                                                         /*}}}*/
 849 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 850 // ---------------------------------------------------------------------
 851 /* */
 852 bool HttpMethod::ServerDie(ServerState *Srv)
 853 {
 854    unsigned int LErrno = errno;
 855
 856    // Dump the buffer to the file
 857    if (Srv->State == ServerState::Data)
 858    {
 859       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 860       // can't be set
 861       if (File->Name() != "/dev/null")
 862          SetNonBlock(File->Fd(),false);
 863       while (Srv->In.WriteSpace() == true)
 864       {
 865          if (Srv->In.Write(File->Fd()) == false)
 866             return _error->Errno("write",_("Error writing to the file"));
 867
 868          // Done
 869          if (Srv->In.IsLimit() == true)
 870             return true;
 871       }
 872    }
 873
 874    // See if this is because the server finished the data stream
 875    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 876        Srv->Encoding != ServerState::Closes)
 877    {
 878       Srv->Close();
 879       if (LErrno == 0)
 880          return _error->Error(_("Error reading from server. Remote end closed connection"));
 881       errno = LErrno;
 882       return _error->Errno("read",_("Error reading from server"));
 883    }
 884    else
 885    {
 886       Srv->In.Limit(-1);
 887
 888       // Nothing left in the buffer
 889       if (Srv->In.WriteSpace() == false)
 890          return false;
 891
 892       // We may have got multiple responses back in one packet..
 893       Srv->Close();
 894       return true;
 895    }
 896
 897    return false;
 898 }
 899                                                                         /*}}}*/
 900 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 901 // ---------------------------------------------------------------------
 902 /* We look at the header data we got back from the server and decide what
 903    to do. Returns
 904      0 - File is open,
 905      1 - IMS hit
 906      3 - Unrecoverable error
 907      4 - Error with error content page
 908      5 - Unrecoverable non-server error (close the connection) */
 909 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 910 {
 911    // Not Modified
 912    if (Srv->Result == 304)
 913    {
 914       unlink(Queue->DestFile.c_str());
 915       Res.IMSHit = true;
 916       Res.LastModified = Queue->LastModified;
 917       return 1;
 918    }
 919
 920    /* We have a reply we dont handle. This should indicate a perm server
 921       failure */
 922    if (Srv->Result < 200 || Srv->Result >= 300)
 923    {
 924       _error->Error("%u %s",Srv->Result,Srv->Code);
 925       if (Srv->HaveContent == true)
 926          return 4;
 927       return 3;
 928    }
 929
 930    // This is some sort of 2xx 'data follows' reply
 931    Res.LastModified = Srv->Date;
 932    Res.Size = Srv->Size;
 933
 934    // Open the file
 935    delete File;
 936    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 937    if (_error->PendingError() == true)
 938       return 5;
 939
 940    FailFile = Queue->DestFile;
 941    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 942    FailFd = File->Fd();
 943    FailTime = Srv->Date;
 944
 945    // Set the expected size
 946    if (Srv->StartPos >= 0)
 947    {
 948       Res.ResumePoint = Srv->StartPos;
 949       ftruncate(File->Fd(),Srv->StartPos);
 950    }
 951
 952    // Set the start point
 953    lseek(File->Fd(),0,SEEK_END);
 954
 955    delete Srv->In.Hash;
 956    Srv->In.Hash = new Hashes;
 957
 958    // Fill the Hash if the file is non-empty (resume)
 959    if (Srv->StartPos > 0)
 960    {
 961       lseek(File->Fd(),0,SEEK_SET);
 962       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
 963       {
 964          _error->Errno("read",_("Problem hashing file"));
 965          return 5;
 966       }
 967       lseek(File->Fd(),0,SEEK_END);
 968    }
 969
 970    SetNonBlock(File->Fd(),true);
 971    return 0;
 972 }
 973                                                                         /*}}}*/
 974 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 975 // ---------------------------------------------------------------------
 976 /* This closes and timestamps the open file. This is neccessary to get
 977    resume behavoir on user abort */
 978 void HttpMethod::SigTerm(int)
 979 {
 980    if (FailFd == -1)
 981       _exit(100);
 982    close(FailFd);
 983
 984    // Timestamp
 985    struct utimbuf UBuf;
 986    UBuf.actime = FailTime;
 987    UBuf.modtime = FailTime;
 988    utime(FailFile.c_str(),&UBuf);
 989
 990    _exit(100);
 991 }
 992                                                                         /*}}}*/
 993 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 994 // ---------------------------------------------------------------------
 995 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 996    depth. */
 997 bool HttpMethod::Fetch(FetchItem *)
 998 {
 999    if (Server == 0)
1000       return true;
1001
1002    // Queue the requests
1003    int Depth = -1;
1004    bool Tail = false;
1005    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
1006         I = I->Next, Depth++)
1007    {
1008       // If pipelining is disabled, we only queue 1 request
1009       if (Server->Pipeline == false && Depth >= 0)
1010          break;
1011
1012       // Make sure we stick with the same server
1013       if (Server->Comp(I->Uri) == false)
1014          break;
1015       if (QueueBack == I)
1016          Tail = true;
1017       if (Tail == true)
1018       {
1019          QueueBack = I->Next;
1020          SendReq(I,Server->Out);
1021          continue;
1022       }
1023    }
1024
1025    return true;
1026 };
1027                                                                         /*}}}*/
1028 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
1029 // ---------------------------------------------------------------------
1030 /* We stash the desired pipeline depth */
1031 bool HttpMethod::Configuration(string Message)
1032 {
1033    if (pkgAcqMethod::Configuration(Message) == false)
1034       return false;
1035
1036    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1037    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1038                                   PipelineDepth);
1039    Debug = _config->FindB("Debug::Acquire::http",false);
1040
1041    return true;
1042 }
1043                                                                         /*}}}*/
1044 // HttpMethod::Loop - Main loop                                         /*{{{*/
1045 // ---------------------------------------------------------------------
1046 /* */
1047 int HttpMethod::Loop()
1048 {
1049    signal(SIGTERM,SigTerm);
1050    signal(SIGINT,SigTerm);
1051
1052    Server = 0;
1053
1054    int FailCounter = 0;
1055    while (1)
1056    {
1057       // We have no commands, wait for some to arrive
1058       if (Queue == 0)
1059       {
1060          if (WaitFd(STDIN_FILENO) == false)
1061             return 0;
1062       }
1063
1064       /* Run messages, we can accept 0 (no message) if we didn't
1065          do a WaitFd above.. Otherwise the FD is closed. */
1066       int Result = Run(true);
1067       if (Result != -1 && (Result != 0 || Queue == 0))
1068          return 100;
1069
1070       if (Queue == 0)
1071          continue;
1072
1073       CFStringEncoding se = kCFStringEncodingUTF8;
1074
1075       CFStringRef sr = CFStringCreateWithCString(kCFAllocatorDefault, Queue->Uri.c_str(), se);
1076       CFURLRef ur = CFURLCreateWithString(kCFAllocatorDefault, sr, NULL);
1077       CFRelease(sr);
1078       CFHTTPMessageRef hm = CFHTTPMessageCreateRequest(kCFAllocatorDefault, CFSTR("GET"), ur, kCFHTTPVersion1_1);
1079       CFRelease(ur);
1080
1081       struct stat SBuf;
1082       if (stat(Queue->DestFile.c_str(), &SBuf) >= 0 && SBuf.st_size > 0) {
1083          sr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("bytes=%li-"), (long) SBuf.st_size - 1);
1084          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Range"), sr);
1085          CFRelease(sr);
1086
1087          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
1088          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Range"), sr);
1089          CFRelease(sr);
1090       } else if (Queue->LastModified != 0) {
1091          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
1092          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Modified-Since"), sr);
1093          CFRelease(sr);
1094       }
1095
1096       CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.98"));
1097       CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm);
1098       CFRelease(hm);
1099
1100       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPShouldAutoredirect, kCFBooleanTrue);
1101       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPAttemptPersistentConnection, kCFBooleanTrue);
1102
1103       URI uri = Queue->Uri;
1104
1105       FetchResult Res;
1106
1107       uint8_t data[10240];
1108       size_t offset = 0;
1109
1110       Status("Connecting to %s", uri.Host.c_str());
1111
1112       if (!CFReadStreamOpen(rs)) {
1113          _error->Error("Unable to open stream");
1114          Fail(true);
1115          goto done;
1116       }
1117
1118       CFIndex rd = CFReadStreamRead(rs, data, sizeof(data));
1119
1120       if (rd == -1) {
1121          _error->Error("Stream read failure");
1122          Fail(true);
1123          goto done;
1124       }
1125
1126       Res.Filename = Queue->DestFile;
1127
1128       hm = (CFHTTPMessageRef) CFReadStreamCopyProperty(rs, kCFStreamPropertyHTTPResponseHeader);
1129       UInt32 sc = CFHTTPMessageGetResponseStatusCode(hm);
1130
1131       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Range"));
1132       if (sr != NULL) {
1133          size_t ln = CFStringGetLength(sr) + 1;
1134          char cr[ln];
1135
1136          if (!CFStringGetCString(sr, cr, ln, se)) {
1137             Fail();
1138             goto done;
1139          }
1140
1141          CFRelease(sr);
1142
1143          if (sscanf(cr, "bytes %lu-%*u/%lu", &offset, &Res.Size) != 2) {
1144             _error->Error(_("The HTTP server sent an invalid Content-Range header"));
1145             Fail();
1146             goto done;
1147          }
1148
1149          if (offset > Res.Size) {
1150             _error->Error(_("This HTTP server has broken range support"));
1151             Fail();
1152             goto done;
1153          }
1154       } else {
1155          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Length"));
1156          if (sr != NULL) {
1157             Res.Size = CFStringGetIntValue(sr);
1158             CFRelease(sr);
1159          }
1160       }
1161
1162       time(&Res.LastModified);
1163
1164       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Last-Modified"));
1165       if (sr != NULL) {
1166          size_t ln = CFStringGetLength(sr) + 1;
1167          char cr[ln];
1168
1169          if (!CFStringGetCString(sr, cr, ln, se)) {
1170             Fail();
1171             goto done;
1172          }
1173
1174          CFRelease(sr);
1175
1176          if (!StrToTime(cr, Res.LastModified)) {
1177             _error->Error(_("Unknown date format"));
1178             Fail();
1179             goto done;
1180          }
1181       }
1182
1183       CFRelease(hm);
1184
1185       if (sc == 304) {
1186          unlink(Queue->DestFile.c_str());
1187          Res.IMSHit = true;
1188          Res.LastModified = Queue->LastModified;
1189          URIDone(Res);
1190       } else if (sc < 200 || sc >= 300)
1191          Fail();
1192       else {
1193          Hashes hash;
1194
1195          File = new FileFd(Queue->DestFile, FileFd::WriteAny);
1196          if (_error->PendingError() == true) {
1197             delete File;
1198             File = NULL;
1199             Fail();
1200             goto done;
1201          }
1202
1203          FailFile = Queue->DestFile;
1204          FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1205          FailFd = File->Fd();
1206          FailTime = Res.LastModified;
1207
1208          Res.ResumePoint = offset;
1209          ftruncate(File->Fd(), offset);
1210
1211          if (offset != 0) {
1212             lseek(File->Fd(), 0, SEEK_SET);
1213             if (!hash.AddFD(File->Fd(), offset)) {
1214                _error->Errno("read", _("Problem hashing file"));
1215                delete File;
1216                File = NULL;
1217                Fail();
1218                goto done;
1219             }
1220          }
1221
1222          lseek(File->Fd(), 0, SEEK_END);
1223
1224          URIStart(Res);
1225
1226          read: if (rd == -1) {
1227             _error->Error("Stream read failure");
1228             Fail(true);
1229          } else if (rd == 0) {
1230             if (Res.Size == 0)
1231                Res.Size = File->Size();
1232
1233             struct utimbuf UBuf;
1234             time(&UBuf.actime);
1235             UBuf.actime = Res.LastModified;
1236             UBuf.modtime = Res.LastModified;
1237             utime(Queue->DestFile.c_str(), &UBuf);
1238
1239             Res.TakeHashes(hash);
1240             URIDone(Res);
1241          } else {
1242             hash.Add(data, rd);
1243
1244             uint8_t *dt = data;
1245             while (rd != 0) {
1246                int sz = write(File->Fd(), dt, rd);
1247
1248                if (sz == -1) {
1249                    delete File;
1250                    File = NULL;
1251                    Fail();
1252                    goto done;
1253                }
1254
1255                dt += sz;
1256                rd -= sz;
1257             }
1258
1259             rd = CFReadStreamRead(rs, data, sizeof(data));
1260             goto read;
1261          }
1262       }
1263
1264     done:
1265       CFRelease(rs);
1266
1267       FailCounter = 0;
1268    }
1269
1270    return 0;
1271 }
1272                                                                         /*}}}*/
1273
1274 int main()
1275 {
1276    struct nlist nl[2];
1277    memset(nl, 0, sizeof(nl));
1278    nl[0].n_un.n_name = "_useMDNSResponder";
1279    nlist("/usr/lib/libc.dylib", nl);
1280    if (nl[0].n_type != N_UNDF)
1281        *(int *) nl[0].n_value = 0;
1282
1283    setlocale(LC_ALL, "");
1284
1285    HttpMethod Mth;
1286
1287    return Mth.Loop();
1288 }
1289
1290