methods/http.cc.orig

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40 #include <string.h>
  41 #include <iostream>
  42 #include <apti18n.h>
  43
  44 // Internet stuff
  45 #include <netdb.h>
  46
  47 #include <CoreFoundation/CoreFoundation.h>
  48 #include <CoreServices/CoreServices.h>
  49 #include <SystemConfiguration/SystemConfiguration.h>
  50
  51 #include "connect.h"
  52 #include "rfc2553emu.h"
  53 #include "http.h"
  54
  55                                                                         /*}}}*/
  56 using namespace std;
  57
  58 void CfrsError(CFReadStreamRef rs) {
  59     CFStreamError se = CFReadStreamGetError(rs);
  60
  61     if (se.domain == kCFStreamErrorDomainCustom) {
  62     } else if (se.domain == kCFStreamErrorDomainPOSIX) {
  63         _error->Error("POSIX: %s", strerror(se.error));
  64     } else if (se.domain == kCFStreamErrorDomainMacOSStatus) {
  65         _error->Error("MacOSStatus: %ld", se.error);
  66     } else if (se.domain == kCFStreamErrorDomainNetDB) {
  67         _error->Error("NetDB: %s", gai_strerror(se.error));
  68     } else if (se.domain == kCFStreamErrorDomainMach) {
  69         _error->Error("Mach: %ld", se.error);
  70     } else if (se.domain == kCFStreamErrorDomainHTTP) {
  71         switch (se.error) {
  72             case kCFStreamErrorHTTPParseFailure:
  73                 _error->Error("Parse failure");
  74             break;
  75
  76             case kCFStreamErrorHTTPRedirectionLoop:
  77                 _error->Error("Redirection loop");
  78             break;
  79
  80             case kCFStreamErrorHTTPBadURL:
  81                 _error->Error("Bad URL");
  82             break;
  83
  84             default:
  85                 _error->Error("Unknown HTTP error: %ld", se.error);
  86             break;
  87         }
  88     } else if (se.domain == kCFStreamErrorDomainSOCKS) {
  89         _error->Error("SOCKS: %ld", se.error);
  90     } else if (se.domain == kCFStreamErrorDomainSystemConfiguration) {
  91         _error->Error("SystemConfiguration: %ld", se.error);
  92     } else if (se.domain == kCFStreamErrorDomainSSL) {
  93         _error->Error("SSL: %ld", se.error);
  94     } else {
  95         _error->Error("Domain #%ld: %ld", se.domain, se.error);
  96     }
  97 }
  98
  99 string HttpMethod::FailFile;
 100 int HttpMethod::FailFd = -1;
 101 time_t HttpMethod::FailTime = 0;
 102 unsigned long PipelineDepth = 10;
 103 unsigned long TimeOut = 120;
 104 bool Debug = false;
 105
 106 unsigned long CircleBuf::BwReadLimit=0;
 107 unsigned long CircleBuf::BwTickReadData=0;
 108 struct timeval CircleBuf::BwReadTick={0,0};
 109 const unsigned int CircleBuf::BW_HZ=10;
 110
 111 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
 112 // ---------------------------------------------------------------------
 113 /* */
 114 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
 115 {
 116    Buf = new unsigned char[Size];
 117    Reset();
 118
 119    CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
 120 }
 121                                                                         /*}}}*/
 122 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
 123 // ---------------------------------------------------------------------
 124 /* */
 125 void CircleBuf::Reset()
 126 {
 127    InP = 0;
 128    OutP = 0;
 129    StrPos = 0;
 130    MaxGet = (unsigned int)-1;
 131    OutQueue = string();
 132    if (Hash != 0)
 133    {
 134       delete Hash;
 135       Hash = new Hashes;
 136    }
 137 };
 138                                                                         /*}}}*/
 139 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
 140 // ---------------------------------------------------------------------
 141 /* This fills up the buffer with as much data as is in the FD, assuming it
 142    is non-blocking.. */
 143 bool CircleBuf::Read(int Fd)
 144 {
 145    unsigned long BwReadMax;
 146
 147    while (1)
 148    {
 149       // Woops, buffer is full
 150       if (InP - OutP == Size)
 151          return true;
 152
 153       // what's left to read in this tick
 154       BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 155
 156       if(CircleBuf::BwReadLimit) {
 157          struct timeval now;
 158          gettimeofday(&now,0);
 159
 160          unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 161             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 162          if(d > 1000000/BW_HZ) {
 163             CircleBuf::BwReadTick = now;
 164             CircleBuf::BwTickReadData = 0;
 165          }
 166
 167          if(CircleBuf::BwTickReadData >= BwReadMax) {
 168             usleep(1000000/BW_HZ);
 169             return true;
 170          }
 171       }
 172
 173       // Write the buffer segment
 174       int Res;
 175       if(CircleBuf::BwReadLimit) {
 176          Res = read(Fd,Buf + (InP%Size),
 177                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 178       } else
 179          Res = read(Fd,Buf + (InP%Size),LeftRead());
 180
 181       if(Res > 0 && BwReadLimit > 0)
 182          CircleBuf::BwTickReadData += Res;
 183
 184       if (Res == 0)
 185          return false;
 186       if (Res < 0)
 187       {
 188          if (errno == EAGAIN)
 189             return true;
 190          return false;
 191       }
 192
 193       if (InP == 0)
 194          gettimeofday(&Start,0);
 195       InP += Res;
 196    }
 197 }
 198                                                                         /*}}}*/
 199 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 200 // ---------------------------------------------------------------------
 201 /* This will hold the string in and fill the buffer with it as it empties */
 202 bool CircleBuf::Read(string Data)
 203 {
 204    OutQueue += Data;
 205    FillOut();
 206    return true;
 207 }
 208                                                                         /*}}}*/
 209 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 210 // ---------------------------------------------------------------------
 211 /* */
 212 void CircleBuf::FillOut()
 213 {
 214    if (OutQueue.empty() == true)
 215       return;
 216    while (1)
 217    {
 218       // Woops, buffer is full
 219       if (InP - OutP == Size)
 220          return;
 221
 222       // Write the buffer segment
 223       unsigned long Sz = LeftRead();
 224       if (OutQueue.length() - StrPos < Sz)
 225          Sz = OutQueue.length() - StrPos;
 226       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 227
 228       // Advance
 229       StrPos += Sz;
 230       InP += Sz;
 231       if (OutQueue.length() == StrPos)
 232       {
 233          StrPos = 0;
 234          OutQueue = "";
 235          return;
 236       }
 237    }
 238 }
 239                                                                         /*}}}*/
 240 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 241 // ---------------------------------------------------------------------
 242 /* This empties the buffer into the FD. */
 243 bool CircleBuf::Write(int Fd)
 244 {
 245    while (1)
 246    {
 247       FillOut();
 248
 249       // Woops, buffer is empty
 250       if (OutP == InP)
 251          return true;
 252
 253       if (OutP == MaxGet)
 254          return true;
 255
 256       // Write the buffer segment
 257       int Res;
 258       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 259
 260       if (Res == 0)
 261          return false;
 262       if (Res < 0)
 263       {
 264          if (errno == EAGAIN)
 265             return true;
 266
 267          return false;
 268       }
 269
 270       if (Hash != 0)
 271          Hash->Add(Buf + (OutP%Size),Res);
 272
 273       OutP += Res;
 274    }
 275 }
 276                                                                         /*}}}*/
 277 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 278 // ---------------------------------------------------------------------
 279 /* This copies till the first empty line */
 280 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 281 {
 282    // We cheat and assume it is unneeded to have more than one buffer load
 283    for (unsigned long I = OutP; I < InP; I++)
 284    {
 285       if (Buf[I%Size] != '\n')
 286          continue;
 287       ++I;
 288
 289       if (Single == false)
 290       {
 291          if (I < InP  && Buf[I%Size] == '\r')
 292             ++I;
 293          if (I >= InP || Buf[I%Size] != '\n')
 294             continue;
 295          ++I;
 296       }
 297
 298       Data = "";
 299       while (OutP < I)
 300       {
 301          unsigned long Sz = LeftWrite();
 302          if (Sz == 0)
 303             return false;
 304          if (I - OutP < Sz)
 305             Sz = I - OutP;
 306          Data += string((char *)(Buf + (OutP%Size)),Sz);
 307          OutP += Sz;
 308       }
 309       return true;
 310    }
 311    return false;
 312 }
 313                                                                         /*}}}*/
 314 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 315 // ---------------------------------------------------------------------
 316 /* */
 317 void CircleBuf::Stats()
 318 {
 319    if (InP == 0)
 320       return;
 321
 322    struct timeval Stop;
 323    gettimeofday(&Stop,0);
 324 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 325              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 326    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 327 }
 328                                                                         /*}}}*/
 329
 330 // ServerState::ServerState - Constructor                               /*{{{*/
 331 // ---------------------------------------------------------------------
 332 /* */
 333 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 334                         In(64*1024), Out(4*1024),
 335                         ServerName(Srv)
 336 {
 337    Reset();
 338 }
 339                                                                         /*}}}*/
 340 // ServerState::Open - Open a connection to the server                  /*{{{*/
 341 // ---------------------------------------------------------------------
 342 /* This opens a connection to the server. */
 343 bool ServerState::Open()
 344 {
 345    // Use the already open connection if possible.
 346    if (ServerFd != -1)
 347       return true;
 348
 349    Close();
 350    In.Reset();
 351    Out.Reset();
 352    Persistent = true;
 353
 354    // Determine the proxy setting
 355    if (getenv("http_proxy") == 0)
 356    {
 357       string DefProxy = _config->Find("Acquire::http::Proxy");
 358       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 359       if (SpecificProxy.empty() == false)
 360       {
 361          if (SpecificProxy == "DIRECT")
 362             Proxy = "";
 363          else
 364             Proxy = SpecificProxy;
 365       }
 366       else
 367          Proxy = DefProxy;
 368    }
 369    else
 370       Proxy = getenv("http_proxy");
 371
 372    // Parse no_proxy, a , separated list of domains
 373    if (getenv("no_proxy") != 0)
 374    {
 375       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 376          Proxy = "";
 377    }
 378
 379    // Determine what host and port to use based on the proxy settings
 380    int Port = 0;
 381    string Host;
 382    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 383    {
 384       if (ServerName.Port != 0)
 385          Port = ServerName.Port;
 386       Host = ServerName.Host;
 387    }
 388    else
 389    {
 390       if (Proxy.Port != 0)
 391          Port = Proxy.Port;
 392       Host = Proxy.Host;
 393    }
 394
 395    // Connect to the remote server
 396    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 397       return false;
 398
 399    return true;
 400 }
 401                                                                         /*}}}*/
 402 // ServerState::Close - Close a connection to the server                /*{{{*/
 403 // ---------------------------------------------------------------------
 404 /* */
 405 bool ServerState::Close()
 406 {
 407    close(ServerFd);
 408    ServerFd = -1;
 409    return true;
 410 }
 411                                                                         /*}}}*/
 412 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 413 // ---------------------------------------------------------------------
 414 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 415    parse error occured */
 416 int ServerState::RunHeaders()
 417 {
 418    State = Header;
 419
 420    Owner->Status(_("Waiting for headers"));
 421
 422    Major = 0;
 423    Minor = 0;
 424    Result = 0;
 425    Size = 0;
 426    StartPos = 0;
 427    Encoding = Closes;
 428    HaveContent = false;
 429    time(&Date);
 430
 431    do
 432    {
 433       string Data;
 434       if (In.WriteTillEl(Data) == false)
 435          continue;
 436
 437       if (Debug == true)
 438          clog << Data;
 439
 440       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 441       {
 442          string::const_iterator J = I;
 443          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 444          if (HeaderLine(string(I,J)) == false)
 445             return 2;
 446          I = J;
 447       }
 448
 449       // 100 Continue is a Nop...
 450       if (Result == 100)
 451          continue;
 452
 453       // Tidy up the connection persistance state.
 454       if (Encoding == Closes && HaveContent == true)
 455          Persistent = false;
 456
 457       return 0;
 458    }
 459    while (Owner->Go(false,this) == true);
 460
 461    return 1;
 462 }
 463                                                                         /*}}}*/
 464 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 465 // ---------------------------------------------------------------------
 466 /* */
 467 bool ServerState::RunData()
 468 {
 469    State = Data;
 470
 471    // Chunked transfer encoding is fun..
 472    if (Encoding == Chunked)
 473    {
 474       while (1)
 475       {
 476          // Grab the block size
 477          bool Last = true;
 478          string Data;
 479          In.Limit(-1);
 480          do
 481          {
 482             if (In.WriteTillEl(Data,true) == true)
 483                break;
 484          }
 485          while ((Last = Owner->Go(false,this)) == true);
 486
 487          if (Last == false)
 488             return false;
 489
 490          // See if we are done
 491          unsigned long Len = strtol(Data.c_str(),0,16);
 492          if (Len == 0)
 493          {
 494             In.Limit(-1);
 495
 496             // We have to remove the entity trailer
 497             Last = true;
 498             do
 499             {
 500                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 501                   break;
 502             }
 503             while ((Last = Owner->Go(false,this)) == true);
 504             if (Last == false)
 505                return false;
 506             return !_error->PendingError();
 507          }
 508
 509          // Transfer the block
 510          In.Limit(Len);
 511          while (Owner->Go(true,this) == true)
 512             if (In.IsLimit() == true)
 513                break;
 514
 515          // Error
 516          if (In.IsLimit() == false)
 517             return false;
 518
 519          // The server sends an extra new line before the next block specifier..
 520          In.Limit(-1);
 521          Last = true;
 522          do
 523          {
 524             if (In.WriteTillEl(Data,true) == true)
 525                break;
 526          }
 527          while ((Last = Owner->Go(false,this)) == true);
 528          if (Last == false)
 529             return false;
 530       }
 531    }
 532    else
 533    {
 534       /* Closes encoding is used when the server did not specify a size, the
 535          loss of the connection means we are done */
 536       if (Encoding == Closes)
 537          In.Limit(-1);
 538       else
 539          In.Limit(Size - StartPos);
 540
 541       // Just transfer the whole block.
 542       do
 543       {
 544          if (In.IsLimit() == false)
 545             continue;
 546
 547          In.Limit(-1);
 548          return !_error->PendingError();
 549       }
 550       while (Owner->Go(true,this) == true);
 551    }
 552
 553    return Owner->Flush(this) && !_error->PendingError();
 554 }
 555                                                                         /*}}}*/
 556 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 557 // ---------------------------------------------------------------------
 558 /* */
 559 bool ServerState::HeaderLine(string Line)
 560 {
 561    if (Line.empty() == true)
 562       return true;
 563
 564    // The http server might be trying to do something evil.
 565    if (Line.length() >= MAXLEN)
 566       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 567
 568    string::size_type Pos = Line.find(' ');
 569    if (Pos == string::npos || Pos+1 > Line.length())
 570    {
 571       // Blah, some servers use "connection:closes", evil.
 572       Pos = Line.find(':');
 573       if (Pos == string::npos || Pos + 2 > Line.length())
 574          return _error->Error(_("Bad header line"));
 575       Pos++;
 576    }
 577
 578    // Parse off any trailing spaces between the : and the next word.
 579    string::size_type Pos2 = Pos;
 580    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 581       Pos2++;
 582
 583    string Tag = string(Line,0,Pos);
 584    string Val = string(Line,Pos2);
 585
 586    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 587    {
 588       // Evil servers return no version
 589       if (Line[4] == '/')
 590       {
 591          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 592                     &Result,Code) != 4)
 593             return _error->Error(_("The HTTP server sent an invalid reply header"));
 594       }
 595       else
 596       {
 597          Major = 0;
 598          Minor = 9;
 599          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 600             return _error->Error(_("The HTTP server sent an invalid reply header"));
 601       }
 602
 603       /* Check the HTTP response header to get the default persistance
 604          state. */
 605       if (Major < 1)
 606          Persistent = false;
 607       else
 608       {
 609          if (Major == 1 && Minor <= 0)
 610             Persistent = false;
 611          else
 612             Persistent = true;
 613       }
 614
 615       return true;
 616    }
 617
 618    if (stringcasecmp(Tag,"Content-Length:") == 0)
 619    {
 620       if (Encoding == Closes)
 621          Encoding = Stream;
 622       HaveContent = true;
 623
 624       // The length is already set from the Content-Range header
 625       if (StartPos != 0)
 626          return true;
 627
 628       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 629          return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
 630       return true;
 631    }
 632
 633    if (stringcasecmp(Tag,"Content-Type:") == 0)
 634    {
 635       HaveContent = true;
 636       return true;
 637    }
 638
 639    if (stringcasecmp(Tag,"Content-Range:") == 0)
 640    {
 641       HaveContent = true;
 642
 643       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 644          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 645       if ((unsigned)StartPos > Size)
 646          return _error->Error(_("This HTTP server has broken range support"));
 647       return true;
 648    }
 649
 650    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 651    {
 652       HaveContent = true;
 653       if (stringcasecmp(Val,"chunked") == 0)
 654          Encoding = Chunked;
 655       return true;
 656    }
 657
 658    if (stringcasecmp(Tag,"Connection:") == 0)
 659    {
 660       if (stringcasecmp(Val,"close") == 0)
 661          Persistent = false;
 662       if (stringcasecmp(Val,"keep-alive") == 0)
 663          Persistent = true;
 664       return true;
 665    }
 666
 667    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 668    {
 669       if (StrToTime(Val,Date) == false)
 670          return _error->Error(_("Unknown date format"));
 671       return true;
 672    }
 673
 674    return true;
 675 }
 676                                                                         /*}}}*/
 677
 678 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 679 // ---------------------------------------------------------------------
 680 /* This places the http request in the outbound buffer */
 681 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 682 {
 683    URI Uri = Itm->Uri;
 684
 685    // The HTTP server expects a hostname with a trailing :port
 686    char Buf[1000];
 687    string ProperHost = Uri.Host;
 688    if (Uri.Port != 0)
 689    {
 690       sprintf(Buf,":%u",Uri.Port);
 691       ProperHost += Buf;
 692    }
 693
 694    // Just in case.
 695    if (Itm->Uri.length() >= sizeof(Buf))
 696        abort();
 697
 698    /* Build the request. We include a keep-alive header only for non-proxy
 699       requests. This is to tweak old http/1.0 servers that do support keep-alive
 700       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 701       will glitch HTTP/1.0 proxies because they do not filter it out and
 702       pass it on, HTTP/1.1 says the connection should default to keep alive
 703       and we expect the proxy to do this */
 704    if (Proxy.empty() == true || Proxy.Host.empty())
 705       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 706               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 707    else
 708    {
 709       /* Generate a cache control header if necessary. We place a max
 710          cache age on index files, optionally set a no-cache directive
 711          and a no-store directive for archives. */
 712       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 713               Itm->Uri.c_str(),ProperHost.c_str());
 714       // only generate a cache control header if we actually want to
 715       // use a cache
 716       if (_config->FindB("Acquire::http::No-Cache",false) == false)
 717       {
 718          if (Itm->IndexFile == true)
 719             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 720                     _config->FindI("Acquire::http::Max-Age",0));
 721          else
 722          {
 723             if (_config->FindB("Acquire::http::No-Store",false) == true)
 724                strcat(Buf,"Cache-Control: no-store\r\n");
 725          }
 726       }
 727    }
 728    // generate a no-cache header if needed
 729    if (_config->FindB("Acquire::http::No-Cache",false) == true)
 730       strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 731
 732
 733    string Req = Buf;
 734
 735    // Check for a partial file
 736    struct stat SBuf;
 737    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 738    {
 739       // In this case we send an if-range query with a range header
 740       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 741               TimeRFC1123(SBuf.st_mtime).c_str());
 742       Req += Buf;
 743    }
 744    else
 745    {
 746       if (Itm->LastModified != 0)
 747       {
 748          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 749          Req += Buf;
 750       }
 751    }
 752
 753    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 754       Req += string("Proxy-Authorization: Basic ") +
 755           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 756
 757    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 758       Req += string("Authorization: Basic ") +
 759           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 760
 761    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 762
 763    if (Debug == true)
 764       cerr << Req << endl;
 765
 766    Out.Read(Req);
 767 }
 768                                                                         /*}}}*/
 769 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 770 // ---------------------------------------------------------------------
 771 /* This runs the select loop over the server FDs, Output file FDs and
 772    stdin. */
 773 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 774 {
 775    // Server has closed the connection
 776    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 777                                ToFile == false))
 778       return false;
 779
 780    fd_set rfds,wfds;
 781    FD_ZERO(&rfds);
 782    FD_ZERO(&wfds);
 783
 784    /* Add the server. We only send more requests if the connection will
 785       be persisting */
 786    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 787        && Srv->Persistent == true)
 788       FD_SET(Srv->ServerFd,&wfds);
 789    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 790       FD_SET(Srv->ServerFd,&rfds);
 791
 792    // Add the file
 793    int FileFD = -1;
 794    if (File != 0)
 795       FileFD = File->Fd();
 796
 797    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 798       FD_SET(FileFD,&wfds);
 799
 800    // Add stdin
 801    FD_SET(STDIN_FILENO,&rfds);
 802
 803    // Figure out the max fd
 804    int MaxFd = FileFD;
 805    if (MaxFd < Srv->ServerFd)
 806       MaxFd = Srv->ServerFd;
 807
 808    // Select
 809    struct timeval tv;
 810    tv.tv_sec = TimeOut;
 811    tv.tv_usec = 0;
 812    int Res = 0;
 813    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 814    {
 815       if (errno == EINTR)
 816          return true;
 817       return _error->Errno("select",_("Select failed"));
 818    }
 819
 820    if (Res == 0)
 821    {
 822       _error->Error(_("Connection timed out"));
 823       return ServerDie(Srv);
 824    }
 825
 826    // Handle server IO
 827    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 828    {
 829       errno = 0;
 830       if (Srv->In.Read(Srv->ServerFd) == false)
 831          return ServerDie(Srv);
 832    }
 833
 834    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 835    {
 836       errno = 0;
 837       if (Srv->Out.Write(Srv->ServerFd) == false)
 838          return ServerDie(Srv);
 839    }
 840
 841    // Send data to the file
 842    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 843    {
 844       if (Srv->In.Write(FileFD) == false)
 845          return _error->Errno("write",_("Error writing to output file"));
 846    }
 847
 848    // Handle commands from APT
 849    if (FD_ISSET(STDIN_FILENO,&rfds))
 850    {
 851       if (Run(true) != -1)
 852          exit(100);
 853    }
 854
 855    return true;
 856 }
 857                                                                         /*}}}*/
 858 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 859 // ---------------------------------------------------------------------
 860 /* This takes the current input buffer from the Server FD and writes it
 861    into the file */
 862 bool HttpMethod::Flush(ServerState *Srv)
 863 {
 864    if (File != 0)
 865    {
 866       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 867       // can't be set
 868       if (File->Name() != "/dev/null")
 869          SetNonBlock(File->Fd(),false);
 870       if (Srv->In.WriteSpace() == false)
 871          return true;
 872
 873       while (Srv->In.WriteSpace() == true)
 874       {
 875          if (Srv->In.Write(File->Fd()) == false)
 876             return _error->Errno("write",_("Error writing to file"));
 877          if (Srv->In.IsLimit() == true)
 878             return true;
 879       }
 880
 881       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 882          return true;
 883    }
 884    return false;
 885 }
 886                                                                         /*}}}*/
 887 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 888 // ---------------------------------------------------------------------
 889 /* */
 890 bool HttpMethod::ServerDie(ServerState *Srv)
 891 {
 892    unsigned int LErrno = errno;
 893
 894    // Dump the buffer to the file
 895    if (Srv->State == ServerState::Data)
 896    {
 897       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 898       // can't be set
 899       if (File->Name() != "/dev/null")
 900          SetNonBlock(File->Fd(),false);
 901       while (Srv->In.WriteSpace() == true)
 902       {
 903          if (Srv->In.Write(File->Fd()) == false)
 904             return _error->Errno("write",_("Error writing to the file"));
 905
 906          // Done
 907          if (Srv->In.IsLimit() == true)
 908             return true;
 909       }
 910    }
 911
 912    // See if this is because the server finished the data stream
 913    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 914        Srv->Encoding != ServerState::Closes)
 915    {
 916       Srv->Close();
 917       if (LErrno == 0)
 918          return _error->Error(_("Error reading from server. Remote end closed connection"));
 919       errno = LErrno;
 920       return _error->Errno("read",_("Error reading from server"));
 921    }
 922    else
 923    {
 924       Srv->In.Limit(-1);
 925
 926       // Nothing left in the buffer
 927       if (Srv->In.WriteSpace() == false)
 928          return false;
 929
 930       // We may have got multiple responses back in one packet..
 931       Srv->Close();
 932       return true;
 933    }
 934
 935    return false;
 936 }
 937                                                                         /*}}}*/
 938 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 939 // ---------------------------------------------------------------------
 940 /* We look at the header data we got back from the server and decide what
 941    to do. Returns
 942      0 - File is open,
 943      1 - IMS hit
 944      3 - Unrecoverable error
 945      4 - Error with error content page
 946      5 - Unrecoverable non-server error (close the connection) */
 947 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 948 {
 949    // Not Modified
 950    if (Srv->Result == 304)
 951    {
 952       unlink(Queue->DestFile.c_str());
 953       Res.IMSHit = true;
 954       Res.LastModified = Queue->LastModified;
 955       return 1;
 956    }
 957
 958    /* We have a reply we dont handle. This should indicate a perm server
 959       failure */
 960    if (Srv->Result < 200 || Srv->Result >= 300)
 961    {
 962       _error->Error("%u %s",Srv->Result,Srv->Code);
 963       if (Srv->HaveContent == true)
 964          return 4;
 965       return 3;
 966    }
 967
 968    // This is some sort of 2xx 'data follows' reply
 969    Res.LastModified = Srv->Date;
 970    Res.Size = Srv->Size;
 971
 972    // Open the file
 973    delete File;
 974    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 975    if (_error->PendingError() == true)
 976       return 5;
 977
 978    FailFile = Queue->DestFile;
 979    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 980    FailFd = File->Fd();
 981    FailTime = Srv->Date;
 982
 983    // Set the expected size
 984    if (Srv->StartPos >= 0)
 985    {
 986       Res.ResumePoint = Srv->StartPos;
 987       ftruncate(File->Fd(),Srv->StartPos);
 988    }
 989
 990    // Set the start point
 991    lseek(File->Fd(),0,SEEK_END);
 992
 993    delete Srv->In.Hash;
 994    Srv->In.Hash = new Hashes;
 995
 996    // Fill the Hash if the file is non-empty (resume)
 997    if (Srv->StartPos > 0)
 998    {
 999       lseek(File->Fd(),0,SEEK_SET);
1000       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
1001       {
1002          _error->Errno("read",_("Problem hashing file"));
1003          return 5;
1004       }
1005       lseek(File->Fd(),0,SEEK_END);
1006    }
1007
1008    SetNonBlock(File->Fd(),true);
1009    return 0;
1010 }
1011                                                                         /*}}}*/
1012 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
1013 // ---------------------------------------------------------------------
1014 /* This closes and timestamps the open file. This is neccessary to get
1015    resume behavoir on user abort */
1016 void HttpMethod::SigTerm(int)
1017 {
1018    if (FailFd == -1)
1019       _exit(100);
1020    close(FailFd);
1021
1022    // Timestamp
1023    struct utimbuf UBuf;
1024    UBuf.actime = FailTime;
1025    UBuf.modtime = FailTime;
1026    utime(FailFile.c_str(),&UBuf);
1027
1028    _exit(100);
1029 }
1030                                                                         /*}}}*/
1031 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
1032 // ---------------------------------------------------------------------
1033 /* This adds an item to the pipeline. We keep the pipeline at a fixed
1034    depth. */
1035 bool HttpMethod::Fetch(FetchItem *)
1036 {
1037    if (Server == 0)
1038       return true;
1039
1040    // Queue the requests
1041    int Depth = -1;
1042    bool Tail = false;
1043    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
1044         I = I->Next, Depth++)
1045    {
1046       // If pipelining is disabled, we only queue 1 request
1047       if (Server->Pipeline == false && Depth >= 0)
1048          break;
1049
1050       // Make sure we stick with the same server
1051       if (Server->Comp(I->Uri) == false)
1052          break;
1053       if (QueueBack == I)
1054          Tail = true;
1055       if (Tail == true)
1056       {
1057          QueueBack = I->Next;
1058          SendReq(I,Server->Out);
1059          continue;
1060       }
1061    }
1062
1063    return true;
1064 };
1065                                                                         /*}}}*/
1066 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
1067 // ---------------------------------------------------------------------
1068 /* We stash the desired pipeline depth */
1069 bool HttpMethod::Configuration(string Message)
1070 {
1071    if (pkgAcqMethod::Configuration(Message) == false)
1072       return false;
1073
1074    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1075    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1076                                   PipelineDepth);
1077    Debug = _config->FindB("Debug::Acquire::http",false);
1078
1079    return true;
1080 }
1081                                                                         /*}}}*/
1082 // HttpMethod::Loop - Main loop                                         /*{{{*/
1083 // ---------------------------------------------------------------------
1084 /* */
1085 int HttpMethod::Loop()
1086 {
1087    signal(SIGTERM,SigTerm);
1088    signal(SIGINT,SigTerm);
1089
1090    Server = 0;
1091
1092    int FailCounter = 0;
1093    while (1)
1094    {
1095       // We have no commands, wait for some to arrive
1096       if (Queue == 0)
1097       {
1098          if (WaitFd(STDIN_FILENO) == false)
1099             return 0;
1100       }
1101
1102       /* Run messages, we can accept 0 (no message) if we didn't
1103          do a WaitFd above.. Otherwise the FD is closed. */
1104       int Result = Run(true);
1105       if (Result != -1 && (Result != 0 || Queue == 0))
1106          return 100;
1107
1108       if (Queue == 0)
1109          continue;
1110
1111       CFStringEncoding se = kCFStringEncodingUTF8;
1112
1113       char *url = strdup(Queue->Uri.c_str());
1114     url:
1115       CFStringRef sr = CFStringCreateWithCString(kCFAllocatorDefault, url, se);
1116       CFURLRef ur = CFURLCreateWithString(kCFAllocatorDefault, sr, NULL);
1117       CFRelease(sr);
1118       CFHTTPMessageRef hm = CFHTTPMessageCreateRequest(kCFAllocatorDefault, CFSTR("GET"), ur, kCFHTTPVersion1_1);
1119       CFRelease(ur);
1120
1121       struct stat SBuf;
1122       if (stat(Queue->DestFile.c_str(), &SBuf) >= 0 && SBuf.st_size > 0) {
1123          sr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("bytes=%li-"), (long) SBuf.st_size - 1);
1124          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Range"), sr);
1125          CFRelease(sr);
1126
1127          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
1128          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Range"), sr);
1129          CFRelease(sr);
1130       } else if (Queue->LastModified != 0) {
1131          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
1132          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Modified-Since"), sr);
1133          CFRelease(sr);
1134       }
1135
1136       CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.98"));
1137       CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm);
1138       CFRelease(hm);
1139
1140       CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL);
1141       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr);
1142       CFRelease(dr);
1143
1144       //CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPShouldAutoredirect, kCFBooleanTrue);
1145       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPAttemptPersistentConnection, kCFBooleanTrue);
1146
1147       URI uri = Queue->Uri;
1148
1149       FetchResult Res;
1150       CFIndex rd;
1151       UInt32 sc;
1152
1153       uint8_t data[10240];
1154       size_t offset = 0;
1155
1156       Status("Connecting to %s", uri.Host.c_str());
1157
1158       if (!CFReadStreamOpen(rs)) {
1159          CfrsError(rs);
1160          Fail(true);
1161          goto done;
1162       }
1163
1164       rd = CFReadStreamRead(rs, data, sizeof(data));
1165
1166       if (rd == -1) {
1167          CfrsError(rs);
1168          Fail(true);
1169          goto done;
1170       }
1171
1172       Res.Filename = Queue->DestFile;
1173
1174       hm = (CFHTTPMessageRef) CFReadStreamCopyProperty(rs, kCFStreamPropertyHTTPResponseHeader);
1175       sc = CFHTTPMessageGetResponseStatusCode(hm);
1176
1177       if (sc == 302) {
1178          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Location"));
1179          if (sr == NULL) {
1180             Fail();
1181             goto done_;
1182          } else {
1183             size_t ln = CFStringGetLength(sr) + 1;
1184             free(url);
1185             url = static_cast<char *>(malloc(ln));
1186
1187             if (!CFStringGetCString(sr, url, ln, se)) {
1188                Fail();
1189                goto done_;
1190             }
1191
1192             CFRelease(sr);
1193             goto url;
1194          }
1195       }
1196
1197       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Range"));
1198       if (sr != NULL) {
1199          size_t ln = CFStringGetLength(sr) + 1;
1200          char cr[ln];
1201
1202          if (!CFStringGetCString(sr, cr, ln, se)) {
1203             Fail();
1204             goto done_;
1205          }
1206
1207          CFRelease(sr);
1208
1209          if (sscanf(cr, "bytes %lu-%*u/%lu", &offset, &Res.Size) != 2) {
1210             _error->Error(_("The HTTP server sent an invalid Content-Range header"));
1211             Fail();
1212             goto done_;
1213          }
1214
1215          if (offset > Res.Size) {
1216             _error->Error(_("This HTTP server has broken range support"));
1217             Fail();
1218             goto done_;
1219          }
1220       } else {
1221          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Length"));
1222          if (sr != NULL) {
1223             Res.Size = CFStringGetIntValue(sr);
1224             CFRelease(sr);
1225          }
1226       }
1227
1228       time(&Res.LastModified);
1229
1230       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Last-Modified"));
1231       if (sr != NULL) {
1232          size_t ln = CFStringGetLength(sr) + 1;
1233          char cr[ln];
1234
1235          if (!CFStringGetCString(sr, cr, ln, se)) {
1236             Fail();
1237             goto done_;
1238          }
1239
1240          CFRelease(sr);
1241
1242          if (!StrToTime(cr, Res.LastModified)) {
1243             _error->Error(_("Unknown date format"));
1244             Fail();
1245             goto done_;
1246          }
1247       }
1248
1249       CFRelease(hm);
1250
1251       if (sc == 304) {
1252          unlink(Queue->DestFile.c_str());
1253          Res.IMSHit = true;
1254          Res.LastModified = Queue->LastModified;
1255          URIDone(Res);
1256       } else if (sc < 200 || sc >= 300)
1257          Fail();
1258       else {
1259          Hashes hash;
1260
1261          File = new FileFd(Queue->DestFile, FileFd::WriteAny);
1262          if (_error->PendingError() == true) {
1263             delete File;
1264             File = NULL;
1265             Fail();
1266             goto done;
1267          }
1268
1269          FailFile = Queue->DestFile;
1270          FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1271          FailFd = File->Fd();
1272          FailTime = Res.LastModified;
1273
1274          Res.ResumePoint = offset;
1275          ftruncate(File->Fd(), offset);
1276
1277          if (offset != 0) {
1278             lseek(File->Fd(), 0, SEEK_SET);
1279             if (!hash.AddFD(File->Fd(), offset)) {
1280                _error->Errno("read", _("Problem hashing file"));
1281                delete File;
1282                File = NULL;
1283                Fail();
1284                goto done;
1285             }
1286          }
1287
1288          lseek(File->Fd(), 0, SEEK_END);
1289
1290          URIStart(Res);
1291
1292          read: if (rd == -1) {
1293             CfrsError(rs);
1294             Fail(true);
1295          } else if (rd == 0) {
1296             if (Res.Size == 0)
1297                Res.Size = File->Size();
1298
1299             struct utimbuf UBuf;
1300             time(&UBuf.actime);
1301             UBuf.actime = Res.LastModified;
1302             UBuf.modtime = Res.LastModified;
1303             utime(Queue->DestFile.c_str(), &UBuf);
1304
1305             Res.TakeHashes(hash);
1306             URIDone(Res);
1307          } else {
1308             hash.Add(data, rd);
1309
1310             uint8_t *dt = data;
1311             while (rd != 0) {
1312                int sz = write(File->Fd(), dt, rd);
1313
1314                if (sz == -1) {
1315                    delete File;
1316                    File = NULL;
1317                    Fail();
1318                    goto done;
1319                }
1320
1321                dt += sz;
1322                rd -= sz;
1323             }
1324
1325             rd = CFReadStreamRead(rs, data, sizeof(data));
1326             goto read;
1327          }
1328       }
1329
1330      goto done;
1331     done_:
1332       CFRelease(hm);
1333     done:
1334       CFReadStreamClose(rs);
1335       CFRelease(rs);
1336       free(url);
1337
1338       FailCounter = 0;
1339    }
1340
1341    return 0;
1342 }
1343                                                                         /*}}}*/
1344
1345 int main()
1346 {
1347    setlocale(LC_ALL, "");
1348
1349    HttpMethod Mth;
1350
1351    return Mth.Loop();
1352 }
1353
1354