methods/http.cc.orig

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on.
  10
  11    It is based on a doubly buffered select loop. A groupe of requests are
  12    fed into a single output buffer that is constantly fed out the
  13    socket. This provides ideal pipelining as in many cases all of the
  14    requests will fit into a single packet. The input socket is buffered
  15    the same way and fed into the fd for the file (may be a pipe in future).
  16
  17    This double buffering provides fairly substantial transfer rates,
  18    compared to wget the http method is about 4% faster. Most importantly,
  19    when HTTP is compared with FTP as a protocol the speed difference is
  20    huge. In tests over the internet from two sites to llug (via ATM) this
  21    program got 230k/s sustained http transfer rates. FTP on the other
  22    hand topped out at 170k/s. That combined with the time to setup the
  23    FTP connection makes HTTP a vastly superior protocol.
  24
  25    ##################################################################### */
  26                                                                         /*}}}*/
  27 // Include Files                                                        /*{{{*/
  28 #include <apt-pkg/fileutl.h>
  29 #include <apt-pkg/acquire-method.h>
  30 #include <apt-pkg/error.h>
  31 #include <apt-pkg/hashes.h>
  32
  33 #include <sys/stat.h>
  34 #include <sys/time.h>
  35 #include <utime.h>
  36 #include <unistd.h>
  37 #include <signal.h>
  38 #include <stdio.h>
  39 #include <errno.h>
  40 #include <string.h>
  41 #include <iostream>
  42 #include <apti18n.h>
  43
  44 // Internet stuff
  45 #include <netdb.h>
  46 #include <arpa/inet.h>
  47
  48 #include <CoreFoundation/CoreFoundation.h>
  49 #include <CoreServices/CoreServices.h>
  50 #include <SystemConfiguration/SystemConfiguration.h>
  51
  52 #include "connect.h"
  53 #include "rfc2553emu.h"
  54 #include "http.h"
  55
  56                                                                         /*}}}*/
  57 using namespace std;
  58
  59 void CfrsError(const char *name, CFReadStreamRef rs) {
  60     CFStreamError se = CFReadStreamGetError(rs);
  61
  62     if (se.domain == kCFStreamErrorDomainCustom) {
  63     } else if (se.domain == kCFStreamErrorDomainPOSIX) {
  64         _error->Error("POSIX: %s", strerror(se.error));
  65     } else if (se.domain == kCFStreamErrorDomainMacOSStatus) {
  66         _error->Error("MacOSStatus: %ld", se.error);
  67     } else if (se.domain == kCFStreamErrorDomainNetDB) {
  68         _error->Error("NetDB: %s %s", name, gai_strerror(se.error));
  69     } else if (se.domain == kCFStreamErrorDomainMach) {
  70         _error->Error("Mach: %ld", se.error);
  71     } else if (se.domain == kCFStreamErrorDomainHTTP) {
  72         switch (se.error) {
  73             case kCFStreamErrorHTTPParseFailure:
  74                 _error->Error("Parse failure");
  75             break;
  76
  77             case kCFStreamErrorHTTPRedirectionLoop:
  78                 _error->Error("Redirection loop");
  79             break;
  80
  81             case kCFStreamErrorHTTPBadURL:
  82                 _error->Error("Bad URL");
  83             break;
  84
  85             default:
  86                 _error->Error("Unknown HTTP error: %ld", se.error);
  87             break;
  88         }
  89     } else if (se.domain == kCFStreamErrorDomainSOCKS) {
  90         _error->Error("SOCKS: %ld", se.error);
  91     } else if (se.domain == kCFStreamErrorDomainSystemConfiguration) {
  92         _error->Error("SystemConfiguration: %ld", se.error);
  93     } else if (se.domain == kCFStreamErrorDomainSSL) {
  94         _error->Error("SSL: %ld", se.error);
  95     } else {
  96         _error->Error("Domain #%ld: %ld", se.domain, se.error);
  97     }
  98 }
  99
 100 string HttpMethod::FailFile;
 101 int HttpMethod::FailFd = -1;
 102 time_t HttpMethod::FailTime = 0;
 103 unsigned long PipelineDepth = 10;
 104 unsigned long TimeOut = 120;
 105 bool Debug = false;
 106
 107 unsigned long CircleBuf::BwReadLimit=0;
 108 unsigned long CircleBuf::BwTickReadData=0;
 109 struct timeval CircleBuf::BwReadTick={0,0};
 110 const unsigned int CircleBuf::BW_HZ=10;
 111
 112 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
 113 // ---------------------------------------------------------------------
 114 /* */
 115 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
 116 {
 117    Buf = new unsigned char[Size];
 118    Reset();
 119
 120    CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
 121 }
 122                                                                         /*}}}*/
 123 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
 124 // ---------------------------------------------------------------------
 125 /* */
 126 void CircleBuf::Reset()
 127 {
 128    InP = 0;
 129    OutP = 0;
 130    StrPos = 0;
 131    MaxGet = (unsigned int)-1;
 132    OutQueue = string();
 133    if (Hash != 0)
 134    {
 135       delete Hash;
 136       Hash = new Hashes;
 137    }
 138 };
 139                                                                         /*}}}*/
 140 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
 141 // ---------------------------------------------------------------------
 142 /* This fills up the buffer with as much data as is in the FD, assuming it
 143    is non-blocking.. */
 144 bool CircleBuf::Read(int Fd)
 145 {
 146    unsigned long BwReadMax;
 147
 148    while (1)
 149    {
 150       // Woops, buffer is full
 151       if (InP - OutP == Size)
 152          return true;
 153
 154       // what's left to read in this tick
 155       BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
 156
 157       if(CircleBuf::BwReadLimit) {
 158          struct timeval now;
 159          gettimeofday(&now,0);
 160
 161          unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
 162             now.tv_usec-CircleBuf::BwReadTick.tv_usec;
 163          if(d > 1000000/BW_HZ) {
 164             CircleBuf::BwReadTick = now;
 165             CircleBuf::BwTickReadData = 0;
 166          }
 167
 168          if(CircleBuf::BwTickReadData >= BwReadMax) {
 169             usleep(1000000/BW_HZ);
 170             return true;
 171          }
 172       }
 173
 174       // Write the buffer segment
 175       int Res;
 176       if(CircleBuf::BwReadLimit) {
 177          Res = read(Fd,Buf + (InP%Size),
 178                     BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
 179       } else
 180          Res = read(Fd,Buf + (InP%Size),LeftRead());
 181
 182       if(Res > 0 && BwReadLimit > 0)
 183          CircleBuf::BwTickReadData += Res;
 184
 185       if (Res == 0)
 186          return false;
 187       if (Res < 0)
 188       {
 189          if (errno == EAGAIN)
 190             return true;
 191          return false;
 192       }
 193
 194       if (InP == 0)
 195          gettimeofday(&Start,0);
 196       InP += Res;
 197    }
 198 }
 199                                                                         /*}}}*/
 200 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 201 // ---------------------------------------------------------------------
 202 /* This will hold the string in and fill the buffer with it as it empties */
 203 bool CircleBuf::Read(string Data)
 204 {
 205    OutQueue += Data;
 206    FillOut();
 207    return true;
 208 }
 209                                                                         /*}}}*/
 210 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 211 // ---------------------------------------------------------------------
 212 /* */
 213 void CircleBuf::FillOut()
 214 {
 215    if (OutQueue.empty() == true)
 216       return;
 217    while (1)
 218    {
 219       // Woops, buffer is full
 220       if (InP - OutP == Size)
 221          return;
 222
 223       // Write the buffer segment
 224       unsigned long Sz = LeftRead();
 225       if (OutQueue.length() - StrPos < Sz)
 226          Sz = OutQueue.length() - StrPos;
 227       memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
 228
 229       // Advance
 230       StrPos += Sz;
 231       InP += Sz;
 232       if (OutQueue.length() == StrPos)
 233       {
 234          StrPos = 0;
 235          OutQueue = "";
 236          return;
 237       }
 238    }
 239 }
 240                                                                         /*}}}*/
 241 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 242 // ---------------------------------------------------------------------
 243 /* This empties the buffer into the FD. */
 244 bool CircleBuf::Write(int Fd)
 245 {
 246    while (1)
 247    {
 248       FillOut();
 249
 250       // Woops, buffer is empty
 251       if (OutP == InP)
 252          return true;
 253
 254       if (OutP == MaxGet)
 255          return true;
 256
 257       // Write the buffer segment
 258       int Res;
 259       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 260
 261       if (Res == 0)
 262          return false;
 263       if (Res < 0)
 264       {
 265          if (errno == EAGAIN)
 266             return true;
 267
 268          return false;
 269       }
 270
 271       if (Hash != 0)
 272          Hash->Add(Buf + (OutP%Size),Res);
 273
 274       OutP += Res;
 275    }
 276 }
 277                                                                         /*}}}*/
 278 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 279 // ---------------------------------------------------------------------
 280 /* This copies till the first empty line */
 281 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 282 {
 283    // We cheat and assume it is unneeded to have more than one buffer load
 284    for (unsigned long I = OutP; I < InP; I++)
 285    {
 286       if (Buf[I%Size] != '\n')
 287          continue;
 288       ++I;
 289
 290       if (Single == false)
 291       {
 292          if (I < InP  && Buf[I%Size] == '\r')
 293             ++I;
 294          if (I >= InP || Buf[I%Size] != '\n')
 295             continue;
 296          ++I;
 297       }
 298
 299       Data = "";
 300       while (OutP < I)
 301       {
 302          unsigned long Sz = LeftWrite();
 303          if (Sz == 0)
 304             return false;
 305          if (I - OutP < Sz)
 306             Sz = I - OutP;
 307          Data += string((char *)(Buf + (OutP%Size)),Sz);
 308          OutP += Sz;
 309       }
 310       return true;
 311    }
 312    return false;
 313 }
 314                                                                         /*}}}*/
 315 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 316 // ---------------------------------------------------------------------
 317 /* */
 318 void CircleBuf::Stats()
 319 {
 320    if (InP == 0)
 321       return;
 322
 323    struct timeval Stop;
 324    gettimeofday(&Stop,0);
 325 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 326              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 327    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 328 }
 329                                                                         /*}}}*/
 330
 331 // ServerState::ServerState - Constructor                               /*{{{*/
 332 // ---------------------------------------------------------------------
 333 /* */
 334 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 335                         In(64*1024), Out(4*1024),
 336                         ServerName(Srv)
 337 {
 338    Reset();
 339 }
 340                                                                         /*}}}*/
 341 // ServerState::Open - Open a connection to the server                  /*{{{*/
 342 // ---------------------------------------------------------------------
 343 /* This opens a connection to the server. */
 344 bool ServerState::Open()
 345 {
 346    // Use the already open connection if possible.
 347    if (ServerFd != -1)
 348       return true;
 349
 350    Close();
 351    In.Reset();
 352    Out.Reset();
 353    Persistent = true;
 354
 355    // Determine the proxy setting
 356    if (getenv("http_proxy") == 0)
 357    {
 358       string DefProxy = _config->Find("Acquire::http::Proxy");
 359       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 360       if (SpecificProxy.empty() == false)
 361       {
 362          if (SpecificProxy == "DIRECT")
 363             Proxy = "";
 364          else
 365             Proxy = SpecificProxy;
 366       }
 367       else
 368          Proxy = DefProxy;
 369    }
 370    else
 371       Proxy = getenv("http_proxy");
 372
 373    // Parse no_proxy, a , separated list of domains
 374    if (getenv("no_proxy") != 0)
 375    {
 376       if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
 377          Proxy = "";
 378    }
 379
 380    // Determine what host and port to use based on the proxy settings
 381    int Port = 0;
 382    string Host;
 383    if (Proxy.empty() == true || Proxy.Host.empty() == true)
 384    {
 385       if (ServerName.Port != 0)
 386          Port = ServerName.Port;
 387       Host = ServerName.Host;
 388    }
 389    else
 390    {
 391       if (Proxy.Port != 0)
 392          Port = Proxy.Port;
 393       Host = Proxy.Host;
 394    }
 395
 396    // Connect to the remote server
 397    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 398       return false;
 399
 400    return true;
 401 }
 402                                                                         /*}}}*/
 403 // ServerState::Close - Close a connection to the server                /*{{{*/
 404 // ---------------------------------------------------------------------
 405 /* */
 406 bool ServerState::Close()
 407 {
 408    close(ServerFd);
 409    ServerFd = -1;
 410    return true;
 411 }
 412                                                                         /*}}}*/
 413 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 414 // ---------------------------------------------------------------------
 415 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 416    parse error occured */
 417 int ServerState::RunHeaders()
 418 {
 419    State = Header;
 420
 421    Owner->Status(_("Waiting for headers"));
 422
 423    Major = 0;
 424    Minor = 0;
 425    Result = 0;
 426    Size = 0;
 427    StartPos = 0;
 428    Encoding = Closes;
 429    HaveContent = false;
 430    time(&Date);
 431
 432    do
 433    {
 434       string Data;
 435       if (In.WriteTillEl(Data) == false)
 436          continue;
 437
 438       if (Debug == true)
 439          clog << Data;
 440
 441       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 442       {
 443          string::const_iterator J = I;
 444          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 445          if (HeaderLine(string(I,J)) == false)
 446             return 2;
 447          I = J;
 448       }
 449
 450       // 100 Continue is a Nop...
 451       if (Result == 100)
 452          continue;
 453
 454       // Tidy up the connection persistance state.
 455       if (Encoding == Closes && HaveContent == true)
 456          Persistent = false;
 457
 458       return 0;
 459    }
 460    while (Owner->Go(false,this) == true);
 461
 462    return 1;
 463 }
 464                                                                         /*}}}*/
 465 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 466 // ---------------------------------------------------------------------
 467 /* */
 468 bool ServerState::RunData()
 469 {
 470    State = Data;
 471
 472    // Chunked transfer encoding is fun..
 473    if (Encoding == Chunked)
 474    {
 475       while (1)
 476       {
 477          // Grab the block size
 478          bool Last = true;
 479          string Data;
 480          In.Limit(-1);
 481          do
 482          {
 483             if (In.WriteTillEl(Data,true) == true)
 484                break;
 485          }
 486          while ((Last = Owner->Go(false,this)) == true);
 487
 488          if (Last == false)
 489             return false;
 490
 491          // See if we are done
 492          unsigned long Len = strtol(Data.c_str(),0,16);
 493          if (Len == 0)
 494          {
 495             In.Limit(-1);
 496
 497             // We have to remove the entity trailer
 498             Last = true;
 499             do
 500             {
 501                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 502                   break;
 503             }
 504             while ((Last = Owner->Go(false,this)) == true);
 505             if (Last == false)
 506                return false;
 507             return !_error->PendingError();
 508          }
 509
 510          // Transfer the block
 511          In.Limit(Len);
 512          while (Owner->Go(true,this) == true)
 513             if (In.IsLimit() == true)
 514                break;
 515
 516          // Error
 517          if (In.IsLimit() == false)
 518             return false;
 519
 520          // The server sends an extra new line before the next block specifier..
 521          In.Limit(-1);
 522          Last = true;
 523          do
 524          {
 525             if (In.WriteTillEl(Data,true) == true)
 526                break;
 527          }
 528          while ((Last = Owner->Go(false,this)) == true);
 529          if (Last == false)
 530             return false;
 531       }
 532    }
 533    else
 534    {
 535       /* Closes encoding is used when the server did not specify a size, the
 536          loss of the connection means we are done */
 537       if (Encoding == Closes)
 538          In.Limit(-1);
 539       else
 540          In.Limit(Size - StartPos);
 541
 542       // Just transfer the whole block.
 543       do
 544       {
 545          if (In.IsLimit() == false)
 546             continue;
 547
 548          In.Limit(-1);
 549          return !_error->PendingError();
 550       }
 551       while (Owner->Go(true,this) == true);
 552    }
 553
 554    return Owner->Flush(this) && !_error->PendingError();
 555 }
 556                                                                         /*}}}*/
 557 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 558 // ---------------------------------------------------------------------
 559 /* */
 560 bool ServerState::HeaderLine(string Line)
 561 {
 562    if (Line.empty() == true)
 563       return true;
 564
 565    // The http server might be trying to do something evil.
 566    if (Line.length() >= MAXLEN)
 567       return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
 568
 569    string::size_type Pos = Line.find(' ');
 570    if (Pos == string::npos || Pos+1 > Line.length())
 571    {
 572       // Blah, some servers use "connection:closes", evil.
 573       Pos = Line.find(':');
 574       if (Pos == string::npos || Pos + 2 > Line.length())
 575          return _error->Error(_("Bad header line"));
 576       Pos++;
 577    }
 578
 579    // Parse off any trailing spaces between the : and the next word.
 580    string::size_type Pos2 = Pos;
 581    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 582       Pos2++;
 583
 584    string Tag = string(Line,0,Pos);
 585    string Val = string(Line,Pos2);
 586
 587    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 588    {
 589       // Evil servers return no version
 590       if (Line[4] == '/')
 591       {
 592          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 593                     &Result,Code) != 4)
 594             return _error->Error(_("The HTTP server sent an invalid reply header"));
 595       }
 596       else
 597       {
 598          Major = 0;
 599          Minor = 9;
 600          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 601             return _error->Error(_("The HTTP server sent an invalid reply header"));
 602       }
 603
 604       /* Check the HTTP response header to get the default persistance
 605          state. */
 606       if (Major < 1)
 607          Persistent = false;
 608       else
 609       {
 610          if (Major == 1 && Minor <= 0)
 611             Persistent = false;
 612          else
 613             Persistent = true;
 614       }
 615
 616       return true;
 617    }
 618
 619    if (stringcasecmp(Tag,"Content-Length:") == 0)
 620    {
 621       if (Encoding == Closes)
 622          Encoding = Stream;
 623       HaveContent = true;
 624
 625       // The length is already set from the Content-Range header
 626       if (StartPos != 0)
 627          return true;
 628
 629       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 630          return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
 631       return true;
 632    }
 633
 634    if (stringcasecmp(Tag,"Content-Type:") == 0)
 635    {
 636       HaveContent = true;
 637       return true;
 638    }
 639
 640    if (stringcasecmp(Tag,"Content-Range:") == 0)
 641    {
 642       HaveContent = true;
 643
 644       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 645          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 646       if ((unsigned)StartPos > Size)
 647          return _error->Error(_("This HTTP server has broken range support"));
 648       return true;
 649    }
 650
 651    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 652    {
 653       HaveContent = true;
 654       if (stringcasecmp(Val,"chunked") == 0)
 655          Encoding = Chunked;
 656       return true;
 657    }
 658
 659    if (stringcasecmp(Tag,"Connection:") == 0)
 660    {
 661       if (stringcasecmp(Val,"close") == 0)
 662          Persistent = false;
 663       if (stringcasecmp(Val,"keep-alive") == 0)
 664          Persistent = true;
 665       return true;
 666    }
 667
 668    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 669    {
 670       if (StrToTime(Val,Date) == false)
 671          return _error->Error(_("Unknown date format"));
 672       return true;
 673    }
 674
 675    return true;
 676 }
 677                                                                         /*}}}*/
 678
 679 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 680 // ---------------------------------------------------------------------
 681 /* This places the http request in the outbound buffer */
 682 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 683 {
 684    URI Uri = Itm->Uri;
 685
 686    // The HTTP server expects a hostname with a trailing :port
 687    char Buf[1000];
 688    string ProperHost = Uri.Host;
 689    if (Uri.Port != 0)
 690    {
 691       sprintf(Buf,":%u",Uri.Port);
 692       ProperHost += Buf;
 693    }
 694
 695    // Just in case.
 696    if (Itm->Uri.length() >= sizeof(Buf))
 697        abort();
 698
 699    /* Build the request. We include a keep-alive header only for non-proxy
 700       requests. This is to tweak old http/1.0 servers that do support keep-alive
 701       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 702       will glitch HTTP/1.0 proxies because they do not filter it out and
 703       pass it on, HTTP/1.1 says the connection should default to keep alive
 704       and we expect the proxy to do this */
 705    if (Proxy.empty() == true || Proxy.Host.empty())
 706       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 707               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 708    else
 709    {
 710       /* Generate a cache control header if necessary. We place a max
 711          cache age on index files, optionally set a no-cache directive
 712          and a no-store directive for archives. */
 713       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 714               Itm->Uri.c_str(),ProperHost.c_str());
 715       // only generate a cache control header if we actually want to
 716       // use a cache
 717       if (_config->FindB("Acquire::http::No-Cache",false) == false)
 718       {
 719          if (Itm->IndexFile == true)
 720             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 721                     _config->FindI("Acquire::http::Max-Age",0));
 722          else
 723          {
 724             if (_config->FindB("Acquire::http::No-Store",false) == true)
 725                strcat(Buf,"Cache-Control: no-store\r\n");
 726          }
 727       }
 728    }
 729    // generate a no-cache header if needed
 730    if (_config->FindB("Acquire::http::No-Cache",false) == true)
 731       strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 732
 733
 734    string Req = Buf;
 735
 736    // Check for a partial file
 737    struct stat SBuf;
 738    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 739    {
 740       // In this case we send an if-range query with a range header
 741       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
 742               TimeRFC1123(SBuf.st_mtime).c_str());
 743       Req += Buf;
 744    }
 745    else
 746    {
 747       if (Itm->LastModified != 0)
 748       {
 749          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 750          Req += Buf;
 751       }
 752    }
 753
 754    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 755       Req += string("Proxy-Authorization: Basic ") +
 756           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 757
 758    if (Uri.User.empty() == false || Uri.Password.empty() == false)
 759       Req += string("Authorization: Basic ") +
 760           Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
 761
 762    Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
 763
 764    if (Debug == true)
 765       cerr << Req << endl;
 766
 767    Out.Read(Req);
 768 }
 769                                                                         /*}}}*/
 770 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 771 // ---------------------------------------------------------------------
 772 /* This runs the select loop over the server FDs, Output file FDs and
 773    stdin. */
 774 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 775 {
 776    // Server has closed the connection
 777    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 778                                ToFile == false))
 779       return false;
 780
 781    fd_set rfds,wfds;
 782    FD_ZERO(&rfds);
 783    FD_ZERO(&wfds);
 784
 785    /* Add the server. We only send more requests if the connection will
 786       be persisting */
 787    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
 788        && Srv->Persistent == true)
 789       FD_SET(Srv->ServerFd,&wfds);
 790    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 791       FD_SET(Srv->ServerFd,&rfds);
 792
 793    // Add the file
 794    int FileFD = -1;
 795    if (File != 0)
 796       FileFD = File->Fd();
 797
 798    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 799       FD_SET(FileFD,&wfds);
 800
 801    // Add stdin
 802    FD_SET(STDIN_FILENO,&rfds);
 803
 804    // Figure out the max fd
 805    int MaxFd = FileFD;
 806    if (MaxFd < Srv->ServerFd)
 807       MaxFd = Srv->ServerFd;
 808
 809    // Select
 810    struct timeval tv;
 811    tv.tv_sec = TimeOut;
 812    tv.tv_usec = 0;
 813    int Res = 0;
 814    if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
 815    {
 816       if (errno == EINTR)
 817          return true;
 818       return _error->Errno("select",_("Select failed"));
 819    }
 820
 821    if (Res == 0)
 822    {
 823       _error->Error(_("Connection timed out"));
 824       return ServerDie(Srv);
 825    }
 826
 827    // Handle server IO
 828    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 829    {
 830       errno = 0;
 831       if (Srv->In.Read(Srv->ServerFd) == false)
 832          return ServerDie(Srv);
 833    }
 834
 835    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 836    {
 837       errno = 0;
 838       if (Srv->Out.Write(Srv->ServerFd) == false)
 839          return ServerDie(Srv);
 840    }
 841
 842    // Send data to the file
 843    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 844    {
 845       if (Srv->In.Write(FileFD) == false)
 846          return _error->Errno("write",_("Error writing to output file"));
 847    }
 848
 849    // Handle commands from APT
 850    if (FD_ISSET(STDIN_FILENO,&rfds))
 851    {
 852       if (Run(true) != -1)
 853          exit(100);
 854    }
 855
 856    return true;
 857 }
 858                                                                         /*}}}*/
 859 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 860 // ---------------------------------------------------------------------
 861 /* This takes the current input buffer from the Server FD and writes it
 862    into the file */
 863 bool HttpMethod::Flush(ServerState *Srv)
 864 {
 865    if (File != 0)
 866    {
 867       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 868       // can't be set
 869       if (File->Name() != "/dev/null")
 870          SetNonBlock(File->Fd(),false);
 871       if (Srv->In.WriteSpace() == false)
 872          return true;
 873
 874       while (Srv->In.WriteSpace() == true)
 875       {
 876          if (Srv->In.Write(File->Fd()) == false)
 877             return _error->Errno("write",_("Error writing to file"));
 878          if (Srv->In.IsLimit() == true)
 879             return true;
 880       }
 881
 882       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 883          return true;
 884    }
 885    return false;
 886 }
 887                                                                         /*}}}*/
 888 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 889 // ---------------------------------------------------------------------
 890 /* */
 891 bool HttpMethod::ServerDie(ServerState *Srv)
 892 {
 893    unsigned int LErrno = errno;
 894
 895    // Dump the buffer to the file
 896    if (Srv->State == ServerState::Data)
 897    {
 898       // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
 899       // can't be set
 900       if (File->Name() != "/dev/null")
 901          SetNonBlock(File->Fd(),false);
 902       while (Srv->In.WriteSpace() == true)
 903       {
 904          if (Srv->In.Write(File->Fd()) == false)
 905             return _error->Errno("write",_("Error writing to the file"));
 906
 907          // Done
 908          if (Srv->In.IsLimit() == true)
 909             return true;
 910       }
 911    }
 912
 913    // See if this is because the server finished the data stream
 914    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 915        Srv->Encoding != ServerState::Closes)
 916    {
 917       Srv->Close();
 918       if (LErrno == 0)
 919          return _error->Error(_("Error reading from server. Remote end closed connection"));
 920       errno = LErrno;
 921       return _error->Errno("read",_("Error reading from server"));
 922    }
 923    else
 924    {
 925       Srv->In.Limit(-1);
 926
 927       // Nothing left in the buffer
 928       if (Srv->In.WriteSpace() == false)
 929          return false;
 930
 931       // We may have got multiple responses back in one packet..
 932       Srv->Close();
 933       return true;
 934    }
 935
 936    return false;
 937 }
 938                                                                         /*}}}*/
 939 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 940 // ---------------------------------------------------------------------
 941 /* We look at the header data we got back from the server and decide what
 942    to do. Returns
 943      0 - File is open,
 944      1 - IMS hit
 945      3 - Unrecoverable error
 946      4 - Error with error content page
 947      5 - Unrecoverable non-server error (close the connection) */
 948 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 949 {
 950    // Not Modified
 951    if (Srv->Result == 304)
 952    {
 953       unlink(Queue->DestFile.c_str());
 954       Res.IMSHit = true;
 955       Res.LastModified = Queue->LastModified;
 956       return 1;
 957    }
 958
 959    /* We have a reply we dont handle. This should indicate a perm server
 960       failure */
 961    if (Srv->Result < 200 || Srv->Result >= 300)
 962    {
 963       _error->Error("%u %s",Srv->Result,Srv->Code);
 964       if (Srv->HaveContent == true)
 965          return 4;
 966       return 3;
 967    }
 968
 969    // This is some sort of 2xx 'data follows' reply
 970    Res.LastModified = Srv->Date;
 971    Res.Size = Srv->Size;
 972
 973    // Open the file
 974    delete File;
 975    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 976    if (_error->PendingError() == true)
 977       return 5;
 978
 979    FailFile = Queue->DestFile;
 980    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 981    FailFd = File->Fd();
 982    FailTime = Srv->Date;
 983
 984    // Set the expected size
 985    if (Srv->StartPos >= 0)
 986    {
 987       Res.ResumePoint = Srv->StartPos;
 988       ftruncate(File->Fd(),Srv->StartPos);
 989    }
 990
 991    // Set the start point
 992    lseek(File->Fd(),0,SEEK_END);
 993
 994    delete Srv->In.Hash;
 995    Srv->In.Hash = new Hashes;
 996
 997    // Fill the Hash if the file is non-empty (resume)
 998    if (Srv->StartPos > 0)
 999    {
1000       lseek(File->Fd(),0,SEEK_SET);
1001       if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
1002       {
1003          _error->Errno("read",_("Problem hashing file"));
1004          return 5;
1005       }
1006       lseek(File->Fd(),0,SEEK_END);
1007    }
1008
1009    SetNonBlock(File->Fd(),true);
1010    return 0;
1011 }
1012                                                                         /*}}}*/
1013 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
1014 // ---------------------------------------------------------------------
1015 /* This closes and timestamps the open file. This is neccessary to get
1016    resume behavoir on user abort */
1017 void HttpMethod::SigTerm(int)
1018 {
1019    if (FailFd == -1)
1020       _exit(100);
1021    close(FailFd);
1022
1023    // Timestamp
1024    struct utimbuf UBuf;
1025    UBuf.actime = FailTime;
1026    UBuf.modtime = FailTime;
1027    utime(FailFile.c_str(),&UBuf);
1028
1029    _exit(100);
1030 }
1031                                                                         /*}}}*/
1032 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
1033 // ---------------------------------------------------------------------
1034 /* This adds an item to the pipeline. We keep the pipeline at a fixed
1035    depth. */
1036 bool HttpMethod::Fetch(FetchItem *)
1037 {
1038    if (Server == 0)
1039       return true;
1040
1041    // Queue the requests
1042    int Depth = -1;
1043    bool Tail = false;
1044    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
1045         I = I->Next, Depth++)
1046    {
1047       // If pipelining is disabled, we only queue 1 request
1048       if (Server->Pipeline == false && Depth >= 0)
1049          break;
1050
1051       // Make sure we stick with the same server
1052       if (Server->Comp(I->Uri) == false)
1053          break;
1054       if (QueueBack == I)
1055          Tail = true;
1056       if (Tail == true)
1057       {
1058          QueueBack = I->Next;
1059          SendReq(I,Server->Out);
1060          continue;
1061       }
1062    }
1063
1064    return true;
1065 };
1066                                                                         /*}}}*/
1067 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
1068 // ---------------------------------------------------------------------
1069 /* We stash the desired pipeline depth */
1070 bool HttpMethod::Configuration(string Message)
1071 {
1072    if (pkgAcqMethod::Configuration(Message) == false)
1073       return false;
1074
1075    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1076    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1077                                   PipelineDepth);
1078    Debug = _config->FindB("Debug::Acquire::http",false);
1079
1080    return true;
1081 }
1082                                                                         /*}}}*/
1083 // HttpMethod::Loop - Main loop                                         /*{{{*/
1084 // ---------------------------------------------------------------------
1085 /* */
1086 int HttpMethod::Loop()
1087 {
1088    signal(SIGTERM,SigTerm);
1089    signal(SIGINT,SigTerm);
1090
1091    Server = 0;
1092
1093    int FailCounter = 0;
1094    while (1)
1095    {
1096       // We have no commands, wait for some to arrive
1097       if (Queue == 0)
1098       {
1099          if (WaitFd(STDIN_FILENO) == false)
1100             return 0;
1101       }
1102
1103       /* Run messages, we can accept 0 (no message) if we didn't
1104          do a WaitFd above.. Otherwise the FD is closed. */
1105       int Result = Run(true);
1106       if (Result != -1 && (Result != 0 || Queue == 0))
1107          return 100;
1108
1109       if (Queue == 0)
1110          continue;
1111
1112       CFStringEncoding se = kCFStringEncodingUTF8;
1113
1114       char *url = strdup(Queue->Uri.c_str());
1115     url:
1116       URI uri = Queue->Uri;
1117       std::string hs = uri.Host;
1118
1119       struct hostent *he = gethostbyname(hs.c_str());
1120       if (he == NULL || he->h_addr_list[0] == NULL) {
1121          _error->Error(hstrerror(h_errno));
1122          Fail(true);
1123          free(url);
1124       }
1125
1126       uri.Host = inet_ntoa(* (struct in_addr *) he->h_addr_list[0]);
1127
1128       std::string urs = uri;
1129
1130       CFStringRef sr = CFStringCreateWithCString(kCFAllocatorDefault, urs.c_str(), se);
1131       CFURLRef ur = CFURLCreateWithString(kCFAllocatorDefault, sr, NULL);
1132       CFRelease(sr);
1133       CFHTTPMessageRef hm = CFHTTPMessageCreateRequest(kCFAllocatorDefault, CFSTR("GET"), ur, kCFHTTPVersion1_1);
1134       CFRelease(ur);
1135
1136       struct stat SBuf;
1137       if (stat(Queue->DestFile.c_str(), &SBuf) >= 0 && SBuf.st_size > 0) {
1138          sr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("bytes=%li-"), (long) SBuf.st_size - 1);
1139          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Range"), sr);
1140          CFRelease(sr);
1141
1142          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
1143          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Range"), sr);
1144          CFRelease(sr);
1145       } else if (Queue->LastModified != 0) {
1146          sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
1147          CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Modified-Since"), sr);
1148          CFRelease(sr);
1149       }
1150
1151       CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.98"));
1152
1153       sr = CFStringCreateWithCString(kCFAllocatorDefault, hs.c_str(), se);
1154       CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Host"), sr);
1155       CFRelease(sr);
1156
1157       CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm);
1158       CFRelease(hm);
1159
1160       CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL);
1161       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr);
1162       CFRelease(dr);
1163
1164       //CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPShouldAutoredirect, kCFBooleanTrue);
1165       CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPAttemptPersistentConnection, kCFBooleanTrue);
1166
1167       FetchResult Res;
1168       CFIndex rd;
1169       UInt32 sc;
1170
1171       uint8_t data[10240];
1172       size_t offset = 0;
1173
1174       Status("Connecting to %s", hs.c_str());
1175
1176       if (!CFReadStreamOpen(rs)) {
1177          CfrsError("Open", rs);
1178          Fail(true);
1179          goto done;
1180       }
1181
1182       rd = CFReadStreamRead(rs, data, sizeof(data));
1183
1184       if (rd == -1) {
1185          CfrsError(uri.Host.c_str(), rs);
1186          Fail(true);
1187          goto done;
1188       }
1189
1190       Res.Filename = Queue->DestFile;
1191
1192       hm = (CFHTTPMessageRef) CFReadStreamCopyProperty(rs, kCFStreamPropertyHTTPResponseHeader);
1193       sc = CFHTTPMessageGetResponseStatusCode(hm);
1194
1195       if (sc == 301 || sc == 302) {
1196          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Location"));
1197          if (sr == NULL) {
1198             Fail();
1199             goto done_;
1200          } else {
1201             size_t ln = CFStringGetLength(sr) + 1;
1202             free(url);
1203             url = static_cast<char *>(malloc(ln));
1204
1205             if (!CFStringGetCString(sr, url, ln, se)) {
1206                Fail();
1207                goto done_;
1208             }
1209
1210             CFRelease(sr);
1211             goto url;
1212          }
1213       }
1214
1215       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Range"));
1216       if (sr != NULL) {
1217          size_t ln = CFStringGetLength(sr) + 1;
1218          char cr[ln];
1219
1220          if (!CFStringGetCString(sr, cr, ln, se)) {
1221             Fail();
1222             goto done_;
1223          }
1224
1225          CFRelease(sr);
1226
1227          if (sscanf(cr, "bytes %lu-%*u/%lu", &offset, &Res.Size) != 2) {
1228             _error->Error(_("The HTTP server sent an invalid Content-Range header"));
1229             Fail();
1230             goto done_;
1231          }
1232
1233          if (offset > Res.Size) {
1234             _error->Error(_("This HTTP server has broken range support"));
1235             Fail();
1236             goto done_;
1237          }
1238       } else {
1239          sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Length"));
1240          if (sr != NULL) {
1241             Res.Size = CFStringGetIntValue(sr);
1242             CFRelease(sr);
1243          }
1244       }
1245
1246       time(&Res.LastModified);
1247
1248       sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Last-Modified"));
1249       if (sr != NULL) {
1250          size_t ln = CFStringGetLength(sr) + 1;
1251          char cr[ln];
1252
1253          if (!CFStringGetCString(sr, cr, ln, se)) {
1254             Fail();
1255             goto done_;
1256          }
1257
1258          CFRelease(sr);
1259
1260          if (!StrToTime(cr, Res.LastModified)) {
1261             _error->Error(_("Unknown date format"));
1262             Fail();
1263             goto done_;
1264          }
1265       }
1266
1267       CFRelease(hm);
1268
1269       if (sc == 304) {
1270          unlink(Queue->DestFile.c_str());
1271          Res.IMSHit = true;
1272          Res.LastModified = Queue->LastModified;
1273          URIDone(Res);
1274       } else if (sc < 200 || sc >= 300)
1275          Fail();
1276       else {
1277          Hashes hash;
1278
1279          File = new FileFd(Queue->DestFile, FileFd::WriteAny);
1280          if (_error->PendingError() == true) {
1281             delete File;
1282             File = NULL;
1283             Fail();
1284             goto done;
1285          }
1286
1287          FailFile = Queue->DestFile;
1288          FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
1289          FailFd = File->Fd();
1290          FailTime = Res.LastModified;
1291
1292          Res.ResumePoint = offset;
1293          ftruncate(File->Fd(), offset);
1294
1295          if (offset != 0) {
1296             lseek(File->Fd(), 0, SEEK_SET);
1297             if (!hash.AddFD(File->Fd(), offset)) {
1298                _error->Errno("read", _("Problem hashing file"));
1299                delete File;
1300                File = NULL;
1301                Fail();
1302                goto done;
1303             }
1304          }
1305
1306          lseek(File->Fd(), 0, SEEK_END);
1307
1308          URIStart(Res);
1309
1310          read: if (rd == -1) {
1311             CfrsError("rd", rs);
1312             Fail(true);
1313          } else if (rd == 0) {
1314             if (Res.Size == 0)
1315                Res.Size = File->Size();
1316
1317             struct utimbuf UBuf;
1318             time(&UBuf.actime);
1319             UBuf.actime = Res.LastModified;
1320             UBuf.modtime = Res.LastModified;
1321             utime(Queue->DestFile.c_str(), &UBuf);
1322
1323             Res.TakeHashes(hash);
1324             URIDone(Res);
1325          } else {
1326             hash.Add(data, rd);
1327
1328             uint8_t *dt = data;
1329             while (rd != 0) {
1330                int sz = write(File->Fd(), dt, rd);
1331
1332                if (sz == -1) {
1333                    delete File;
1334                    File = NULL;
1335                    Fail();
1336                    goto done;
1337                }
1338
1339                dt += sz;
1340                rd -= sz;
1341             }
1342
1343             rd = CFReadStreamRead(rs, data, sizeof(data));
1344             goto read;
1345          }
1346       }
1347
1348      goto done;
1349     done_:
1350       CFRelease(hm);
1351     done:
1352       CFReadStreamClose(rs);
1353       CFRelease(rs);
1354       free(url);
1355
1356       FailCounter = 0;
1357    }
1358
1359    return 0;
1360 }
1361                                                                         /*}}}*/
1362
1363 int main()
1364 {
1365    setlocale(LC_ALL, "");
1366
1367    HttpMethod Mth;
1368
1369    return Mth.Loop();
1370 }
1371
1372