methods/http.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 // $Id: http.cc,v 1.37 1999/08/03 06:08:23 jgg Exp $
   4 /* ######################################################################
   5
   6    HTTP Aquire Method - This is the HTTP aquire method for APT.
   7
   8    It uses HTTP/1.1 and many of the fancy options there-in, such as
   9    pipelining, range, if-range and so on. It accepts on the command line
  10    a list of url destination pairs and writes to stdout the status of the
  11    operation as defined in the APT method spec.
  12
  13    It is based on a doubly buffered select loop. All the requests are
  14    fed into a single output buffer that is constantly fed out the
  15    socket. This provides ideal pipelining as in many cases all of the
  16    requests will fit into a single packet. The input socket is buffered
  17    the same way and fed into the fd for the file.
  18
  19    This double buffering provides fairly substantial transfer rates,
  20    compared to wget the http method is about 4% faster. Most importantly,
  21    when HTTP is compared with FTP as a protocol the speed difference is
  22    huge. In tests over the internet from two sites to llug (via ATM) this
  23    program got 230k/s sustained http transfer rates. FTP on the other
  24    hand topped out at 170k/s. That combined with the time to setup the
  25    FTP connection makes HTTP a vastly superior protocol.
  26
  27    ##################################################################### */
  28                                                                         /*}}}*/
  29 // Include Files                                                        /*{{{*/
  30 #include <apt-pkg/fileutl.h>
  31 #include <apt-pkg/acquire-method.h>
  32 #include <apt-pkg/error.h>
  33 #include <apt-pkg/md5.h>
  34
  35 #include <sys/stat.h>
  36 #include <sys/time.h>
  37 #include <utime.h>
  38 #include <unistd.h>
  39 #include <signal.h>
  40 #include <stdio.h>
  41 #include <errno.h>
  42
  43 // Internet stuff
  44 #include <netdb.h>
  45
  46 #include "connect.h"
  47 #include "rfc2553emu.h"
  48 #include "http.h"
  49
  50                                                                         /*}}}*/
  51
  52 string HttpMethod::FailFile;
  53 int HttpMethod::FailFd = -1;
  54 time_t HttpMethod::FailTime = 0;
  55 unsigned long PipelineDepth = 10;
  56 unsigned long TimeOut = 120;
  57
  58 // CircleBuf::CircleBuf - Circular input buffer                         /*{{{*/
  59 // ---------------------------------------------------------------------
  60 /* */
  61 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), MD5(0)
  62 {
  63    Buf = new unsigned char[Size];
  64    Reset();
  65 }
  66                                                                         /*}}}*/
  67 // CircleBuf::Reset - Reset to the default state                        /*{{{*/
  68 // ---------------------------------------------------------------------
  69 /* */
  70 void CircleBuf::Reset()
  71 {
  72    InP = 0;
  73    OutP = 0;
  74    StrPos = 0;
  75    MaxGet = (unsigned int)-1;
  76    OutQueue = string();
  77    if (MD5 != 0)
  78    {
  79       delete MD5;
  80       MD5 = new MD5Summation;
  81    }
  82 };
  83                                                                         /*}}}*/
  84 // CircleBuf::Read - Read from a FD into the circular buffer            /*{{{*/
  85 // ---------------------------------------------------------------------
  86 /* This fills up the buffer with as much data as is in the FD, assuming it
  87    is non-blocking.. */
  88 bool CircleBuf::Read(int Fd)
  89 {
  90    while (1)
  91    {
  92       // Woops, buffer is full
  93       if (InP - OutP == Size)
  94          return true;
  95
  96       // Write the buffer segment
  97       int Res;
  98       Res = read(Fd,Buf + (InP%Size),LeftRead());
  99
 100       if (Res == 0)
 101          return false;
 102       if (Res < 0)
 103       {
 104          if (errno == EAGAIN)
 105             return true;
 106          return false;
 107       }
 108
 109       if (InP == 0)
 110          gettimeofday(&Start,0);
 111       InP += Res;
 112    }
 113 }
 114                                                                         /*}}}*/
 115 // CircleBuf::Read - Put the string into the buffer                     /*{{{*/
 116 // ---------------------------------------------------------------------
 117 /* This will hold the string in and fill the buffer with it as it empties */
 118 bool CircleBuf::Read(string Data)
 119 {
 120    OutQueue += Data;
 121    FillOut();
 122    return true;
 123 }
 124                                                                         /*}}}*/
 125 // CircleBuf::FillOut - Fill the buffer from the output queue           /*{{{*/
 126 // ---------------------------------------------------------------------
 127 /* */
 128 void CircleBuf::FillOut()
 129 {
 130    if (OutQueue.empty() == true)
 131       return;
 132    while (1)
 133    {
 134       // Woops, buffer is full
 135       if (InP - OutP == Size)
 136          return;
 137
 138       // Write the buffer segment
 139       unsigned long Sz = LeftRead();
 140       if (OutQueue.length() - StrPos < Sz)
 141          Sz = OutQueue.length() - StrPos;
 142       memcpy(Buf + (InP%Size),OutQueue.begin() + StrPos,Sz);
 143
 144       // Advance
 145       StrPos += Sz;
 146       InP += Sz;
 147       if (OutQueue.length() == StrPos)
 148       {
 149          StrPos = 0;
 150          OutQueue = "";
 151          return;
 152       }
 153    }
 154 }
 155                                                                         /*}}}*/
 156 // CircleBuf::Write - Write from the buffer into a FD                   /*{{{*/
 157 // ---------------------------------------------------------------------
 158 /* This empties the buffer into the FD. */
 159 bool CircleBuf::Write(int Fd)
 160 {
 161    while (1)
 162    {
 163       FillOut();
 164
 165       // Woops, buffer is empty
 166       if (OutP == InP)
 167          return true;
 168
 169       if (OutP == MaxGet)
 170          return true;
 171
 172       // Write the buffer segment
 173       int Res;
 174       Res = write(Fd,Buf + (OutP%Size),LeftWrite());
 175
 176       if (Res == 0)
 177          return false;
 178       if (Res < 0)
 179       {
 180          if (errno == EAGAIN)
 181             return true;
 182
 183          return false;
 184       }
 185
 186       if (MD5 != 0)
 187          MD5->Add(Buf + (OutP%Size),Res);
 188
 189       OutP += Res;
 190    }
 191 }
 192                                                                         /*}}}*/
 193 // CircleBuf::WriteTillEl - Write from the buffer to a string           /*{{{*/
 194 // ---------------------------------------------------------------------
 195 /* This copies till the first empty line */
 196 bool CircleBuf::WriteTillEl(string &Data,bool Single)
 197 {
 198    // We cheat and assume it is unneeded to have more than one buffer load
 199    for (unsigned long I = OutP; I < InP; I++)
 200    {
 201       if (Buf[I%Size] != '\n')
 202          continue;
 203       for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 204
 205       if (Single == false)
 206       {
 207          if (Buf[I%Size] != '\n')
 208             continue;
 209          for (I++; I < InP && Buf[I%Size] == '\r'; I++);
 210       }
 211
 212       if (I > InP)
 213          I = InP;
 214
 215       Data = "";
 216       while (OutP < I)
 217       {
 218          unsigned long Sz = LeftWrite();
 219          if (Sz == 0)
 220             return false;
 221          if (I - OutP < LeftWrite())
 222             Sz = I - OutP;
 223          Data += string((char *)(Buf + (OutP%Size)),Sz);
 224          OutP += Sz;
 225       }
 226       return true;
 227    }
 228    return false;
 229 }
 230                                                                         /*}}}*/
 231 // CircleBuf::Stats - Print out stats information                       /*{{{*/
 232 // ---------------------------------------------------------------------
 233 /* */
 234 void CircleBuf::Stats()
 235 {
 236    if (InP == 0)
 237       return;
 238
 239    struct timeval Stop;
 240    gettimeofday(&Stop,0);
 241 /*   float Diff = Stop.tv_sec - Start.tv_sec +
 242              (float)(Stop.tv_usec - Start.tv_usec)/1000000;
 243    clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
 244 }
 245                                                                         /*}}}*/
 246
 247 // ServerState::ServerState - Constructor                               /*{{{*/
 248 // ---------------------------------------------------------------------
 249 /* */
 250 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
 251                         In(64*1024), Out(4*1024),
 252                         ServerName(Srv)
 253 {
 254    Reset();
 255 }
 256                                                                         /*}}}*/
 257 // ServerState::Open - Open a connection to the server                  /*{{{*/
 258 // ---------------------------------------------------------------------
 259 /* This opens a connection to the server. */
 260 string LastHost;
 261 int LastPort = 0;
 262 struct addrinfo *LastHostAddr = 0;
 263 bool ServerState::Open()
 264 {
 265    // Use the already open connection if possible.
 266    if (ServerFd != -1)
 267       return true;
 268
 269    Close();
 270    In.Reset();
 271    Out.Reset();
 272
 273    // Determine the proxy setting
 274    if (getenv("http_proxy") == 0)
 275    {
 276       string DefProxy = _config->Find("Acquire::http::Proxy");
 277       string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
 278       if (SpecificProxy.empty() == false)
 279       {
 280          if (SpecificProxy == "DIRECT")
 281             Proxy = "";
 282          else
 283             Proxy = SpecificProxy;
 284       }
 285       else
 286          Proxy = DefProxy;
 287    }
 288    else
 289       Proxy = getenv("http_proxy");
 290
 291    // Determine what host and port to use based on the proxy settings
 292    int Port = 0;
 293    string Host;
 294    if (Proxy.empty() == true)
 295    {
 296       if (ServerName.Port != 0)
 297          Port = ServerName.Port;
 298       Host = ServerName.Host;
 299    }
 300    else
 301    {
 302       if (Proxy.Port != 0)
 303          Port = Proxy.Port;
 304       Host = Proxy.Host;
 305    }
 306
 307    // Connect to the remote server
 308    if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
 309       return false;
 310
 311    return true;
 312 }
 313                                                                         /*}}}*/
 314 // ServerState::Close - Close a connection to the server                /*{{{*/
 315 // ---------------------------------------------------------------------
 316 /* */
 317 bool ServerState::Close()
 318 {
 319    close(ServerFd);
 320    ServerFd = -1;
 321    return true;
 322 }
 323                                                                         /*}}}*/
 324 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
 325 // ---------------------------------------------------------------------
 326 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
 327    parse error occured */
 328 int ServerState::RunHeaders()
 329 {
 330    State = Header;
 331
 332    Owner->Status("Waiting for file");
 333
 334    Major = 0;
 335    Minor = 0;
 336    Result = 0;
 337    Size = 0;
 338    StartPos = 0;
 339    Encoding = Closes;
 340    HaveContent = false;
 341    time(&Date);
 342
 343    do
 344    {
 345       string Data;
 346       if (In.WriteTillEl(Data) == false)
 347          continue;
 348
 349       for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
 350       {
 351          string::const_iterator J = I;
 352          for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
 353          if (HeaderLine(string(I,J-I)) == false)
 354             return 2;
 355          I = J;
 356       }
 357       return 0;
 358    }
 359    while (Owner->Go(false,this) == true);
 360
 361    return 1;
 362 }
 363                                                                         /*}}}*/
 364 // ServerState::RunData - Transfer the data from the socket             /*{{{*/
 365 // ---------------------------------------------------------------------
 366 /* */
 367 bool ServerState::RunData()
 368 {
 369    State = Data;
 370
 371    // Chunked transfer encoding is fun..
 372    if (Encoding == Chunked)
 373    {
 374       while (1)
 375       {
 376          // Grab the block size
 377          bool Last = true;
 378          string Data;
 379          In.Limit(-1);
 380          do
 381          {
 382             if (In.WriteTillEl(Data,true) == true)
 383                break;
 384          }
 385          while ((Last = Owner->Go(false,this)) == true);
 386
 387          if (Last == false)
 388             return false;
 389
 390          // See if we are done
 391          unsigned long Len = strtol(Data.c_str(),0,16);
 392          if (Len == 0)
 393          {
 394             In.Limit(-1);
 395
 396             // We have to remove the entity trailer
 397             Last = true;
 398             do
 399             {
 400                if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
 401                   break;
 402             }
 403             while ((Last = Owner->Go(false,this)) == true);
 404             if (Last == false)
 405                return false;
 406             return !_error->PendingError();
 407          }
 408
 409          // Transfer the block
 410          In.Limit(Len);
 411          while (Owner->Go(true,this) == true)
 412             if (In.IsLimit() == true)
 413                break;
 414
 415          // Error
 416          if (In.IsLimit() == false)
 417             return false;
 418
 419          // The server sends an extra new line before the next block specifier..
 420          In.Limit(-1);
 421          Last = true;
 422          do
 423          {
 424             if (In.WriteTillEl(Data,true) == true)
 425                break;
 426          }
 427          while ((Last = Owner->Go(false,this)) == true);
 428          if (Last == false)
 429             return false;
 430       }
 431    }
 432    else
 433    {
 434       /* Closes encoding is used when the server did not specify a size, the
 435          loss of the connection means we are done */
 436       if (Encoding == Closes)
 437          In.Limit(-1);
 438       else
 439          In.Limit(Size - StartPos);
 440
 441       // Just transfer the whole block.
 442       do
 443       {
 444          if (In.IsLimit() == false)
 445             continue;
 446
 447          In.Limit(-1);
 448          return !_error->PendingError();
 449       }
 450       while (Owner->Go(true,this) == true);
 451    }
 452
 453    return Owner->Flush(this) && !_error->PendingError();
 454 }
 455                                                                         /*}}}*/
 456 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 457 // ---------------------------------------------------------------------
 458 /* */
 459 bool ServerState::HeaderLine(string Line)
 460 {
 461    if (Line.empty() == true)
 462       return true;
 463
 464    // The http server might be trying to do something evil.
 465    if (Line.length() >= MAXLEN)
 466       return _error->Error("Got a single header line over %u chars",MAXLEN);
 467
 468    string::size_type Pos = Line.find(' ');
 469    if (Pos == string::npos || Pos+1 > Line.length())
 470    {
 471       // Blah, some servers use "connection:closes", evil.
 472       Pos = Line.find(':');
 473       if (Pos == string::npos || Pos + 2 > Line.length())
 474          return _error->Error("Bad header line");
 475       Pos++;
 476    }
 477
 478    // Parse off any trailing spaces between the : and the next word.
 479    string::size_type Pos2 = Pos;
 480    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 481       Pos2++;
 482
 483    string Tag = string(Line,0,Pos);
 484    string Val = string(Line,Pos2);
 485
 486    if (stringcasecmp(Tag.begin(),Tag.begin()+4,"HTTP") == 0)
 487    {
 488       // Evil servers return no version
 489       if (Line[4] == '/')
 490       {
 491          if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
 492                     &Result,Code) != 4)
 493             return _error->Error("The http server sent an invalid reply header");
 494       }
 495       else
 496       {
 497          Major = 0;
 498          Minor = 9;
 499          if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
 500             return _error->Error("The http server sent an invalid reply header");
 501       }
 502
 503       return true;
 504    }
 505
 506    if (stringcasecmp(Tag,"Content-Length:") == 0)
 507    {
 508       if (Encoding == Closes)
 509          Encoding = Stream;
 510       HaveContent = true;
 511
 512       // The length is already set from the Content-Range header
 513       if (StartPos != 0)
 514          return true;
 515
 516       if (sscanf(Val.c_str(),"%lu",&Size) != 1)
 517          return _error->Error("The http server sent an invalid Content-Length header");
 518       return true;
 519    }
 520
 521    if (stringcasecmp(Tag,"Content-Type:") == 0)
 522    {
 523       HaveContent = true;
 524       return true;
 525    }
 526
 527    if (stringcasecmp(Tag,"Content-Range:") == 0)
 528    {
 529       HaveContent = true;
 530
 531       if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
 532          return _error->Error("The http server sent an invalid Content-Range header");
 533       if ((unsigned)StartPos > Size)
 534          return _error->Error("This http server has broken range support");
 535       return true;
 536    }
 537
 538    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 539    {
 540       HaveContent = true;
 541       if (stringcasecmp(Val,"chunked") == 0)
 542          Encoding = Chunked;
 543
 544       return true;
 545    }
 546
 547    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 548    {
 549       if (StrToTime(Val,Date) == false)
 550          return _error->Error("Unknown date format");
 551       return true;
 552    }
 553
 554    return true;
 555 }
 556                                                                         /*}}}*/
 557
 558 // HttpMethod::SendReq - Send the HTTP request                          /*{{{*/
 559 // ---------------------------------------------------------------------
 560 /* This places the http request in the outbound buffer */
 561 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
 562 {
 563    URI Uri = Itm->Uri;
 564
 565    // The HTTP server expects a hostname with a trailing :port
 566    char Buf[1000];
 567    string ProperHost = Uri.Host;
 568    if (Uri.Port != 0)
 569    {
 570       sprintf(Buf,":%u",Uri.Port);
 571       ProperHost += Buf;
 572    }
 573
 574    // Just in case.
 575    if (Itm->Uri.length() >= sizeof(Buf))
 576        abort();
 577
 578    /* Build the request. We include a keep-alive header only for non-proxy
 579       requests. This is to tweak old http/1.0 servers that do support keep-alive
 580       but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
 581       will glitch HTTP/1.0 proxies because they do not filter it out and
 582       pass it on, HTTP/1.1 says the connection should default to keep alive
 583       and we expect the proxy to do this */
 584    if (Proxy.empty() == true)
 585       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
 586               QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
 587    else
 588    {
 589       /* Generate a cache control header if necessary. We place a max
 590          cache age on index files, optionally set a no-cache directive
 591          and a no-store directive for archives. */
 592       sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
 593               Itm->Uri.c_str(),ProperHost.c_str());
 594       if (_config->FindB("Acquire::http::No-Cache",false) == true)
 595          strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
 596       else
 597       {
 598          if (Itm->IndexFile == true)
 599             sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
 600                     _config->FindI("Acquire::http::Max-Age",60*60*24));
 601          else
 602          {
 603             if (_config->FindB("Acquire::http::No-Store",false) == true)
 604                strcat(Buf,"Cache-Control: no-store\r\n");
 605          }
 606       }
 607    }
 608
 609    string Req = Buf;
 610
 611    // Check for a partial file
 612    struct stat SBuf;
 613    if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 614    {
 615       // In this case we send an if-range query with a range header
 616       sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf.st_size - 1,
 617               TimeRFC1123(SBuf.st_mtime).c_str());
 618       Req += Buf;
 619    }
 620    else
 621    {
 622       if (Itm->LastModified != 0)
 623       {
 624          sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
 625          Req += Buf;
 626       }
 627    }
 628
 629    if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
 630       Req += string("Proxy-Authorization: Basic ") +
 631           Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
 632
 633    Req += "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
 634 //   cerr << Req << endl;
 635
 636    Out.Read(Req);
 637 }
 638                                                                         /*}}}*/
 639 // HttpMethod::Go - Run a single loop                                   /*{{{*/
 640 // ---------------------------------------------------------------------
 641 /* This runs the select loop over the server FDs, Output file FDs and
 642    stdin. */
 643 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
 644 {
 645    // Server has closed the connection
 646    if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
 647                                ToFile == false))
 648       return false;
 649
 650    fd_set rfds,wfds,efds;
 651    FD_ZERO(&rfds);
 652    FD_ZERO(&wfds);
 653    FD_ZERO(&efds);
 654
 655    // Add the server
 656    if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1)
 657       FD_SET(Srv->ServerFd,&wfds);
 658    if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
 659       FD_SET(Srv->ServerFd,&rfds);
 660
 661    // Add the file
 662    int FileFD = -1;
 663    if (File != 0)
 664       FileFD = File->Fd();
 665
 666    if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
 667       FD_SET(FileFD,&wfds);
 668
 669    // Add stdin
 670    FD_SET(STDIN_FILENO,&rfds);
 671
 672    // Error Set
 673    if (FileFD != -1)
 674       FD_SET(FileFD,&efds);
 675    if (Srv->ServerFd != -1)
 676       FD_SET(Srv->ServerFd,&efds);
 677
 678    // Figure out the max fd
 679    int MaxFd = FileFD;
 680    if (MaxFd < Srv->ServerFd)
 681       MaxFd = Srv->ServerFd;
 682
 683    // Select
 684    struct timeval tv;
 685    tv.tv_sec = TimeOut;
 686    tv.tv_usec = 0;
 687    int Res = 0;
 688    if ((Res = select(MaxFd+1,&rfds,&wfds,&efds,&tv)) < 0)
 689       return _error->Errno("select","Select failed");
 690
 691    if (Res == 0)
 692    {
 693       _error->Error("Connection timed out");
 694       return ServerDie(Srv);
 695    }
 696
 697    // Some kind of exception (error) on the sockets, die
 698    if ((FileFD != -1 && FD_ISSET(FileFD,&efds)) ||
 699        (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&efds)))
 700       return _error->Error("Socket Exception");
 701
 702    // Handle server IO
 703    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
 704    {
 705       errno = 0;
 706       if (Srv->In.Read(Srv->ServerFd) == false)
 707          return ServerDie(Srv);
 708    }
 709
 710    if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
 711    {
 712       errno = 0;
 713       if (Srv->Out.Write(Srv->ServerFd) == false)
 714          return ServerDie(Srv);
 715    }
 716
 717    // Send data to the file
 718    if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
 719    {
 720       if (Srv->In.Write(FileFD) == false)
 721          return _error->Errno("write","Error writing to output file");
 722    }
 723
 724    // Handle commands from APT
 725    if (FD_ISSET(STDIN_FILENO,&rfds))
 726    {
 727       if (Run(true) != -1)
 728          exit(100);
 729    }
 730
 731    return true;
 732 }
 733                                                                         /*}}}*/
 734 // HttpMethod::Flush - Dump the buffer into the file                    /*{{{*/
 735 // ---------------------------------------------------------------------
 736 /* This takes the current input buffer from the Server FD and writes it
 737    into the file */
 738 bool HttpMethod::Flush(ServerState *Srv)
 739 {
 740    if (File != 0)
 741    {
 742       SetNonBlock(File->Fd(),false);
 743       if (Srv->In.WriteSpace() == false)
 744          return true;
 745
 746       while (Srv->In.WriteSpace() == true)
 747       {
 748          if (Srv->In.Write(File->Fd()) == false)
 749             return _error->Errno("write","Error writing to file");
 750          if (Srv->In.IsLimit() == true)
 751             return true;
 752       }
 753
 754       if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
 755          return true;
 756    }
 757    return false;
 758 }
 759                                                                         /*}}}*/
 760 // HttpMethod::ServerDie - The server has closed the connection.        /*{{{*/
 761 // ---------------------------------------------------------------------
 762 /* */
 763 bool HttpMethod::ServerDie(ServerState *Srv)
 764 {
 765    unsigned int LErrno = errno;
 766
 767    // Dump the buffer to the file
 768    if (Srv->State == ServerState::Data)
 769    {
 770       SetNonBlock(File->Fd(),false);
 771       while (Srv->In.WriteSpace() == true)
 772       {
 773          if (Srv->In.Write(File->Fd()) == false)
 774             return _error->Errno("write","Error writing to the file");
 775
 776          // Done
 777          if (Srv->In.IsLimit() == true)
 778             return true;
 779       }
 780    }
 781
 782    // See if this is because the server finished the data stream
 783    if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
 784        Srv->Encoding != ServerState::Closes)
 785    {
 786       Srv->Close();
 787       if (LErrno == 0)
 788          return _error->Error("Error reading from server Remote end closed connection");
 789       errno = LErrno;
 790       return _error->Errno("read","Error reading from server");
 791    }
 792    else
 793    {
 794       Srv->In.Limit(-1);
 795
 796       // Nothing left in the buffer
 797       if (Srv->In.WriteSpace() == false)
 798          return false;
 799
 800       // We may have got multiple responses back in one packet..
 801       Srv->Close();
 802       return true;
 803    }
 804
 805    return false;
 806 }
 807                                                                         /*}}}*/
 808 // HttpMethod::DealWithHeaders - Handle the retrieved header data       /*{{{*/
 809 // ---------------------------------------------------------------------
 810 /* We look at the header data we got back from the server and decide what
 811    to do. Returns
 812      0 - File is open,
 813      1 - IMS hit
 814      3 - Unrecoverable error
 815      4 - Error with error content page
 816      5 - Unrecoverable non-server error (close the connection) */
 817 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
 818 {
 819    // Not Modified
 820    if (Srv->Result == 304)
 821    {
 822       unlink(Queue->DestFile.c_str());
 823       Res.IMSHit = true;
 824       Res.LastModified = Queue->LastModified;
 825       return 1;
 826    }
 827
 828    /* We have a reply we dont handle. This should indicate a perm server
 829       failure */
 830    if (Srv->Result < 200 || Srv->Result >= 300)
 831    {
 832       _error->Error("%u %s",Srv->Result,Srv->Code);
 833       if (Srv->HaveContent == true)
 834          return 4;
 835       return 3;
 836    }
 837
 838    // This is some sort of 2xx 'data follows' reply
 839    Res.LastModified = Srv->Date;
 840    Res.Size = Srv->Size;
 841
 842    // Open the file
 843    delete File;
 844    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 845    if (_error->PendingError() == true)
 846       return 5;
 847
 848    FailFile = Queue->DestFile;
 849    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 850    FailFd = File->Fd();
 851    FailTime = Srv->Date;
 852
 853    // Set the expected size
 854    if (Srv->StartPos >= 0)
 855    {
 856       Res.ResumePoint = Srv->StartPos;
 857       ftruncate(File->Fd(),Srv->StartPos);
 858    }
 859
 860    // Set the start point
 861    lseek(File->Fd(),0,SEEK_END);
 862
 863    delete Srv->In.MD5;
 864    Srv->In.MD5 = new MD5Summation;
 865
 866    // Fill the MD5 Hash if the file is non-empty (resume)
 867    if (Srv->StartPos > 0)
 868    {
 869       lseek(File->Fd(),0,SEEK_SET);
 870       if (Srv->In.MD5->AddFD(File->Fd(),Srv->StartPos) == false)
 871       {
 872          _error->Errno("read","Problem hashing file");
 873          return 5;
 874       }
 875       lseek(File->Fd(),0,SEEK_END);
 876    }
 877
 878    SetNonBlock(File->Fd(),true);
 879    return 0;
 880 }
 881                                                                         /*}}}*/
 882 // HttpMethod::SigTerm - Handle a fatal signal                          /*{{{*/
 883 // ---------------------------------------------------------------------
 884 /* This closes and timestamps the open file. This is neccessary to get
 885    resume behavoir on user abort */
 886 void HttpMethod::SigTerm(int)
 887 {
 888    if (FailFd == -1)
 889       exit(100);
 890    close(FailFd);
 891
 892    // Timestamp
 893    struct utimbuf UBuf;
 894    UBuf.actime = FailTime;
 895    UBuf.modtime = FailTime;
 896    utime(FailFile.c_str(),&UBuf);
 897
 898    exit(100);
 899 }
 900                                                                         /*}}}*/
 901 // HttpMethod::Fetch - Fetch an item                                    /*{{{*/
 902 // ---------------------------------------------------------------------
 903 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 904    depth. */
 905 bool HttpMethod::Fetch(FetchItem *)
 906 {
 907    if (Server == 0)
 908       return true;
 909
 910    // Queue the requests
 911    int Depth = -1;
 912    bool Tail = false;
 913    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; I = I->Next, Depth++)
 914    {
 915       // Make sure we stick with the same server
 916       if (Server->Comp(I->Uri) == false)
 917          break;
 918       if (QueueBack == I)
 919          Tail = true;
 920       if (Tail == true)
 921       {
 922          QueueBack = I->Next;
 923          SendReq(I,Server->Out);
 924          continue;
 925       }
 926    }
 927
 928    return true;
 929 };
 930                                                                         /*}}}*/
 931 // HttpMethod::Configuration - Handle a configuration message           /*{{{*/
 932 // ---------------------------------------------------------------------
 933 /* We stash the desired pipeline depth */
 934 bool HttpMethod::Configuration(string Message)
 935 {
 936    if (pkgAcqMethod::Configuration(Message) == false)
 937       return false;
 938
 939    TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
 940    PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
 941                                   PipelineDepth);
 942
 943    return true;
 944 }
 945                                                                         /*}}}*/
 946 // HttpMethod::Loop - Main loop                                         /*{{{*/
 947 // ---------------------------------------------------------------------
 948 /* */
 949 int HttpMethod::Loop()
 950 {
 951    signal(SIGTERM,SigTerm);
 952    signal(SIGINT,SigTerm);
 953
 954    Server = 0;
 955
 956    int FailCounter = 0;
 957    while (1)
 958    {
 959       // We have no commands, wait for some to arrive
 960       if (Queue == 0)
 961       {
 962          if (WaitFd(STDIN_FILENO) == false)
 963             return 0;
 964       }
 965
 966       /* Run messages, we can accept 0 (no message) if we didn't
 967          do a WaitFd above.. Otherwise the FD is closed. */
 968       int Result = Run(true);
 969       if (Result != -1 && (Result != 0 || Queue == 0))
 970          return 100;
 971
 972       if (Queue == 0)
 973          continue;
 974
 975       // Connect to the server
 976       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 977       {
 978          delete Server;
 979          Server = new ServerState(Queue->Uri,this);
 980       }
 981
 982       // Reset the pipeline
 983       if (Server->ServerFd == -1)
 984          QueueBack = Queue;
 985
 986       // Connnect to the host
 987       if (Server->Open() == false)
 988       {
 989          Fail(true);
 990          delete Server;
 991          Server = 0;
 992          continue;
 993       }
 994
 995       // Fill the pipeline.
 996       Fetch(0);
 997
 998       // Fetch the next URL header data from the server.
 999       switch (Server->RunHeaders())
1000       {
1001          case 0:
1002          break;
1003
1004          // The header data is bad
1005          case 2:
1006          {
1007             _error->Error("Bad header Data");
1008             Fail(true);
1009             continue;
1010          }
1011
1012          // The server closed a connection during the header get..
1013          default:
1014          case 1:
1015          {
1016             FailCounter++;
1017             _error->Discard();
1018             Server->Close();
1019
1020             if (FailCounter >= 2)
1021             {
1022                Fail("Connection failed",true);
1023                FailCounter = 0;
1024             }
1025
1026             continue;
1027          }
1028       };
1029
1030       // Decide what to do.
1031       FetchResult Res;
1032       Res.Filename = Queue->DestFile;
1033       switch (DealWithHeaders(Res,Server))
1034       {
1035          // Ok, the file is Open
1036          case 0:
1037          {
1038             URIStart(Res);
1039
1040             // Run the data
1041             bool Result =  Server->RunData();
1042
1043             // Close the file, destroy the FD object and timestamp it
1044             FailFd = -1;
1045             delete File;
1046             File = 0;
1047
1048             // Timestamp
1049             struct utimbuf UBuf;
1050             time(&UBuf.actime);
1051             UBuf.actime = Server->Date;
1052             UBuf.modtime = Server->Date;
1053             utime(Queue->DestFile.c_str(),&UBuf);
1054
1055             // Send status to APT
1056             if (Result == true)
1057             {
1058                Res.MD5Sum = Server->In.MD5->Result();
1059                URIDone(Res);
1060             }
1061             else
1062                Fail(true);
1063
1064             break;
1065          }
1066
1067          // IMS hit
1068          case 1:
1069          {
1070             URIDone(Res);
1071             break;
1072          }
1073
1074          // Hard server error, not found or something
1075          case 3:
1076          {
1077             Fail();
1078             break;
1079          }
1080
1081          // Hard internal error, kill the connection and fail
1082          case 5:
1083          {
1084             Fail();
1085             Server->Close();
1086             break;
1087          }
1088
1089          // We need to flush the data, the header is like a 404 w/ error text
1090          case 4:
1091          {
1092             Fail();
1093
1094             // Send to content to dev/null
1095             File = new FileFd("/dev/null",FileFd::WriteExists);
1096             Server->RunData();
1097             delete File;
1098             File = 0;
1099             break;
1100          }
1101
1102          default:
1103          Fail("Internal error");
1104          break;
1105       }
1106
1107       FailCounter = 0;
1108    }
1109
1110    return 0;
1111 }
1112                                                                         /*}}}*/
1113
1114 int main()
1115 {
1116    HttpMethod Mth;
1117
1118    return Mth.Loop();
1119 }