methods/server.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 /* ######################################################################
   4
   5    HTTP and HTTPS share a lot of common code and these classes are
   6    exactly the dumping ground for this common code
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include <config.h>
  12
  13 #include <apt-pkg/acquire-method.h>
  14 #include <apt-pkg/configuration.h>
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/fileutl.h>
  17 #include <apt-pkg/strutl.h>
  18
  19 #include <ctype.h>
  20 #include <signal.h>
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <sys/stat.h>
  24 #include <sys/time.h>
  25 #include <time.h>
  26 #include <unistd.h>
  27 #include <iostream>
  28 #include <limits>
  29 #include <map>
  30 #include <string>
  31 #include <vector>
  32
  33 #include "server.h"
  34
  35 #include <apti18n.h>
  36                                                                         /*}}}*/
  37 using namespace std;
  38
  39 string ServerMethod::FailFile;
  40 int ServerMethod::FailFd = -1;
  41 time_t ServerMethod::FailTime = 0;
  42
  43 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
  46    parse error occurred */
  47 ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
  48                                                       const std::string &Uri)
  49 {
  50    State = Header;
  51
  52    Owner->Status(_("Waiting for headers"));
  53
  54    Major = 0;
  55    Minor = 0;
  56    Result = 0;
  57    TotalFileSize = 0;
  58    JunkSize = 0;
  59    StartPos = 0;
  60    Encoding = Closes;
  61    HaveContent = false;
  62    time(&Date);
  63
  64    do
  65    {
  66       string Data;
  67       if (ReadHeaderLines(Data) == false)
  68          continue;
  69
  70       if (Owner->Debug == true)
  71          clog << "Answer for: " << Uri << endl << Data;
  72
  73       for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
  74       {
  75          string::const_iterator J = I;
  76          for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
  77          if (HeaderLine(string(I,J)) == false)
  78             return RUN_HEADERS_PARSE_ERROR;
  79          I = J;
  80       }
  81
  82       // 100 Continue is a Nop...
  83       if (Result == 100)
  84          continue;
  85
  86       // Tidy up the connection persistence state.
  87       if (Encoding == Closes && HaveContent == true)
  88          Persistent = false;
  89
  90       return RUN_HEADERS_OK;
  91    }
  92    while (LoadNextResponse(false, File) == true);
  93
  94    return RUN_HEADERS_IO_ERROR;
  95 }
  96                                                                         /*}}}*/
  97 // ServerState::HeaderLine - Process a header line                      /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* */
 100 bool ServerState::HeaderLine(string Line)
 101 {
 102    if (Line.empty() == true)
 103       return true;
 104
 105    string::size_type Pos = Line.find(' ');
 106    if (Pos == string::npos || Pos+1 > Line.length())
 107    {
 108       // Blah, some servers use "connection:closes", evil.
 109       Pos = Line.find(':');
 110       if (Pos == string::npos || Pos + 2 > Line.length())
 111          return _error->Error(_("Bad header line"));
 112       Pos++;
 113    }
 114
 115    // Parse off any trailing spaces between the : and the next word.
 116    string::size_type Pos2 = Pos;
 117    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 118       Pos2++;
 119
 120    string Tag = string(Line,0,Pos);
 121    string Val = string(Line,Pos2);
 122
 123    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 124    {
 125       // Evil servers return no version
 126       if (Line[4] == '/')
 127       {
 128          int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
 129          if (elements == 3)
 130          {
 131             Code[0] = '\0';
 132             if (Owner != NULL && Owner->Debug == true)
 133                clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
 134          }
 135          else if (elements != 4)
 136             return _error->Error(_("The HTTP server sent an invalid reply header"));
 137       }
 138       else
 139       {
 140          Major = 0;
 141          Minor = 9;
 142          if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
 143             return _error->Error(_("The HTTP server sent an invalid reply header"));
 144       }
 145
 146       /* Check the HTTP response header to get the default persistence
 147          state. */
 148       if (Major < 1)
 149          Persistent = false;
 150       else
 151       {
 152          if (Major == 1 && Minor == 0)
 153             Persistent = false;
 154          else
 155             Persistent = true;
 156       }
 157
 158       return true;
 159    }
 160
 161    if (stringcasecmp(Tag,"Content-Length:") == 0)
 162    {
 163       if (Encoding == Closes)
 164          Encoding = Stream;
 165       HaveContent = true;
 166
 167       unsigned long long * DownloadSizePtr = &DownloadSize;
 168       if (Result == 416)
 169          DownloadSizePtr = &JunkSize;
 170
 171       *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10);
 172       if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max())
 173          return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
 174       else if (*DownloadSizePtr == 0)
 175          HaveContent = false;
 176
 177       // On partial content (206) the Content-Length less than the real
 178       // size, so do not set it here but leave that to the Content-Range
 179       // header instead
 180       if(Result != 206 && TotalFileSize == 0)
 181          TotalFileSize = DownloadSize;
 182
 183       return true;
 184    }
 185
 186    if (stringcasecmp(Tag,"Content-Type:") == 0)
 187    {
 188       HaveContent = true;
 189       return true;
 190    }
 191
 192    if (stringcasecmp(Tag,"Content-Range:") == 0)
 193    {
 194       HaveContent = true;
 195
 196       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
 197       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
 198          ; // we got the expected filesize which is all we wanted
 199       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
 200          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 201       if ((unsigned long long)StartPos > TotalFileSize)
 202          return _error->Error(_("This HTTP server has broken range support"));
 203
 204       // figure out what we will download
 205       DownloadSize = TotalFileSize - StartPos;
 206       return true;
 207    }
 208
 209    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 210    {
 211       HaveContent = true;
 212       if (stringcasecmp(Val,"chunked") == 0)
 213          Encoding = Chunked;
 214       return true;
 215    }
 216
 217    if (stringcasecmp(Tag,"Connection:") == 0)
 218    {
 219       if (stringcasecmp(Val,"close") == 0)
 220          Persistent = false;
 221       if (stringcasecmp(Val,"keep-alive") == 0)
 222          Persistent = true;
 223       return true;
 224    }
 225
 226    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 227    {
 228       if (RFC1123StrToTime(Val.c_str(), Date) == false)
 229          return _error->Error(_("Unknown date format"));
 230       return true;
 231    }
 232
 233    if (stringcasecmp(Tag,"Location:") == 0)
 234    {
 235       Location = Val;
 236       return true;
 237    }
 238
 239    return true;
 240 }
 241                                                                         /*}}}*/
 242 // ServerState::ServerState - Constructor                               /*{{{*/
 243 ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOut(120), Owner(Owner)
 244 {
 245    Reset();
 246 }
 247                                                                         /*}}}*/
 248
 249 bool ServerMethod::Configuration(string Message)                        /*{{{*/
 250 {
 251    return pkgAcqMethod::Configuration(Message);
 252 }
 253                                                                         /*}}}*/
 254
 255 // ServerMethod::DealWithHeaders - Handle the retrieved header data     /*{{{*/
 256 // ---------------------------------------------------------------------
 257 /* We look at the header data we got back from the server and decide what
 258    to do. Returns DealWithHeadersResult (see http.h for details).
 259  */
 260 ServerMethod::DealWithHeadersResult
 261 ServerMethod::DealWithHeaders(FetchResult &Res)
 262 {
 263    // Not Modified
 264    if (Server->Result == 304)
 265    {
 266       unlink(Queue->DestFile.c_str());
 267       Res.IMSHit = true;
 268       Res.LastModified = Queue->LastModified;
 269       return IMS_HIT;
 270    }
 271
 272    /* Redirect
 273     *
 274     * Note that it is only OK for us to treat all redirection the same
 275     * because we *always* use GET, not other HTTP methods.  There are
 276     * three redirection codes for which it is not appropriate that we
 277     * redirect.  Pass on those codes so the error handling kicks in.
 278     */
 279    if (AllowRedirect
 280        && (Server->Result > 300 && Server->Result < 400)
 281        && (Server->Result != 300       // Multiple Choices
 282            && Server->Result != 304    // Not Modified
 283            && Server->Result != 306))  // (Not part of HTTP/1.1, reserved)
 284    {
 285       if (Server->Location.empty() == true);
 286       else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
 287       {
 288          URI Uri = Queue->Uri;
 289          if (Uri.Host.empty() == false)
 290             NextURI = URI::SiteOnly(Uri);
 291          else
 292             NextURI.clear();
 293          NextURI.append(DeQuoteString(Server->Location));
 294          return TRY_AGAIN_OR_REDIRECT;
 295       }
 296       else
 297       {
 298          NextURI = DeQuoteString(Server->Location);
 299          URI tmpURI = NextURI;
 300          URI Uri = Queue->Uri;
 301          // same protocol redirects are okay
 302          if (tmpURI.Access == Uri.Access)
 303             return TRY_AGAIN_OR_REDIRECT;
 304          // as well as http to https
 305          else if (Uri.Access == "http" && tmpURI.Access == "https")
 306             return TRY_AGAIN_OR_REDIRECT;
 307       }
 308       /* else pass through for error message */
 309    }
 310    // retry after an invalid range response without partial data
 311    else if (Server->Result == 416)
 312    {
 313       struct stat SBuf;
 314       if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 315       {
 316          if ((unsigned long long)SBuf.st_size == Server->TotalFileSize)
 317          {
 318             // the file is completely downloaded, but was not moved
 319             if (Server->HaveContent == true)
 320             {
 321                // Send to error page to dev/null
 322                FileFd DevNull("/dev/null",FileFd::WriteExists);
 323                Server->RunData(&DevNull);
 324             }
 325             Server->HaveContent = false;
 326             Server->StartPos = Server->TotalFileSize;
 327             Server->Result = 200;
 328          }
 329          else if (unlink(Queue->DestFile.c_str()) == 0)
 330          {
 331             NextURI = Queue->Uri;
 332             return TRY_AGAIN_OR_REDIRECT;
 333          }
 334       }
 335    }
 336
 337    /* We have a reply we dont handle. This should indicate a perm server
 338       failure */
 339    if (Server->Result < 200 || Server->Result >= 300)
 340    {
 341       char err[255];
 342       snprintf(err,sizeof(err)-1,"HttpError%i",Server->Result);
 343       SetFailReason(err);
 344       _error->Error("%u %s",Server->Result,Server->Code);
 345       if (Server->HaveContent == true)
 346          return ERROR_WITH_CONTENT_PAGE;
 347       return ERROR_UNRECOVERABLE;
 348    }
 349
 350    // This is some sort of 2xx 'data follows' reply
 351    Res.LastModified = Server->Date;
 352    Res.Size = Server->TotalFileSize;
 353
 354    // Open the file
 355    delete File;
 356    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 357    if (_error->PendingError() == true)
 358       return ERROR_NOT_FROM_SERVER;
 359
 360    FailFile = Queue->DestFile;
 361    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 362    FailFd = File->Fd();
 363    FailTime = Server->Date;
 364
 365    if (Server->InitHashes(*File) == false)
 366    {
 367       _error->Errno("read",_("Problem hashing file"));
 368       return ERROR_NOT_FROM_SERVER;
 369    }
 370    if (Server->StartPos > 0)
 371       Res.ResumePoint = Server->StartPos;
 372
 373    SetNonBlock(File->Fd(),true);
 374    return FILE_IS_OPEN;
 375 }
 376                                                                         /*}}}*/
 377 // ServerMethod::SigTerm - Handle a fatal signal                        /*{{{*/
 378 // ---------------------------------------------------------------------
 379 /* This closes and timestamps the open file. This is necessary to get
 380    resume behavoir on user abort */
 381 void ServerMethod::SigTerm(int)
 382 {
 383    if (FailFd == -1)
 384       _exit(100);
 385
 386    struct timeval times[2];
 387    times[0].tv_sec = FailTime;
 388    times[1].tv_sec = FailTime;
 389    times[0].tv_usec = times[1].tv_usec = 0;
 390    utimes(FailFile.c_str(), times);
 391    close(FailFd);
 392
 393    _exit(100);
 394 }
 395                                                                         /*}}}*/
 396 // ServerMethod::Fetch - Fetch an item                                  /*{{{*/
 397 // ---------------------------------------------------------------------
 398 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 399    depth. */
 400 bool ServerMethod::Fetch(FetchItem *)
 401 {
 402    if (Server == 0)
 403       return true;
 404
 405    // Queue the requests
 406    int Depth = -1;
 407    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 408         I = I->Next, Depth++)
 409    {
 410       // If pipelining is disabled, we only queue 1 request
 411       if (Server->Pipeline == false && Depth >= 0)
 412          break;
 413
 414       // Make sure we stick with the same server
 415       if (Server->Comp(I->Uri) == false)
 416          break;
 417       if (QueueBack == I)
 418       {
 419          QueueBack = I->Next;
 420          SendReq(I);
 421          continue;
 422       }
 423    }
 424
 425    return true;
 426 }
 427                                                                         /*}}}*/
 428 // ServerMethod::Loop - Main loop                                       /*{{{*/
 429 int ServerMethod::Loop()
 430 {
 431    typedef vector<string> StringVector;
 432    typedef vector<string>::iterator StringVectorIterator;
 433    map<string, StringVector> Redirected;
 434
 435    signal(SIGTERM,SigTerm);
 436    signal(SIGINT,SigTerm);
 437
 438    Server = 0;
 439
 440    int FailCounter = 0;
 441    while (1)
 442    {
 443       // We have no commands, wait for some to arrive
 444       if (Queue == 0)
 445       {
 446          if (WaitFd(STDIN_FILENO) == false)
 447             return 0;
 448       }
 449
 450       /* Run messages, we can accept 0 (no message) if we didn't
 451          do a WaitFd above.. Otherwise the FD is closed. */
 452       int Result = Run(true);
 453       if (Result != -1 && (Result != 0 || Queue == 0))
 454       {
 455          if(FailReason.empty() == false ||
 456             _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
 457             return 100;
 458          else
 459             return 0;
 460       }
 461
 462       if (Queue == 0)
 463          continue;
 464
 465       // Connect to the server
 466       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 467       {
 468          delete Server;
 469          Server = CreateServerState(Queue->Uri);
 470       }
 471       /* If the server has explicitly said this is the last connection
 472          then we pre-emptively shut down the pipeline and tear down
 473          the connection. This will speed up HTTP/1.0 servers a tad
 474          since we don't have to wait for the close sequence to
 475          complete */
 476       if (Server->Persistent == false)
 477          Server->Close();
 478
 479       // Reset the pipeline
 480       if (Server->IsOpen() == false)
 481          QueueBack = Queue;
 482
 483       // Connnect to the host
 484       if (Server->Open() == false)
 485       {
 486          Fail(true);
 487          delete Server;
 488          Server = 0;
 489          continue;
 490       }
 491
 492       // Fill the pipeline.
 493       Fetch(0);
 494
 495       // Fetch the next URL header data from the server.
 496       switch (Server->RunHeaders(File, Queue->Uri))
 497       {
 498          case ServerState::RUN_HEADERS_OK:
 499          break;
 500
 501          // The header data is bad
 502          case ServerState::RUN_HEADERS_PARSE_ERROR:
 503          {
 504             _error->Error(_("Bad header data"));
 505             Fail(true);
 506             RotateDNS();
 507             continue;
 508          }
 509
 510          // The server closed a connection during the header get..
 511          default:
 512          case ServerState::RUN_HEADERS_IO_ERROR:
 513          {
 514             FailCounter++;
 515             _error->Discard();
 516             Server->Close();
 517             Server->Pipeline = false;
 518
 519             if (FailCounter >= 2)
 520             {
 521                Fail(_("Connection failed"),true);
 522                FailCounter = 0;
 523             }
 524
 525             RotateDNS();
 526             continue;
 527          }
 528       };
 529
 530       // Decide what to do.
 531       FetchResult Res;
 532       Res.Filename = Queue->DestFile;
 533       switch (DealWithHeaders(Res))
 534       {
 535          // Ok, the file is Open
 536          case FILE_IS_OPEN:
 537          {
 538             URIStart(Res);
 539
 540             // Run the data
 541             bool Result = true;
 542             if (Server->HaveContent)
 543                Result = Server->RunData(File);
 544
 545             /* If the server is sending back sizeless responses then fill in
 546                the size now */
 547             if (Res.Size == 0)
 548                Res.Size = File->Size();
 549
 550             // Close the file, destroy the FD object and timestamp it
 551             FailFd = -1;
 552             delete File;
 553             File = 0;
 554
 555             // Timestamp
 556             struct timeval times[2];
 557             times[0].tv_sec = times[1].tv_sec = Server->Date;
 558             times[0].tv_usec = times[1].tv_usec = 0;
 559             utimes(Queue->DestFile.c_str(), times);
 560
 561             // Send status to APT
 562             if (Result == true)
 563             {
 564                Res.TakeHashes(*Server->GetHashes());
 565                URIDone(Res);
 566             }
 567             else
 568             {
 569                if (Server->IsOpen() == false)
 570                {
 571                   FailCounter++;
 572                   _error->Discard();
 573                   Server->Close();
 574
 575                   if (FailCounter >= 2)
 576                   {
 577                      Fail(_("Connection failed"),true);
 578                      FailCounter = 0;
 579                   }
 580
 581                   QueueBack = Queue;
 582                }
 583                else
 584                   Fail(true);
 585             }
 586             break;
 587          }
 588
 589          // IMS hit
 590          case IMS_HIT:
 591          {
 592             URIDone(Res);
 593             break;
 594          }
 595
 596          // Hard server error, not found or something
 597          case ERROR_UNRECOVERABLE:
 598          {
 599             Fail();
 600             break;
 601          }
 602
 603          // Hard internal error, kill the connection and fail
 604          case ERROR_NOT_FROM_SERVER:
 605          {
 606             delete File;
 607             File = 0;
 608
 609             Fail();
 610             RotateDNS();
 611             Server->Close();
 612             break;
 613          }
 614
 615          // We need to flush the data, the header is like a 404 w/ error text
 616          case ERROR_WITH_CONTENT_PAGE:
 617          {
 618             Fail();
 619
 620             // Send to content to dev/null
 621             File = new FileFd("/dev/null",FileFd::WriteExists);
 622             Server->RunData(File);
 623             delete File;
 624             File = 0;
 625             break;
 626          }
 627
 628          // Try again with a new URL
 629          case TRY_AGAIN_OR_REDIRECT:
 630          {
 631             // Clear rest of response if there is content
 632             if (Server->HaveContent)
 633             {
 634                File = new FileFd("/dev/null",FileFd::WriteExists);
 635                Server->RunData(File);
 636                delete File;
 637                File = 0;
 638             }
 639
 640             /* Detect redirect loops.  No more redirects are allowed
 641                after the same URI is seen twice in a queue item. */
 642             StringVector &R = Redirected[Queue->DestFile];
 643             bool StopRedirects = false;
 644             if (R.empty() == true)
 645                R.push_back(Queue->Uri);
 646             else if (R[0] == "STOP" || R.size() > 10)
 647                StopRedirects = true;
 648             else
 649             {
 650                for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
 651                   if (Queue->Uri == *I)
 652                   {
 653                      R[0] = "STOP";
 654                      break;
 655                   }
 656
 657                R.push_back(Queue->Uri);
 658             }
 659
 660             if (StopRedirects == false)
 661                Redirect(NextURI);
 662             else
 663                Fail();
 664
 665             break;
 666          }
 667
 668          default:
 669          Fail(_("Internal error"));
 670          break;
 671       }
 672
 673       FailCounter = 0;
 674    }
 675
 676    return 0;
 677 }
 678                                                                         /*}}}*/