methods/server.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 /* ######################################################################
   4
   5    HTTP and HTTPS share a lot of common code and these classes are
   6    exactly the dumping ground for this common code
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include <config.h>
  12
  13 #include <apt-pkg/acquire-method.h>
  14 #include <apt-pkg/configuration.h>
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/fileutl.h>
  17 #include <apt-pkg/strutl.h>
  18
  19 #include <ctype.h>
  20 #include <signal.h>
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <sys/stat.h>
  24 #include <sys/time.h>
  25 #include <time.h>
  26 #include <unistd.h>
  27 #include <iostream>
  28 #include <limits>
  29 #include <map>
  30 #include <string>
  31 #include <vector>
  32
  33 #include "server.h"
  34
  35 #include <apti18n.h>
  36                                                                         /*}}}*/
  37 using namespace std;
  38
  39 string ServerMethod::FailFile;
  40 int ServerMethod::FailFd = -1;
  41 time_t ServerMethod::FailTime = 0;
  42
  43 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
  46    parse error occurred */
  47 ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
  48                                                       const std::string &Uri)
  49 {
  50    State = Header;
  51
  52    Owner->Status(_("Waiting for headers"));
  53
  54    Major = 0;
  55    Minor = 0;
  56    Result = 0;
  57    Size = 0;
  58    StartPos = 0;
  59    Encoding = Closes;
  60    HaveContent = false;
  61    time(&Date);
  62
  63    do
  64    {
  65       string Data;
  66       if (ReadHeaderLines(Data) == false)
  67          continue;
  68
  69       if (Owner->Debug == true)
  70          clog << "Answer for: " << Uri << endl << Data;
  71
  72       for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
  73       {
  74          string::const_iterator J = I;
  75          for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
  76          if (HeaderLine(string(I,J)) == false)
  77             return RUN_HEADERS_PARSE_ERROR;
  78          I = J;
  79       }
  80
  81       // 100 Continue is a Nop...
  82       if (Result == 100)
  83          continue;
  84
  85       // Tidy up the connection persistence state.
  86       if (Encoding == Closes && HaveContent == true)
  87          Persistent = false;
  88
  89       return RUN_HEADERS_OK;
  90    }
  91    while (LoadNextResponse(false, File) == true);
  92
  93    return RUN_HEADERS_IO_ERROR;
  94 }
  95                                                                         /*}}}*/
  96 // ServerState::HeaderLine - Process a header line                      /*{{{*/
  97 // ---------------------------------------------------------------------
  98 /* */
  99 bool ServerState::HeaderLine(string Line)
 100 {
 101    if (Line.empty() == true)
 102       return true;
 103
 104    string::size_type Pos = Line.find(' ');
 105    if (Pos == string::npos || Pos+1 > Line.length())
 106    {
 107       // Blah, some servers use "connection:closes", evil.
 108       Pos = Line.find(':');
 109       if (Pos == string::npos || Pos + 2 > Line.length())
 110          return _error->Error(_("Bad header line"));
 111       Pos++;
 112    }
 113
 114    // Parse off any trailing spaces between the : and the next word.
 115    string::size_type Pos2 = Pos;
 116    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 117       Pos2++;
 118
 119    string Tag = string(Line,0,Pos);
 120    string Val = string(Line,Pos2);
 121
 122    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 123    {
 124       // Evil servers return no version
 125       if (Line[4] == '/')
 126       {
 127          int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
 128          if (elements == 3)
 129          {
 130             Code[0] = '\0';
 131             if (Owner->Debug == true)
 132                clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
 133          }
 134          else if (elements != 4)
 135             return _error->Error(_("The HTTP server sent an invalid reply header"));
 136       }
 137       else
 138       {
 139          Major = 0;
 140          Minor = 9;
 141          if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
 142             return _error->Error(_("The HTTP server sent an invalid reply header"));
 143       }
 144
 145       /* Check the HTTP response header to get the default persistence
 146          state. */
 147       if (Major < 1)
 148          Persistent = false;
 149       else
 150       {
 151          if (Major == 1 && Minor == 0)
 152             Persistent = false;
 153          else
 154             Persistent = true;
 155       }
 156
 157       return true;
 158    }
 159
 160    if (stringcasecmp(Tag,"Content-Length:") == 0)
 161    {
 162       if (Encoding == Closes)
 163          Encoding = Stream;
 164       HaveContent = true;
 165
 166       // The length is already set from the Content-Range header
 167       if (StartPos != 0)
 168          return true;
 169
 170       Size = strtoull(Val.c_str(), NULL, 10);
 171       if (Size >= std::numeric_limits<unsigned long long>::max())
 172          return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
 173       else if (Size == 0)
 174          HaveContent = false;
 175       return true;
 176    }
 177
 178    if (stringcasecmp(Tag,"Content-Type:") == 0)
 179    {
 180       HaveContent = true;
 181       return true;
 182    }
 183
 184    if (stringcasecmp(Tag,"Content-Range:") == 0)
 185    {
 186       HaveContent = true;
 187
 188       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
 189       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
 190       {
 191          StartPos = 1; // ignore Content-Length, it would override Size
 192          HaveContent = false;
 193       }
 194       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
 195          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 196       if ((unsigned long long)StartPos > Size)
 197          return _error->Error(_("This HTTP server has broken range support"));
 198       return true;
 199    }
 200
 201    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 202    {
 203       HaveContent = true;
 204       if (stringcasecmp(Val,"chunked") == 0)
 205          Encoding = Chunked;
 206       return true;
 207    }
 208
 209    if (stringcasecmp(Tag,"Connection:") == 0)
 210    {
 211       if (stringcasecmp(Val,"close") == 0)
 212          Persistent = false;
 213       if (stringcasecmp(Val,"keep-alive") == 0)
 214          Persistent = true;
 215       return true;
 216    }
 217
 218    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 219    {
 220       if (RFC1123StrToTime(Val.c_str(), Date) == false)
 221          return _error->Error(_("Unknown date format"));
 222       return true;
 223    }
 224
 225    if (stringcasecmp(Tag,"Location:") == 0)
 226    {
 227       Location = Val;
 228       return true;
 229    }
 230
 231    return true;
 232 }
 233                                                                         /*}}}*/
 234 // ServerState::ServerState - Constructor                               /*{{{*/
 235 ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOut(120), Owner(Owner)
 236 {
 237    Reset();
 238 }
 239                                                                         /*}}}*/
 240
 241 bool ServerMethod::Configuration(string Message)                        /*{{{*/
 242 {
 243    return pkgAcqMethod::Configuration(Message);
 244 }
 245                                                                         /*}}}*/
 246
 247 // ServerMethod::DealWithHeaders - Handle the retrieved header data     /*{{{*/
 248 // ---------------------------------------------------------------------
 249 /* We look at the header data we got back from the server and decide what
 250    to do. Returns DealWithHeadersResult (see http.h for details).
 251  */
 252 ServerMethod::DealWithHeadersResult
 253 ServerMethod::DealWithHeaders(FetchResult &Res)
 254 {
 255    // Not Modified
 256    if (Server->Result == 304)
 257    {
 258       unlink(Queue->DestFile.c_str());
 259       Res.IMSHit = true;
 260       Res.LastModified = Queue->LastModified;
 261       return IMS_HIT;
 262    }
 263
 264    /* Redirect
 265     *
 266     * Note that it is only OK for us to treat all redirection the same
 267     * because we *always* use GET, not other HTTP methods.  There are
 268     * three redirection codes for which it is not appropriate that we
 269     * redirect.  Pass on those codes so the error handling kicks in.
 270     */
 271    if (AllowRedirect
 272        && (Server->Result > 300 && Server->Result < 400)
 273        && (Server->Result != 300       // Multiple Choices
 274            && Server->Result != 304    // Not Modified
 275            && Server->Result != 306))  // (Not part of HTTP/1.1, reserved)
 276    {
 277       if (Server->Location.empty() == true);
 278       else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
 279       {
 280          URI Uri = Queue->Uri;
 281          if (Uri.Host.empty() == false)
 282             NextURI = URI::SiteOnly(Uri);
 283          else
 284             NextURI.clear();
 285          NextURI.append(DeQuoteString(Server->Location));
 286          return TRY_AGAIN_OR_REDIRECT;
 287       }
 288       else
 289       {
 290          NextURI = DeQuoteString(Server->Location);
 291          URI tmpURI = NextURI;
 292          URI Uri = Queue->Uri;
 293          // same protocol redirects are okay
 294          if (tmpURI.Access == Uri.Access)
 295             return TRY_AGAIN_OR_REDIRECT;
 296          // as well as http to https
 297          else if (Uri.Access == "http" && tmpURI.Access == "https")
 298             return TRY_AGAIN_OR_REDIRECT;
 299       }
 300       /* else pass through for error message */
 301    }
 302    // retry after an invalid range response without partial data
 303    else if (Server->Result == 416)
 304    {
 305       struct stat SBuf;
 306       if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 307       {
 308          if ((unsigned long long)SBuf.st_size == Server->Size)
 309          {
 310             // the file is completely downloaded, but was not moved
 311             Server->StartPos = Server->Size;
 312             Server->Result = 200;
 313             Server->HaveContent = false;
 314          }
 315          else if (unlink(Queue->DestFile.c_str()) == 0)
 316          {
 317             NextURI = Queue->Uri;
 318             return TRY_AGAIN_OR_REDIRECT;
 319          }
 320       }
 321    }
 322
 323    /* We have a reply we dont handle. This should indicate a perm server
 324       failure */
 325    if (Server->Result < 200 || Server->Result >= 300)
 326    {
 327       std::string err;
 328       strprintf(err, "HttpError%u", Server->Result);
 329       SetFailReason(err);
 330       _error->Error("%u %s", Server->Result, Server->Code);
 331       if (Server->HaveContent == true)
 332          return ERROR_WITH_CONTENT_PAGE;
 333       return ERROR_UNRECOVERABLE;
 334    }
 335
 336    // This is some sort of 2xx 'data follows' reply
 337    Res.LastModified = Server->Date;
 338    Res.Size = Server->Size;
 339
 340    // Open the file
 341    delete File;
 342    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 343    if (_error->PendingError() == true)
 344       return ERROR_NOT_FROM_SERVER;
 345
 346    FailFile = Queue->DestFile;
 347    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 348    FailFd = File->Fd();
 349    FailTime = Server->Date;
 350
 351    if (Server->InitHashes(*File) == false)
 352    {
 353       _error->Errno("read",_("Problem hashing file"));
 354       return ERROR_NOT_FROM_SERVER;
 355    }
 356    if (Server->StartPos > 0)
 357       Res.ResumePoint = Server->StartPos;
 358
 359    SetNonBlock(File->Fd(),true);
 360    return FILE_IS_OPEN;
 361 }
 362                                                                         /*}}}*/
 363 // ServerMethod::SigTerm - Handle a fatal signal                        /*{{{*/
 364 // ---------------------------------------------------------------------
 365 /* This closes and timestamps the open file. This is necessary to get
 366    resume behavoir on user abort */
 367 void ServerMethod::SigTerm(int)
 368 {
 369    if (FailFd == -1)
 370       _exit(100);
 371
 372    struct timeval times[2];
 373    times[0].tv_sec = FailTime;
 374    times[1].tv_sec = FailTime;
 375    times[0].tv_usec = times[1].tv_usec = 0;
 376    utimes(FailFile.c_str(), times);
 377    close(FailFd);
 378
 379    _exit(100);
 380 }
 381                                                                         /*}}}*/
 382 // ServerMethod::Fetch - Fetch an item                                  /*{{{*/
 383 // ---------------------------------------------------------------------
 384 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 385    depth. */
 386 bool ServerMethod::Fetch(FetchItem *)
 387 {
 388    if (Server == 0)
 389       return true;
 390
 391    // Queue the requests
 392    int Depth = -1;
 393    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 394         I = I->Next, Depth++)
 395    {
 396       if (Depth >= 0)
 397       {
 398          // If pipelining is disabled, we only queue 1 request
 399          if (Server->Pipeline == false)
 400             break;
 401          // if we have no hashes, do at most one such request
 402          // as we can't fixup pipeling misbehaviors otherwise
 403          else if (I->ExpectedHashes.usable() == false)
 404             break;
 405       }
 406
 407       // Make sure we stick with the same server
 408       if (Server->Comp(I->Uri) == false)
 409          break;
 410       if (QueueBack == I)
 411       {
 412          QueueBack = I->Next;
 413          SendReq(I);
 414          continue;
 415       }
 416    }
 417
 418    return true;
 419 }
 420                                                                         /*}}}*/
 421 // ServerMethod::Loop - Main loop                                       /*{{{*/
 422 int ServerMethod::Loop()
 423 {
 424    typedef vector<string> StringVector;
 425    typedef vector<string>::iterator StringVectorIterator;
 426    map<string, StringVector> Redirected;
 427
 428    signal(SIGTERM,SigTerm);
 429    signal(SIGINT,SigTerm);
 430
 431    Server = 0;
 432
 433    int FailCounter = 0;
 434    while (1)
 435    {
 436       // We have no commands, wait for some to arrive
 437       if (Queue == 0)
 438       {
 439          if (WaitFd(STDIN_FILENO) == false)
 440             return 0;
 441       }
 442
 443       /* Run messages, we can accept 0 (no message) if we didn't
 444          do a WaitFd above.. Otherwise the FD is closed. */
 445       int Result = Run(true);
 446       if (Result != -1 && (Result != 0 || Queue == 0))
 447       {
 448          if(FailReason.empty() == false ||
 449             _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
 450             return 100;
 451          else
 452             return 0;
 453       }
 454
 455       if (Queue == 0)
 456          continue;
 457
 458       // Connect to the server
 459       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 460       {
 461          delete Server;
 462          Server = CreateServerState(Queue->Uri);
 463       }
 464       /* If the server has explicitly said this is the last connection
 465          then we pre-emptively shut down the pipeline and tear down
 466          the connection. This will speed up HTTP/1.0 servers a tad
 467          since we don't have to wait for the close sequence to
 468          complete */
 469       if (Server->Persistent == false)
 470          Server->Close();
 471
 472       // Reset the pipeline
 473       if (Server->IsOpen() == false)
 474          QueueBack = Queue;
 475
 476       // Connnect to the host
 477       if (Server->Open() == false)
 478       {
 479          Fail(true);
 480          delete Server;
 481          Server = 0;
 482          continue;
 483       }
 484
 485       // Fill the pipeline.
 486       Fetch(0);
 487
 488       // Fetch the next URL header data from the server.
 489       switch (Server->RunHeaders(File, Queue->Uri))
 490       {
 491          case ServerState::RUN_HEADERS_OK:
 492          break;
 493
 494          // The header data is bad
 495          case ServerState::RUN_HEADERS_PARSE_ERROR:
 496          {
 497             _error->Error(_("Bad header data"));
 498             Fail(true);
 499             RotateDNS();
 500             continue;
 501          }
 502
 503          // The server closed a connection during the header get..
 504          default:
 505          case ServerState::RUN_HEADERS_IO_ERROR:
 506          {
 507             FailCounter++;
 508             _error->Discard();
 509             Server->Close();
 510             Server->Pipeline = false;
 511
 512             if (FailCounter >= 2)
 513             {
 514                Fail(_("Connection failed"),true);
 515                FailCounter = 0;
 516             }
 517
 518             RotateDNS();
 519             continue;
 520          }
 521       };
 522
 523       // Decide what to do.
 524       FetchResult Res;
 525       Res.Filename = Queue->DestFile;
 526       switch (DealWithHeaders(Res))
 527       {
 528          // Ok, the file is Open
 529          case FILE_IS_OPEN:
 530          {
 531             URIStart(Res);
 532
 533             // Run the data
 534             bool Result = true;
 535
 536             // ensure we don't fetch too much
 537             // we could do "Server->MaximumSize = Queue->MaximumSize" here
 538             // but that would break the clever pipeline messup detection
 539             // so instead we use the size of the biggest item in the queue
 540             Server->MaximumSize = FindMaximumObjectSizeInQueue();
 541
 542             if (Server->HaveContent)
 543                Result = Server->RunData(File);
 544
 545             /* If the server is sending back sizeless responses then fill in
 546                the size now */
 547             if (Res.Size == 0)
 548                Res.Size = File->Size();
 549
 550             // Close the file, destroy the FD object and timestamp it
 551             FailFd = -1;
 552             delete File;
 553             File = 0;
 554
 555             // Timestamp
 556             struct timeval times[2];
 557             times[0].tv_sec = times[1].tv_sec = Server->Date;
 558             times[0].tv_usec = times[1].tv_usec = 0;
 559             utimes(Queue->DestFile.c_str(), times);
 560
 561             // Send status to APT
 562             if (Result == true)
 563             {
 564                Hashes * const resultHashes = Server->GetHashes();
 565                HashStringList const hashList = resultHashes->GetHashStringList();
 566                if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
 567                {
 568                   // we did not get the expected hash… mhhh:
 569                   // could it be that server/proxy messed up pipelining?
 570                   FetchItem * BeforeI = Queue;
 571                   for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
 572                   {
 573                      if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
 574                      {
 575                         // yes, he did! Disable pipelining and rewrite queue
 576                         if (Server->Pipeline == true)
 577                         {
 578                            // FIXME: fake a warning message as we have no proper way of communicating here
 579                            std::string out;
 580                            strprintf(out, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::PipelineDepth");
 581                            std::cerr << "W: " << out << std::endl;
 582                            Server->Pipeline = false;
 583                            // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
 584                         }
 585                         Rename(Res.Filename, I->DestFile);
 586                         Res.Filename = I->DestFile;
 587                         BeforeI->Next = I->Next;
 588                         I->Next = Queue;
 589                         Queue = I;
 590                         break;
 591                      }
 592                      BeforeI = I;
 593                   }
 594                }
 595                Res.TakeHashes(*resultHashes);
 596                URIDone(Res);
 597             }
 598             else
 599             {
 600                if (Server->IsOpen() == false)
 601                {
 602                   FailCounter++;
 603                   _error->Discard();
 604                   Server->Close();
 605
 606                   if (FailCounter >= 2)
 607                   {
 608                      Fail(_("Connection failed"),true);
 609                      FailCounter = 0;
 610                   }
 611
 612                   QueueBack = Queue;
 613                }
 614                else
 615                {
 616                   Server->Close();
 617                   Fail(true);
 618                }
 619             }
 620             break;
 621          }
 622
 623          // IMS hit
 624          case IMS_HIT:
 625          {
 626             URIDone(Res);
 627             break;
 628          }
 629
 630          // Hard server error, not found or something
 631          case ERROR_UNRECOVERABLE:
 632          {
 633             Fail();
 634             break;
 635          }
 636
 637          // Hard internal error, kill the connection and fail
 638          case ERROR_NOT_FROM_SERVER:
 639          {
 640             delete File;
 641             File = 0;
 642
 643             Fail();
 644             RotateDNS();
 645             Server->Close();
 646             break;
 647          }
 648
 649          // We need to flush the data, the header is like a 404 w/ error text
 650          case ERROR_WITH_CONTENT_PAGE:
 651          {
 652             Fail();
 653
 654             // Send to content to dev/null
 655             File = new FileFd("/dev/null",FileFd::WriteExists);
 656             Server->RunData(File);
 657             delete File;
 658             File = 0;
 659             break;
 660          }
 661
 662          // Try again with a new URL
 663          case TRY_AGAIN_OR_REDIRECT:
 664          {
 665             // Clear rest of response if there is content
 666             if (Server->HaveContent)
 667             {
 668                File = new FileFd("/dev/null",FileFd::WriteExists);
 669                Server->RunData(File);
 670                delete File;
 671                File = 0;
 672             }
 673
 674             /* Detect redirect loops.  No more redirects are allowed
 675                after the same URI is seen twice in a queue item. */
 676             StringVector &R = Redirected[Queue->DestFile];
 677             bool StopRedirects = false;
 678             if (R.empty() == true)
 679                R.push_back(Queue->Uri);
 680             else if (R[0] == "STOP" || R.size() > 10)
 681                StopRedirects = true;
 682             else
 683             {
 684                for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
 685                   if (Queue->Uri == *I)
 686                   {
 687                      R[0] = "STOP";
 688                      break;
 689                   }
 690
 691                R.push_back(Queue->Uri);
 692             }
 693
 694             if (StopRedirects == false)
 695                Redirect(NextURI);
 696             else
 697                Fail();
 698
 699             break;
 700          }
 701
 702          default:
 703          Fail(_("Internal error"));
 704          break;
 705       }
 706
 707       FailCounter = 0;
 708    }
 709
 710    return 0;
 711 }
 712                                                                         /*}}}*/
 713                                                                         /*{{{*/
 714 unsigned long long
 715 ServerMethod::FindMaximumObjectSizeInQueue() const
 716 {
 717    unsigned long long MaxSizeInQueue = 0;
 718    for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
 719       MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
 720    return MaxSizeInQueue;
 721 }
 722                                                                         /*}}}*/