methods/server.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 /* ######################################################################
   4
   5    HTTP and HTTPS share a lot of common code and these classes are
   6    exactly the dumping ground for this common code
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include <config.h>
  12
  13 #include <apt-pkg/acquire-method.h>
  14 #include <apt-pkg/configuration.h>
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/fileutl.h>
  17 #include <apt-pkg/strutl.h>
  18
  19 #include <ctype.h>
  20 #include <signal.h>
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <sys/stat.h>
  24 #include <sys/time.h>
  25 #include <time.h>
  26 #include <unistd.h>
  27 #include <iostream>
  28 #include <limits>
  29 #include <map>
  30 #include <string>
  31 #include <vector>
  32
  33 #include "server.h"
  34
  35 #include <apti18n.h>
  36                                                                         /*}}}*/
  37 using namespace std;
  38
  39 string ServerMethod::FailFile;
  40 int ServerMethod::FailFd = -1;
  41 time_t ServerMethod::FailTime = 0;
  42
  43 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
  46    parse error occurred */
  47 ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
  48                                                       const std::string &Uri)
  49 {
  50    State = Header;
  51
  52    Owner->Status(_("Waiting for headers"));
  53
  54    Major = 0;
  55    Minor = 0;
  56    Result = 0;
  57    Size = 0;
  58    JunkSize = 0;
  59    StartPos = 0;
  60    Encoding = Closes;
  61    HaveContent = false;
  62    time(&Date);
  63
  64    do
  65    {
  66       string Data;
  67       if (ReadHeaderLines(Data) == false)
  68          continue;
  69
  70       if (Owner->Debug == true)
  71          clog << "Answer for: " << Uri << endl << Data;
  72
  73       for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
  74       {
  75          string::const_iterator J = I;
  76          for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
  77          if (HeaderLine(string(I,J)) == false)
  78             return RUN_HEADERS_PARSE_ERROR;
  79          I = J;
  80       }
  81
  82       // 100 Continue is a Nop...
  83       if (Result == 100)
  84          continue;
  85
  86       // Tidy up the connection persistence state.
  87       if (Encoding == Closes && HaveContent == true)
  88          Persistent = false;
  89
  90       return RUN_HEADERS_OK;
  91    }
  92    while (LoadNextResponse(false, File) == true);
  93
  94    return RUN_HEADERS_IO_ERROR;
  95 }
  96                                                                         /*}}}*/
  97 // ServerState::HeaderLine - Process a header line                      /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* */
 100 bool ServerState::HeaderLine(string Line)
 101 {
 102    if (Line.empty() == true)
 103       return true;
 104
 105    string::size_type Pos = Line.find(' ');
 106    if (Pos == string::npos || Pos+1 > Line.length())
 107    {
 108       // Blah, some servers use "connection:closes", evil.
 109       Pos = Line.find(':');
 110       if (Pos == string::npos || Pos + 2 > Line.length())
 111          return _error->Error(_("Bad header line"));
 112       Pos++;
 113    }
 114
 115    // Parse off any trailing spaces between the : and the next word.
 116    string::size_type Pos2 = Pos;
 117    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 118       Pos2++;
 119
 120    string Tag = string(Line,0,Pos);
 121    string Val = string(Line,Pos2);
 122
 123    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 124    {
 125       // Evil servers return no version
 126       if (Line[4] == '/')
 127       {
 128          int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
 129          if (elements == 3)
 130          {
 131             Code[0] = '\0';
 132             if (Owner->Debug == true)
 133                clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
 134          }
 135          else if (elements != 4)
 136             return _error->Error(_("The HTTP server sent an invalid reply header"));
 137       }
 138       else
 139       {
 140          Major = 0;
 141          Minor = 9;
 142          if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
 143             return _error->Error(_("The HTTP server sent an invalid reply header"));
 144       }
 145
 146       /* Check the HTTP response header to get the default persistence
 147          state. */
 148       if (Major < 1)
 149          Persistent = false;
 150       else
 151       {
 152          if (Major == 1 && Minor == 0)
 153             Persistent = false;
 154          else
 155             Persistent = true;
 156       }
 157
 158       return true;
 159    }
 160
 161    if (stringcasecmp(Tag,"Content-Length:") == 0)
 162    {
 163       if (Encoding == Closes)
 164          Encoding = Stream;
 165       HaveContent = true;
 166
 167       unsigned long long * SizePtr = &Size;
 168       if (Result == 416)
 169          SizePtr = &JunkSize;
 170
 171       *SizePtr = strtoull(Val.c_str(), NULL, 10);
 172       if (*SizePtr >= std::numeric_limits<unsigned long long>::max())
 173          return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
 174       else if (*SizePtr == 0)
 175          HaveContent = false;
 176       return true;
 177    }
 178
 179    if (stringcasecmp(Tag,"Content-Type:") == 0)
 180    {
 181       HaveContent = true;
 182       return true;
 183    }
 184
 185    if (stringcasecmp(Tag,"Content-Range:") == 0)
 186    {
 187       HaveContent = true;
 188
 189       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
 190       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
 191          ; // we got the expected filesize which is all we wanted
 192       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
 193          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 194       if ((unsigned long long)StartPos > Size)
 195          return _error->Error(_("This HTTP server has broken range support"));
 196       return true;
 197    }
 198
 199    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 200    {
 201       HaveContent = true;
 202       if (stringcasecmp(Val,"chunked") == 0)
 203          Encoding = Chunked;
 204       return true;
 205    }
 206
 207    if (stringcasecmp(Tag,"Connection:") == 0)
 208    {
 209       if (stringcasecmp(Val,"close") == 0)
 210          Persistent = false;
 211       if (stringcasecmp(Val,"keep-alive") == 0)
 212          Persistent = true;
 213       return true;
 214    }
 215
 216    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 217    {
 218       if (RFC1123StrToTime(Val.c_str(), Date) == false)
 219          return _error->Error(_("Unknown date format"));
 220       return true;
 221    }
 222
 223    if (stringcasecmp(Tag,"Location:") == 0)
 224    {
 225       Location = Val;
 226       return true;
 227    }
 228
 229    return true;
 230 }
 231                                                                         /*}}}*/
 232 // ServerState::ServerState - Constructor                               /*{{{*/
 233 ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOut(120), Owner(Owner)
 234 {
 235    Reset();
 236 }
 237                                                                         /*}}}*/
 238
 239 bool ServerMethod::Configuration(string Message)                        /*{{{*/
 240 {
 241    return pkgAcqMethod::Configuration(Message);
 242 }
 243                                                                         /*}}}*/
 244
 245 // ServerMethod::DealWithHeaders - Handle the retrieved header data     /*{{{*/
 246 // ---------------------------------------------------------------------
 247 /* We look at the header data we got back from the server and decide what
 248    to do. Returns DealWithHeadersResult (see http.h for details).
 249  */
 250 ServerMethod::DealWithHeadersResult
 251 ServerMethod::DealWithHeaders(FetchResult &Res)
 252 {
 253    // Not Modified
 254    if (Server->Result == 304)
 255    {
 256       unlink(Queue->DestFile.c_str());
 257       Res.IMSHit = true;
 258       Res.LastModified = Queue->LastModified;
 259       return IMS_HIT;
 260    }
 261
 262    /* Redirect
 263     *
 264     * Note that it is only OK for us to treat all redirection the same
 265     * because we *always* use GET, not other HTTP methods.  There are
 266     * three redirection codes for which it is not appropriate that we
 267     * redirect.  Pass on those codes so the error handling kicks in.
 268     */
 269    if (AllowRedirect
 270        && (Server->Result > 300 && Server->Result < 400)
 271        && (Server->Result != 300       // Multiple Choices
 272            && Server->Result != 304    // Not Modified
 273            && Server->Result != 306))  // (Not part of HTTP/1.1, reserved)
 274    {
 275       if (Server->Location.empty() == true);
 276       else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
 277       {
 278          URI Uri = Queue->Uri;
 279          if (Uri.Host.empty() == false)
 280             NextURI = URI::SiteOnly(Uri);
 281          else
 282             NextURI.clear();
 283          NextURI.append(DeQuoteString(Server->Location));
 284          return TRY_AGAIN_OR_REDIRECT;
 285       }
 286       else
 287       {
 288          NextURI = DeQuoteString(Server->Location);
 289          URI tmpURI = NextURI;
 290          URI Uri = Queue->Uri;
 291          // same protocol redirects are okay
 292          if (tmpURI.Access == Uri.Access)
 293             return TRY_AGAIN_OR_REDIRECT;
 294          // as well as http to https
 295          else if (Uri.Access == "http" && tmpURI.Access == "https")
 296             return TRY_AGAIN_OR_REDIRECT;
 297       }
 298       /* else pass through for error message */
 299    }
 300    // retry after an invalid range response without partial data
 301    else if (Server->Result == 416)
 302    {
 303       struct stat SBuf;
 304       if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 305       {
 306          if ((unsigned long long)SBuf.st_size == Server->Size)
 307          {
 308             // the file is completely downloaded, but was not moved
 309             if (Server->HaveContent == true)
 310             {
 311                // Send to error page to dev/null
 312                FileFd DevNull("/dev/null",FileFd::WriteExists);
 313                Server->RunData(&DevNull);
 314             }
 315             Server->HaveContent = false;
 316             Server->StartPos = Server->Size;
 317             Server->Result = 200;
 318          }
 319          else if (unlink(Queue->DestFile.c_str()) == 0)
 320          {
 321             NextURI = Queue->Uri;
 322             return TRY_AGAIN_OR_REDIRECT;
 323          }
 324       }
 325    }
 326
 327    /* We have a reply we dont handle. This should indicate a perm server
 328       failure */
 329    if (Server->Result < 200 || Server->Result >= 300)
 330    {
 331       std::string err;
 332       strprintf(err, "HttpError%u", Server->Result);
 333       SetFailReason(err);
 334       _error->Error("%u %s", Server->Result, Server->Code);
 335       if (Server->HaveContent == true)
 336          return ERROR_WITH_CONTENT_PAGE;
 337       return ERROR_UNRECOVERABLE;
 338    }
 339
 340    // This is some sort of 2xx 'data follows' reply
 341    Res.LastModified = Server->Date;
 342    Res.Size = Server->Size;
 343
 344    // Open the file
 345    delete File;
 346    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 347    if (_error->PendingError() == true)
 348       return ERROR_NOT_FROM_SERVER;
 349
 350    FailFile = Queue->DestFile;
 351    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 352    FailFd = File->Fd();
 353    FailTime = Server->Date;
 354
 355    if (Server->InitHashes(*File) == false)
 356    {
 357       _error->Errno("read",_("Problem hashing file"));
 358       return ERROR_NOT_FROM_SERVER;
 359    }
 360    if (Server->StartPos > 0)
 361       Res.ResumePoint = Server->StartPos;
 362
 363    SetNonBlock(File->Fd(),true);
 364    return FILE_IS_OPEN;
 365 }
 366                                                                         /*}}}*/
 367 // ServerMethod::SigTerm - Handle a fatal signal                        /*{{{*/
 368 // ---------------------------------------------------------------------
 369 /* This closes and timestamps the open file. This is necessary to get
 370    resume behavoir on user abort */
 371 void ServerMethod::SigTerm(int)
 372 {
 373    if (FailFd == -1)
 374       _exit(100);
 375
 376    struct timeval times[2];
 377    times[0].tv_sec = FailTime;
 378    times[1].tv_sec = FailTime;
 379    times[0].tv_usec = times[1].tv_usec = 0;
 380    utimes(FailFile.c_str(), times);
 381    close(FailFd);
 382
 383    _exit(100);
 384 }
 385                                                                         /*}}}*/
 386 // ServerMethod::Fetch - Fetch an item                                  /*{{{*/
 387 // ---------------------------------------------------------------------
 388 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 389    depth. */
 390 bool ServerMethod::Fetch(FetchItem *)
 391 {
 392    if (Server == 0)
 393       return true;
 394
 395    // Queue the requests
 396    int Depth = -1;
 397    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 398         I = I->Next, Depth++)
 399    {
 400       if (Depth >= 0)
 401       {
 402          // If pipelining is disabled, we only queue 1 request
 403          if (Server->Pipeline == false)
 404             break;
 405          // if we have no hashes, do at most one such request
 406          // as we can't fixup pipeling misbehaviors otherwise
 407          else if (I->ExpectedHashes.usable() == false)
 408             break;
 409       }
 410
 411       // Make sure we stick with the same server
 412       if (Server->Comp(I->Uri) == false)
 413          break;
 414       if (QueueBack == I)
 415       {
 416          QueueBack = I->Next;
 417          SendReq(I);
 418          continue;
 419       }
 420    }
 421
 422    return true;
 423 }
 424                                                                         /*}}}*/
 425 // ServerMethod::Loop - Main loop                                       /*{{{*/
 426 int ServerMethod::Loop()
 427 {
 428    typedef vector<string> StringVector;
 429    typedef vector<string>::iterator StringVectorIterator;
 430    map<string, StringVector> Redirected;
 431
 432    signal(SIGTERM,SigTerm);
 433    signal(SIGINT,SigTerm);
 434
 435    Server = 0;
 436
 437    int FailCounter = 0;
 438    while (1)
 439    {
 440       // We have no commands, wait for some to arrive
 441       if (Queue == 0)
 442       {
 443          if (WaitFd(STDIN_FILENO) == false)
 444             return 0;
 445       }
 446
 447       /* Run messages, we can accept 0 (no message) if we didn't
 448          do a WaitFd above.. Otherwise the FD is closed. */
 449       int Result = Run(true);
 450       if (Result != -1 && (Result != 0 || Queue == 0))
 451       {
 452          if(FailReason.empty() == false ||
 453             _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
 454             return 100;
 455          else
 456             return 0;
 457       }
 458
 459       if (Queue == 0)
 460          continue;
 461
 462       // Connect to the server
 463       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 464       {
 465          delete Server;
 466          Server = CreateServerState(Queue->Uri);
 467       }
 468       /* If the server has explicitly said this is the last connection
 469          then we pre-emptively shut down the pipeline and tear down
 470          the connection. This will speed up HTTP/1.0 servers a tad
 471          since we don't have to wait for the close sequence to
 472          complete */
 473       if (Server->Persistent == false)
 474          Server->Close();
 475
 476       // Reset the pipeline
 477       if (Server->IsOpen() == false)
 478          QueueBack = Queue;
 479
 480       // Connnect to the host
 481       if (Server->Open() == false)
 482       {
 483          Fail(true);
 484          delete Server;
 485          Server = 0;
 486          continue;
 487       }
 488
 489       // Fill the pipeline.
 490       Fetch(0);
 491
 492       // Fetch the next URL header data from the server.
 493       switch (Server->RunHeaders(File, Queue->Uri))
 494       {
 495          case ServerState::RUN_HEADERS_OK:
 496          break;
 497
 498          // The header data is bad
 499          case ServerState::RUN_HEADERS_PARSE_ERROR:
 500          {
 501             _error->Error(_("Bad header data"));
 502             Fail(true);
 503             RotateDNS();
 504             continue;
 505          }
 506
 507          // The server closed a connection during the header get..
 508          default:
 509          case ServerState::RUN_HEADERS_IO_ERROR:
 510          {
 511             FailCounter++;
 512             _error->Discard();
 513             Server->Close();
 514             Server->Pipeline = false;
 515
 516             if (FailCounter >= 2)
 517             {
 518                Fail(_("Connection failed"),true);
 519                FailCounter = 0;
 520             }
 521
 522             RotateDNS();
 523             continue;
 524          }
 525       };
 526
 527       // Decide what to do.
 528       FetchResult Res;
 529       Res.Filename = Queue->DestFile;
 530       switch (DealWithHeaders(Res))
 531       {
 532          // Ok, the file is Open
 533          case FILE_IS_OPEN:
 534          {
 535             URIStart(Res);
 536
 537             // Run the data
 538             bool Result = true;
 539
 540             // ensure we don't fetch too much
 541             // we could do "Server->MaximumSize = Queue->MaximumSize" here
 542             // but that would break the clever pipeline messup detection
 543             // so instead we use the size of the biggest item in the queue
 544             Server->MaximumSize = FindMaximumObjectSizeInQueue();
 545
 546             if (Server->HaveContent)
 547                Result = Server->RunData(File);
 548
 549             /* If the server is sending back sizeless responses then fill in
 550                the size now */
 551             if (Res.Size == 0)
 552                Res.Size = File->Size();
 553
 554             // Close the file, destroy the FD object and timestamp it
 555             FailFd = -1;
 556             delete File;
 557             File = 0;
 558
 559             // Timestamp
 560             struct timeval times[2];
 561             times[0].tv_sec = times[1].tv_sec = Server->Date;
 562             times[0].tv_usec = times[1].tv_usec = 0;
 563             utimes(Queue->DestFile.c_str(), times);
 564
 565             // Send status to APT
 566             if (Result == true)
 567             {
 568                Hashes * const resultHashes = Server->GetHashes();
 569                HashStringList const hashList = resultHashes->GetHashStringList();
 570                if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
 571                {
 572                   // we did not get the expected hash… mhhh:
 573                   // could it be that server/proxy messed up pipelining?
 574                   FetchItem * BeforeI = Queue;
 575                   for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
 576                   {
 577                      if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
 578                      {
 579                         // yes, he did! Disable pipelining and rewrite queue
 580                         if (Server->Pipeline == true)
 581                         {
 582                            // FIXME: fake a warning message as we have no proper way of communicating here
 583                            std::string out;
 584                            strprintf(out, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::PipelineDepth");
 585                            std::cerr << "W: " << out << std::endl;
 586                            Server->Pipeline = false;
 587                            // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
 588                         }
 589                         Rename(Res.Filename, I->DestFile);
 590                         Res.Filename = I->DestFile;
 591                         BeforeI->Next = I->Next;
 592                         I->Next = Queue;
 593                         Queue = I;
 594                         break;
 595                      }
 596                      BeforeI = I;
 597                   }
 598                }
 599                Res.TakeHashes(*resultHashes);
 600                URIDone(Res);
 601             }
 602             else
 603             {
 604                if (Server->IsOpen() == false)
 605                {
 606                   FailCounter++;
 607                   _error->Discard();
 608                   Server->Close();
 609
 610                   if (FailCounter >= 2)
 611                   {
 612                      Fail(_("Connection failed"),true);
 613                      FailCounter = 0;
 614                   }
 615
 616                   QueueBack = Queue;
 617                }
 618                else
 619                {
 620                   Server->Close();
 621                   Fail(true);
 622                }
 623             }
 624             break;
 625          }
 626
 627          // IMS hit
 628          case IMS_HIT:
 629          {
 630             URIDone(Res);
 631             break;
 632          }
 633
 634          // Hard server error, not found or something
 635          case ERROR_UNRECOVERABLE:
 636          {
 637             Fail();
 638             break;
 639          }
 640
 641          // Hard internal error, kill the connection and fail
 642          case ERROR_NOT_FROM_SERVER:
 643          {
 644             delete File;
 645             File = 0;
 646
 647             Fail();
 648             RotateDNS();
 649             Server->Close();
 650             break;
 651          }
 652
 653          // We need to flush the data, the header is like a 404 w/ error text
 654          case ERROR_WITH_CONTENT_PAGE:
 655          {
 656             Fail();
 657
 658             // Send to content to dev/null
 659             File = new FileFd("/dev/null",FileFd::WriteExists);
 660             Server->RunData(File);
 661             delete File;
 662             File = 0;
 663             break;
 664          }
 665
 666          // Try again with a new URL
 667          case TRY_AGAIN_OR_REDIRECT:
 668          {
 669             // Clear rest of response if there is content
 670             if (Server->HaveContent)
 671             {
 672                File = new FileFd("/dev/null",FileFd::WriteExists);
 673                Server->RunData(File);
 674                delete File;
 675                File = 0;
 676             }
 677
 678             /* Detect redirect loops.  No more redirects are allowed
 679                after the same URI is seen twice in a queue item. */
 680             StringVector &R = Redirected[Queue->DestFile];
 681             bool StopRedirects = false;
 682             if (R.empty() == true)
 683                R.push_back(Queue->Uri);
 684             else if (R[0] == "STOP" || R.size() > 10)
 685                StopRedirects = true;
 686             else
 687             {
 688                for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
 689                   if (Queue->Uri == *I)
 690                   {
 691                      R[0] = "STOP";
 692                      break;
 693                   }
 694
 695                R.push_back(Queue->Uri);
 696             }
 697
 698             if (StopRedirects == false)
 699                Redirect(NextURI);
 700             else
 701                Fail();
 702
 703             break;
 704          }
 705
 706          default:
 707          Fail(_("Internal error"));
 708          break;
 709       }
 710
 711       FailCounter = 0;
 712    }
 713
 714    return 0;
 715 }
 716                                                                         /*}}}*/
 717                                                                         /*{{{*/
 718 unsigned long long
 719 ServerMethod::FindMaximumObjectSizeInQueue() const
 720 {
 721    unsigned long long MaxSizeInQueue = 0;
 722    for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
 723       MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
 724    return MaxSizeInQueue;
 725 }
 726                                                                         /*}}}*/