methods/server.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 /* ######################################################################
   4
   5    HTTP and HTTPS share a lot of common code and these classes are
   6    exactly the dumping ground for this common code
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include <config.h>
  12
  13 #include <apt-pkg/acquire-method.h>
  14 #include <apt-pkg/configuration.h>
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/fileutl.h>
  17 #include <apt-pkg/strutl.h>
  18
  19 #include <ctype.h>
  20 #include <signal.h>
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <sys/stat.h>
  24 #include <sys/time.h>
  25 #include <time.h>
  26 #include <unistd.h>
  27 #include <iostream>
  28 #include <limits>
  29 #include <map>
  30 #include <string>
  31 #include <vector>
  32
  33 #include "server.h"
  34
  35 #include <apti18n.h>
  36                                                                         /*}}}*/
  37 using namespace std;
  38
  39 string ServerMethod::FailFile;
  40 int ServerMethod::FailFd = -1;
  41 time_t ServerMethod::FailTime = 0;
  42
  43 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
  46    parse error occurred */
  47 ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
  48                                                       const std::string &Uri)
  49 {
  50    State = Header;
  51
  52    Owner->Status(_("Waiting for headers"));
  53
  54    Major = 0;
  55    Minor = 0;
  56    Result = 0;
  57    Size = 0;
  58    JunkSize = 0;
  59    StartPos = 0;
  60    Encoding = Closes;
  61    HaveContent = false;
  62    time(&Date);
  63
  64    do
  65    {
  66       string Data;
  67       if (ReadHeaderLines(Data) == false)
  68          continue;
  69
  70       if (Owner->Debug == true)
  71          clog << "Answer for: " << Uri << endl << Data;
  72
  73       for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
  74       {
  75          string::const_iterator J = I;
  76          for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
  77          if (HeaderLine(string(I,J)) == false)
  78             return RUN_HEADERS_PARSE_ERROR;
  79          I = J;
  80       }
  81
  82       // 100 Continue is a Nop...
  83       if (Result == 100)
  84          continue;
  85
  86       // Tidy up the connection persistence state.
  87       if (Encoding == Closes && HaveContent == true)
  88          Persistent = false;
  89
  90       return RUN_HEADERS_OK;
  91    }
  92    while (LoadNextResponse(false, File) == true);
  93
  94    return RUN_HEADERS_IO_ERROR;
  95 }
  96                                                                         /*}}}*/
  97 // ServerState::HeaderLine - Process a header line                      /*{{{*/
  98 // ---------------------------------------------------------------------
  99 /* */
 100 bool ServerState::HeaderLine(string Line)
 101 {
 102    if (Line.empty() == true)
 103       return true;
 104
 105    string::size_type Pos = Line.find(' ');
 106    if (Pos == string::npos || Pos+1 > Line.length())
 107    {
 108       // Blah, some servers use "connection:closes", evil.
 109       Pos = Line.find(':');
 110       if (Pos == string::npos || Pos + 2 > Line.length())
 111          return _error->Error(_("Bad header line"));
 112       Pos++;
 113    }
 114
 115    // Parse off any trailing spaces between the : and the next word.
 116    string::size_type Pos2 = Pos;
 117    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 118       Pos2++;
 119
 120    string Tag = string(Line,0,Pos);
 121    string Val = string(Line,Pos2);
 122
 123    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 124    {
 125       // Evil servers return no version
 126       if (Line[4] == '/')
 127       {
 128          int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
 129          if (elements == 3)
 130          {
 131             Code[0] = '\0';
 132             if (Owner != NULL && Owner->Debug == true)
 133                clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
 134          }
 135          else if (elements != 4)
 136             return _error->Error(_("The HTTP server sent an invalid reply header"));
 137       }
 138       else
 139       {
 140          Major = 0;
 141          Minor = 9;
 142          if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
 143             return _error->Error(_("The HTTP server sent an invalid reply header"));
 144       }
 145
 146       /* Check the HTTP response header to get the default persistence
 147          state. */
 148       if (Major < 1)
 149          Persistent = false;
 150       else
 151       {
 152          if (Major == 1 && Minor == 0)
 153             Persistent = false;
 154          else
 155             Persistent = true;
 156       }
 157
 158       return true;
 159    }
 160
 161    if (stringcasecmp(Tag,"Content-Length:") == 0)
 162    {
 163       if (Encoding == Closes)
 164          Encoding = Stream;
 165       HaveContent = true;
 166
 167       unsigned long long * SizePtr = &Size;
 168       if (Result == 416)
 169          SizePtr = &JunkSize;
 170
 171       *SizePtr = strtoull(Val.c_str(), NULL, 10);
 172       if (*SizePtr >= std::numeric_limits<unsigned long long>::max())
 173          return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
 174       else if (*SizePtr == 0)
 175          HaveContent = false;
 176       return true;
 177    }
 178
 179    if (stringcasecmp(Tag,"Content-Type:") == 0)
 180    {
 181       HaveContent = true;
 182       return true;
 183    }
 184
 185    if (stringcasecmp(Tag,"Content-Range:") == 0)
 186    {
 187       HaveContent = true;
 188
 189       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
 190       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
 191          ; // we got the expected filesize which is all we wanted
 192       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
 193          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 194       if ((unsigned long long)StartPos > Size)
 195          return _error->Error(_("This HTTP server has broken range support"));
 196       return true;
 197    }
 198
 199    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 200    {
 201       HaveContent = true;
 202       if (stringcasecmp(Val,"chunked") == 0)
 203          Encoding = Chunked;
 204       return true;
 205    }
 206
 207    if (stringcasecmp(Tag,"Connection:") == 0)
 208    {
 209       if (stringcasecmp(Val,"close") == 0)
 210          Persistent = false;
 211       if (stringcasecmp(Val,"keep-alive") == 0)
 212          Persistent = true;
 213       return true;
 214    }
 215
 216    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 217    {
 218       if (RFC1123StrToTime(Val.c_str(), Date) == false)
 219          return _error->Error(_("Unknown date format"));
 220       return true;
 221    }
 222
 223    if (stringcasecmp(Tag,"Location:") == 0)
 224    {
 225       Location = Val;
 226       return true;
 227    }
 228
 229    return true;
 230 }
 231                                                                         /*}}}*/
 232 // ServerState::ServerState - Constructor                               /*{{{*/
 233 ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOut(120), Owner(Owner)
 234 {
 235    Reset();
 236 }
 237                                                                         /*}}}*/
 238 bool ServerState::AddPartialFileToHashes(FileFd &File)                  /*{{{*/
 239 {
 240    File.Truncate(StartPos);
 241    return GetHashes()->AddFD(File, StartPos);
 242 }
 243                                                                         /*}}}*/
 244
 245 bool ServerMethod::Configuration(string Message)                        /*{{{*/
 246 {
 247    if (pkgAcqMethod::Configuration(Message) == false)
 248       return false;
 249
 250    DropPrivsOrDie();
 251
 252    return true;
 253 }
 254                                                                         /*}}}*/
 255
 256 // ServerMethod::DealWithHeaders - Handle the retrieved header data     /*{{{*/
 257 // ---------------------------------------------------------------------
 258 /* We look at the header data we got back from the server and decide what
 259    to do. Returns DealWithHeadersResult (see http.h for details).
 260  */
 261 ServerMethod::DealWithHeadersResult
 262 ServerMethod::DealWithHeaders(FetchResult &Res)
 263 {
 264    // Not Modified
 265    if (Server->Result == 304)
 266    {
 267       unlink(Queue->DestFile.c_str());
 268       Res.IMSHit = true;
 269       Res.LastModified = Queue->LastModified;
 270       return IMS_HIT;
 271    }
 272
 273    /* Redirect
 274     *
 275     * Note that it is only OK for us to treat all redirection the same
 276     * because we *always* use GET, not other HTTP methods.  There are
 277     * three redirection codes for which it is not appropriate that we
 278     * redirect.  Pass on those codes so the error handling kicks in.
 279     */
 280    if (AllowRedirect
 281        && (Server->Result > 300 && Server->Result < 400)
 282        && (Server->Result != 300       // Multiple Choices
 283            && Server->Result != 304    // Not Modified
 284            && Server->Result != 306))  // (Not part of HTTP/1.1, reserved)
 285    {
 286       if (Server->Location.empty() == true);
 287       else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
 288       {
 289          URI Uri = Queue->Uri;
 290          if (Uri.Host.empty() == false)
 291             NextURI = URI::SiteOnly(Uri);
 292          else
 293             NextURI.clear();
 294          NextURI.append(DeQuoteString(Server->Location));
 295          return TRY_AGAIN_OR_REDIRECT;
 296       }
 297       else
 298       {
 299          NextURI = DeQuoteString(Server->Location);
 300          URI tmpURI = NextURI;
 301          URI Uri = Queue->Uri;
 302          // same protocol redirects are okay
 303          if (tmpURI.Access == Uri.Access)
 304             return TRY_AGAIN_OR_REDIRECT;
 305          // as well as http to https
 306          else if (Uri.Access == "http" && tmpURI.Access == "https")
 307             return TRY_AGAIN_OR_REDIRECT;
 308       }
 309       /* else pass through for error message */
 310    }
 311    // retry after an invalid range response without partial data
 312    else if (Server->Result == 416)
 313    {
 314       struct stat SBuf;
 315       if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 316       {
 317          if ((unsigned long long)SBuf.st_size == Server->Size)
 318          {
 319             // the file is completely downloaded, but was not moved
 320             if (Server->HaveContent == true)
 321             {
 322                // Send to error page to dev/null
 323                FileFd DevNull("/dev/null",FileFd::WriteExists);
 324                Server->RunData(&DevNull);
 325             }
 326             Server->HaveContent = false;
 327             Server->StartPos = Server->Size;
 328             Server->Result = 200;
 329          }
 330          else if (unlink(Queue->DestFile.c_str()) == 0)
 331          {
 332             NextURI = Queue->Uri;
 333             return TRY_AGAIN_OR_REDIRECT;
 334          }
 335       }
 336    }
 337
 338    /* We have a reply we dont handle. This should indicate a perm server
 339       failure */
 340    if (Server->Result < 200 || Server->Result >= 300)
 341    {
 342       std::string err;
 343       strprintf(err, "HttpError%u", Server->Result);
 344       SetFailReason(err);
 345       _error->Error("%u %s", Server->Result, Server->Code);
 346       if (Server->HaveContent == true)
 347          return ERROR_WITH_CONTENT_PAGE;
 348       return ERROR_UNRECOVERABLE;
 349    }
 350
 351    // This is some sort of 2xx 'data follows' reply
 352    Res.LastModified = Server->Date;
 353    Res.Size = Server->Size;
 354
 355    // Open the file
 356    delete File;
 357    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 358    if (_error->PendingError() == true)
 359       return ERROR_NOT_FROM_SERVER;
 360
 361    FailFile = Queue->DestFile;
 362    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 363    FailFd = File->Fd();
 364    FailTime = Server->Date;
 365
 366    if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
 367    {
 368       _error->Errno("read",_("Problem hashing file"));
 369       return ERROR_NOT_FROM_SERVER;
 370    }
 371    if (Server->StartPos > 0)
 372       Res.ResumePoint = Server->StartPos;
 373
 374    SetNonBlock(File->Fd(),true);
 375    return FILE_IS_OPEN;
 376 }
 377                                                                         /*}}}*/
 378 // ServerMethod::SigTerm - Handle a fatal signal                        /*{{{*/
 379 // ---------------------------------------------------------------------
 380 /* This closes and timestamps the open file. This is necessary to get
 381    resume behavoir on user abort */
 382 void ServerMethod::SigTerm(int)
 383 {
 384    if (FailFd == -1)
 385       _exit(100);
 386
 387    struct timeval times[2];
 388    times[0].tv_sec = FailTime;
 389    times[1].tv_sec = FailTime;
 390    times[0].tv_usec = times[1].tv_usec = 0;
 391    utimes(FailFile.c_str(), times);
 392    close(FailFd);
 393
 394    _exit(100);
 395 }
 396                                                                         /*}}}*/
 397 // ServerMethod::Fetch - Fetch an item                                  /*{{{*/
 398 // ---------------------------------------------------------------------
 399 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 400    depth. */
 401 bool ServerMethod::Fetch(FetchItem *)
 402 {
 403    if (Server == 0)
 404       return true;
 405
 406    // Queue the requests
 407    int Depth = -1;
 408    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 409         I = I->Next, Depth++)
 410    {
 411       if (Depth >= 0)
 412       {
 413          // If pipelining is disabled, we only queue 1 request
 414          if (Server->Pipeline == false)
 415             break;
 416          // if we have no hashes, do at most one such request
 417          // as we can't fixup pipeling misbehaviors otherwise
 418          else if (I->ExpectedHashes.usable() == false)
 419             break;
 420       }
 421
 422       // Make sure we stick with the same server
 423       if (Server->Comp(I->Uri) == false)
 424          break;
 425       if (QueueBack == I)
 426       {
 427          QueueBack = I->Next;
 428          SendReq(I);
 429          continue;
 430       }
 431    }
 432
 433    return true;
 434 }
 435                                                                         /*}}}*/
 436 // ServerMethod::Loop - Main loop                                       /*{{{*/
 437 int ServerMethod::Loop()
 438 {
 439    typedef vector<string> StringVector;
 440    typedef vector<string>::iterator StringVectorIterator;
 441    map<string, StringVector> Redirected;
 442
 443    signal(SIGTERM,SigTerm);
 444    signal(SIGINT,SigTerm);
 445
 446    Server = 0;
 447
 448    int FailCounter = 0;
 449    while (1)
 450    {
 451       // We have no commands, wait for some to arrive
 452       if (Queue == 0)
 453       {
 454          if (WaitFd(STDIN_FILENO) == false)
 455             return 0;
 456       }
 457
 458       /* Run messages, we can accept 0 (no message) if we didn't
 459          do a WaitFd above.. Otherwise the FD is closed. */
 460       int Result = Run(true);
 461       if (Result != -1 && (Result != 0 || Queue == 0))
 462       {
 463          if(FailReason.empty() == false ||
 464             _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
 465             return 100;
 466          else
 467             return 0;
 468       }
 469
 470       if (Queue == 0)
 471          continue;
 472
 473       // Connect to the server
 474       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 475       {
 476          delete Server;
 477          Server = CreateServerState(Queue->Uri);
 478       }
 479       /* If the server has explicitly said this is the last connection
 480          then we pre-emptively shut down the pipeline and tear down
 481          the connection. This will speed up HTTP/1.0 servers a tad
 482          since we don't have to wait for the close sequence to
 483          complete */
 484       if (Server->Persistent == false)
 485          Server->Close();
 486
 487       // Reset the pipeline
 488       if (Server->IsOpen() == false)
 489          QueueBack = Queue;
 490
 491       // Connnect to the host
 492       if (Server->Open() == false)
 493       {
 494          Fail(true);
 495          delete Server;
 496          Server = 0;
 497          continue;
 498       }
 499
 500       // Fill the pipeline.
 501       Fetch(0);
 502
 503       // Fetch the next URL header data from the server.
 504       switch (Server->RunHeaders(File, Queue->Uri))
 505       {
 506          case ServerState::RUN_HEADERS_OK:
 507          break;
 508
 509          // The header data is bad
 510          case ServerState::RUN_HEADERS_PARSE_ERROR:
 511          {
 512             _error->Error(_("Bad header data"));
 513             Fail(true);
 514             RotateDNS();
 515             continue;
 516          }
 517
 518          // The server closed a connection during the header get..
 519          default:
 520          case ServerState::RUN_HEADERS_IO_ERROR:
 521          {
 522             FailCounter++;
 523             _error->Discard();
 524             Server->Close();
 525             Server->Pipeline = false;
 526
 527             if (FailCounter >= 2)
 528             {
 529                Fail(_("Connection failed"),true);
 530                FailCounter = 0;
 531             }
 532
 533             RotateDNS();
 534             continue;
 535          }
 536       };
 537
 538       // Decide what to do.
 539       FetchResult Res;
 540       Res.Filename = Queue->DestFile;
 541       switch (DealWithHeaders(Res))
 542       {
 543          // Ok, the file is Open
 544          case FILE_IS_OPEN:
 545          {
 546             URIStart(Res);
 547
 548             // Run the data
 549             bool Result = true;
 550
 551             // ensure we don't fetch too much
 552             // we could do "Server->MaximumSize = Queue->MaximumSize" here
 553             // but that would break the clever pipeline messup detection
 554             // so instead we use the size of the biggest item in the queue
 555             Server->MaximumSize = FindMaximumObjectSizeInQueue();
 556
 557             if (Server->HaveContent)
 558                Result = Server->RunData(File);
 559
 560             /* If the server is sending back sizeless responses then fill in
 561                the size now */
 562             if (Res.Size == 0)
 563                Res.Size = File->Size();
 564
 565             // Close the file, destroy the FD object and timestamp it
 566             FailFd = -1;
 567             delete File;
 568             File = 0;
 569
 570             // Timestamp
 571             struct timeval times[2];
 572             times[0].tv_sec = times[1].tv_sec = Server->Date;
 573             times[0].tv_usec = times[1].tv_usec = 0;
 574             utimes(Queue->DestFile.c_str(), times);
 575
 576             // Send status to APT
 577             if (Result == true)
 578             {
 579                Hashes * const resultHashes = Server->GetHashes();
 580                HashStringList const hashList = resultHashes->GetHashStringList();
 581                if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
 582                {
 583                   // we did not get the expected hash… mhhh:
 584                   // could it be that server/proxy messed up pipelining?
 585                   FetchItem * BeforeI = Queue;
 586                   for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
 587                   {
 588                      if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
 589                      {
 590                         // yes, he did! Disable pipelining and rewrite queue
 591                         if (Server->Pipeline == true)
 592                         {
 593                            // FIXME: fake a warning message as we have no proper way of communicating here
 594                            std::string out;
 595                            strprintf(out, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::PipelineDepth");
 596                            std::cerr << "W: " << out << std::endl;
 597                            Server->Pipeline = false;
 598                            // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
 599                         }
 600                         Rename(Res.Filename, I->DestFile);
 601                         Res.Filename = I->DestFile;
 602                         BeforeI->Next = I->Next;
 603                         I->Next = Queue;
 604                         Queue = I;
 605                         break;
 606                      }
 607                      BeforeI = I;
 608                   }
 609                }
 610                Res.TakeHashes(*resultHashes);
 611                URIDone(Res);
 612             }
 613             else
 614             {
 615                if (Server->IsOpen() == false)
 616                {
 617                   FailCounter++;
 618                   _error->Discard();
 619                   Server->Close();
 620
 621                   if (FailCounter >= 2)
 622                   {
 623                      Fail(_("Connection failed"),true);
 624                      FailCounter = 0;
 625                   }
 626
 627                   QueueBack = Queue;
 628                }
 629                else
 630                {
 631                   Server->Close();
 632                   Fail(true);
 633                }
 634             }
 635             break;
 636          }
 637
 638          // IMS hit
 639          case IMS_HIT:
 640          {
 641             URIDone(Res);
 642             break;
 643          }
 644
 645          // Hard server error, not found or something
 646          case ERROR_UNRECOVERABLE:
 647          {
 648             Fail();
 649             break;
 650          }
 651
 652          // Hard internal error, kill the connection and fail
 653          case ERROR_NOT_FROM_SERVER:
 654          {
 655             delete File;
 656             File = 0;
 657
 658             Fail();
 659             RotateDNS();
 660             Server->Close();
 661             break;
 662          }
 663
 664          // We need to flush the data, the header is like a 404 w/ error text
 665          case ERROR_WITH_CONTENT_PAGE:
 666          {
 667             Fail();
 668
 669             // Send to content to dev/null
 670             File = new FileFd("/dev/null",FileFd::WriteExists);
 671             Server->RunData(File);
 672             delete File;
 673             File = 0;
 674             break;
 675          }
 676
 677          // Try again with a new URL
 678          case TRY_AGAIN_OR_REDIRECT:
 679          {
 680             // Clear rest of response if there is content
 681             if (Server->HaveContent)
 682             {
 683                File = new FileFd("/dev/null",FileFd::WriteExists);
 684                Server->RunData(File);
 685                delete File;
 686                File = 0;
 687             }
 688
 689             /* Detect redirect loops.  No more redirects are allowed
 690                after the same URI is seen twice in a queue item. */
 691             StringVector &R = Redirected[Queue->DestFile];
 692             bool StopRedirects = false;
 693             if (R.empty() == true)
 694                R.push_back(Queue->Uri);
 695             else if (R[0] == "STOP" || R.size() > 10)
 696                StopRedirects = true;
 697             else
 698             {
 699                for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
 700                   if (Queue->Uri == *I)
 701                   {
 702                      R[0] = "STOP";
 703                      break;
 704                   }
 705
 706                R.push_back(Queue->Uri);
 707             }
 708
 709             if (StopRedirects == false)
 710                Redirect(NextURI);
 711             else
 712                Fail();
 713
 714             break;
 715          }
 716
 717          default:
 718          Fail(_("Internal error"));
 719          break;
 720       }
 721
 722       FailCounter = 0;
 723    }
 724
 725    return 0;
 726 }
 727                                                                         /*}}}*/
 728                                                                         /*{{{*/
 729 unsigned long long
 730 ServerMethod::FindMaximumObjectSizeInQueue() const
 731 {
 732    unsigned long long MaxSizeInQueue = 0;
 733    for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
 734       MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
 735    return MaxSizeInQueue;
 736 }
 737                                                                         /*}}}*/