methods/server.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 /* ######################################################################
   4
   5    HTTP and HTTPS share a lot of common code and these classes are
   6    exactly the dumping ground for this common code
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include <config.h>
  12
  13 #include <apt-pkg/acquire-method.h>
  14 #include <apt-pkg/configuration.h>
  15 #include <apt-pkg/error.h>
  16 #include <apt-pkg/fileutl.h>
  17 #include <apt-pkg/strutl.h>
  18
  19 #include <ctype.h>
  20 #include <signal.h>
  21 #include <stdio.h>
  22 #include <stdlib.h>
  23 #include <sys/stat.h>
  24 #include <sys/time.h>
  25 #include <time.h>
  26 #include <unistd.h>
  27 #include <iostream>
  28 #include <limits>
  29 #include <map>
  30 #include <string>
  31 #include <vector>
  32
  33 #include "server.h"
  34
  35 #include <apti18n.h>
  36                                                                         /*}}}*/
  37 using namespace std;
  38
  39 string ServerMethod::FailFile;
  40 int ServerMethod::FailFd = -1;
  41 time_t ServerMethod::FailTime = 0;
  42
  43 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
  44 // ---------------------------------------------------------------------
  45 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
  46    parse error occurred */
  47 ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File)
  48 {
  49    State = Header;
  50
  51    Owner->Status(_("Waiting for headers"));
  52
  53    Major = 0;
  54    Minor = 0;
  55    Result = 0;
  56    Size = 0;
  57    StartPos = 0;
  58    Encoding = Closes;
  59    HaveContent = false;
  60    time(&Date);
  61
  62    do
  63    {
  64       string Data;
  65       if (ReadHeaderLines(Data) == false)
  66          continue;
  67
  68       if (Owner->Debug == true)
  69          clog << Data;
  70
  71       for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
  72       {
  73          string::const_iterator J = I;
  74          for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
  75          if (HeaderLine(string(I,J)) == false)
  76             return RUN_HEADERS_PARSE_ERROR;
  77          I = J;
  78       }
  79
  80       // 100 Continue is a Nop...
  81       if (Result == 100)
  82          continue;
  83
  84       // Tidy up the connection persistence state.
  85       if (Encoding == Closes && HaveContent == true)
  86          Persistent = false;
  87
  88       return RUN_HEADERS_OK;
  89    }
  90    while (LoadNextResponse(false, File) == true);
  91
  92    return RUN_HEADERS_IO_ERROR;
  93 }
  94                                                                         /*}}}*/
  95 // ServerState::HeaderLine - Process a header line                      /*{{{*/
  96 // ---------------------------------------------------------------------
  97 /* */
  98 bool ServerState::HeaderLine(string Line)
  99 {
 100    if (Line.empty() == true)
 101       return true;
 102
 103    string::size_type Pos = Line.find(' ');
 104    if (Pos == string::npos || Pos+1 > Line.length())
 105    {
 106       // Blah, some servers use "connection:closes", evil.
 107       Pos = Line.find(':');
 108       if (Pos == string::npos || Pos + 2 > Line.length())
 109          return _error->Error(_("Bad header line"));
 110       Pos++;
 111    }
 112
 113    // Parse off any trailing spaces between the : and the next word.
 114    string::size_type Pos2 = Pos;
 115    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 116       Pos2++;
 117
 118    string Tag = string(Line,0,Pos);
 119    string Val = string(Line,Pos2);
 120
 121    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 122    {
 123       // Evil servers return no version
 124       if (Line[4] == '/')
 125       {
 126          int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
 127          if (elements == 3)
 128          {
 129             Code[0] = '\0';
 130             if (Owner->Debug == true)
 131                clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
 132          }
 133          else if (elements != 4)
 134             return _error->Error(_("The HTTP server sent an invalid reply header"));
 135       }
 136       else
 137       {
 138          Major = 0;
 139          Minor = 9;
 140          if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
 141             return _error->Error(_("The HTTP server sent an invalid reply header"));
 142       }
 143
 144       /* Check the HTTP response header to get the default persistence
 145          state. */
 146       if (Major < 1)
 147          Persistent = false;
 148       else
 149       {
 150          if (Major == 1 && Minor == 0)
 151             Persistent = false;
 152          else
 153             Persistent = true;
 154       }
 155
 156       return true;
 157    }
 158
 159    if (stringcasecmp(Tag,"Content-Length:") == 0)
 160    {
 161       if (Encoding == Closes)
 162          Encoding = Stream;
 163       HaveContent = true;
 164
 165       // The length is already set from the Content-Range header
 166       if (StartPos != 0)
 167          return true;
 168
 169       Size = strtoull(Val.c_str(), NULL, 10);
 170       if (Size >= std::numeric_limits<unsigned long long>::max())
 171          return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
 172       else if (Size == 0)
 173          HaveContent = false;
 174       return true;
 175    }
 176
 177    if (stringcasecmp(Tag,"Content-Type:") == 0)
 178    {
 179       HaveContent = true;
 180       return true;
 181    }
 182
 183    if (stringcasecmp(Tag,"Content-Range:") == 0)
 184    {
 185       HaveContent = true;
 186
 187       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
 188       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
 189       {
 190          StartPos = 1; // ignore Content-Length, it would override Size
 191          HaveContent = false;
 192       }
 193       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
 194          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 195       if ((unsigned long long)StartPos > Size)
 196          return _error->Error(_("This HTTP server has broken range support"));
 197       return true;
 198    }
 199
 200    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 201    {
 202       HaveContent = true;
 203       if (stringcasecmp(Val,"chunked") == 0)
 204          Encoding = Chunked;
 205       return true;
 206    }
 207
 208    if (stringcasecmp(Tag,"Connection:") == 0)
 209    {
 210       if (stringcasecmp(Val,"close") == 0)
 211          Persistent = false;
 212       if (stringcasecmp(Val,"keep-alive") == 0)
 213          Persistent = true;
 214       return true;
 215    }
 216
 217    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 218    {
 219       if (RFC1123StrToTime(Val.c_str(), Date) == false)
 220          return _error->Error(_("Unknown date format"));
 221       return true;
 222    }
 223
 224    if (stringcasecmp(Tag,"Location:") == 0)
 225    {
 226       Location = Val;
 227       return true;
 228    }
 229
 230    return true;
 231 }
 232                                                                         /*}}}*/
 233 // ServerState::ServerState - Constructor                               /*{{{*/
 234 ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOut(120), Owner(Owner)
 235 {
 236    Reset();
 237 }
 238                                                                         /*}}}*/
 239
 240 bool ServerMethod::Configuration(string Message)                        /*{{{*/
 241 {
 242    return pkgAcqMethod::Configuration(Message);
 243 }
 244                                                                         /*}}}*/
 245
 246 // ServerMethod::DealWithHeaders - Handle the retrieved header data     /*{{{*/
 247 // ---------------------------------------------------------------------
 248 /* We look at the header data we got back from the server and decide what
 249    to do. Returns DealWithHeadersResult (see http.h for details).
 250  */
 251 ServerMethod::DealWithHeadersResult
 252 ServerMethod::DealWithHeaders(FetchResult &Res)
 253 {
 254    // Not Modified
 255    if (Server->Result == 304)
 256    {
 257       unlink(Queue->DestFile.c_str());
 258       Res.IMSHit = true;
 259       Res.LastModified = Queue->LastModified;
 260       return IMS_HIT;
 261    }
 262
 263    /* Redirect
 264     *
 265     * Note that it is only OK for us to treat all redirection the same
 266     * because we *always* use GET, not other HTTP methods.  There are
 267     * three redirection codes for which it is not appropriate that we
 268     * redirect.  Pass on those codes so the error handling kicks in.
 269     */
 270    if (AllowRedirect
 271        && (Server->Result > 300 && Server->Result < 400)
 272        && (Server->Result != 300       // Multiple Choices
 273            && Server->Result != 304    // Not Modified
 274            && Server->Result != 306))  // (Not part of HTTP/1.1, reserved)
 275    {
 276       if (Server->Location.empty() == true);
 277       else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
 278       {
 279          URI Uri = Queue->Uri;
 280          if (Uri.Host.empty() == false)
 281             NextURI = URI::SiteOnly(Uri);
 282          else
 283             NextURI.clear();
 284          NextURI.append(DeQuoteString(Server->Location));
 285          return TRY_AGAIN_OR_REDIRECT;
 286       }
 287       else
 288       {
 289          NextURI = DeQuoteString(Server->Location);
 290          URI tmpURI = NextURI;
 291          URI Uri = Queue->Uri;
 292          // same protocol redirects are okay
 293          if (tmpURI.Access == Uri.Access)
 294             return TRY_AGAIN_OR_REDIRECT;
 295          // as well as http to https
 296          else if (Uri.Access == "http" && tmpURI.Access == "https")
 297             return TRY_AGAIN_OR_REDIRECT;
 298       }
 299       /* else pass through for error message */
 300    }
 301    // retry after an invalid range response without partial data
 302    else if (Server->Result == 416)
 303    {
 304       struct stat SBuf;
 305       if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 306       {
 307          if ((unsigned long long)SBuf.st_size == Server->Size)
 308          {
 309             // the file is completely downloaded, but was not moved
 310             Server->StartPos = Server->Size;
 311             Server->Result = 200;
 312             Server->HaveContent = false;
 313          }
 314          else if (unlink(Queue->DestFile.c_str()) == 0)
 315          {
 316             NextURI = Queue->Uri;
 317             return TRY_AGAIN_OR_REDIRECT;
 318          }
 319       }
 320    }
 321
 322    /* We have a reply we dont handle. This should indicate a perm server
 323       failure */
 324    if (Server->Result < 200 || Server->Result >= 300)
 325    {
 326       char err[255];
 327       snprintf(err,sizeof(err)-1,"HttpError%i",Server->Result);
 328       SetFailReason(err);
 329       _error->Error("%u %s",Server->Result,Server->Code);
 330       if (Server->HaveContent == true)
 331          return ERROR_WITH_CONTENT_PAGE;
 332       return ERROR_UNRECOVERABLE;
 333    }
 334
 335    // This is some sort of 2xx 'data follows' reply
 336    Res.LastModified = Server->Date;
 337    Res.Size = Server->Size;
 338
 339    // Open the file
 340    delete File;
 341    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 342    if (_error->PendingError() == true)
 343       return ERROR_NOT_FROM_SERVER;
 344
 345    FailFile = Queue->DestFile;
 346    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 347    FailFd = File->Fd();
 348    FailTime = Server->Date;
 349
 350    if (Server->InitHashes(*File) == false)
 351    {
 352       _error->Errno("read",_("Problem hashing file"));
 353       return ERROR_NOT_FROM_SERVER;
 354    }
 355    if (Server->StartPos > 0)
 356       Res.ResumePoint = Server->StartPos;
 357
 358    SetNonBlock(File->Fd(),true);
 359    return FILE_IS_OPEN;
 360 }
 361                                                                         /*}}}*/
 362 // ServerMethod::SigTerm - Handle a fatal signal                        /*{{{*/
 363 // ---------------------------------------------------------------------
 364 /* This closes and timestamps the open file. This is necessary to get
 365    resume behavoir on user abort */
 366 void ServerMethod::SigTerm(int)
 367 {
 368    if (FailFd == -1)
 369       _exit(100);
 370
 371    struct timeval times[2];
 372    times[0].tv_sec = FailTime;
 373    times[1].tv_sec = FailTime;
 374    times[0].tv_usec = times[1].tv_usec = 0;
 375    utimes(FailFile.c_str(), times);
 376    close(FailFd);
 377
 378    _exit(100);
 379 }
 380                                                                         /*}}}*/
 381 // ServerMethod::Fetch - Fetch an item                                  /*{{{*/
 382 // ---------------------------------------------------------------------
 383 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 384    depth. */
 385 bool ServerMethod::Fetch(FetchItem *)
 386 {
 387    if (Server == 0)
 388       return true;
 389
 390    // Queue the requests
 391    int Depth = -1;
 392    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 393         I = I->Next, Depth++)
 394    {
 395       if (Depth >= 0)
 396       {
 397          // If pipelining is disabled, we only queue 1 request
 398          if (Server->Pipeline == false)
 399             break;
 400          // if we have no hashes, do at most one such request
 401          // as we can't fixup pipeling misbehaviors otherwise
 402          else if (I->ExpectedHashes.usable() == false)
 403             break;
 404       }
 405
 406       // Make sure we stick with the same server
 407       if (Server->Comp(I->Uri) == false)
 408          break;
 409       if (QueueBack == I)
 410       {
 411          QueueBack = I->Next;
 412          SendReq(I);
 413          continue;
 414       }
 415    }
 416
 417    return true;
 418 }
 419                                                                         /*}}}*/
 420 // ServerMethod::Loop - Main loop                                       /*{{{*/
 421 int ServerMethod::Loop()
 422 {
 423    typedef vector<string> StringVector;
 424    typedef vector<string>::iterator StringVectorIterator;
 425    map<string, StringVector> Redirected;
 426
 427    signal(SIGTERM,SigTerm);
 428    signal(SIGINT,SigTerm);
 429
 430    Server = 0;
 431
 432    int FailCounter = 0;
 433    while (1)
 434    {
 435       // We have no commands, wait for some to arrive
 436       if (Queue == 0)
 437       {
 438          if (WaitFd(STDIN_FILENO) == false)
 439             return 0;
 440       }
 441
 442       /* Run messages, we can accept 0 (no message) if we didn't
 443          do a WaitFd above.. Otherwise the FD is closed. */
 444       int Result = Run(true);
 445       if (Result != -1 && (Result != 0 || Queue == 0))
 446       {
 447          if(FailReason.empty() == false ||
 448             _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
 449             return 100;
 450          else
 451             return 0;
 452       }
 453
 454       if (Queue == 0)
 455          continue;
 456
 457       // Connect to the server
 458       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 459       {
 460          delete Server;
 461          Server = CreateServerState(Queue->Uri);
 462       }
 463       /* If the server has explicitly said this is the last connection
 464          then we pre-emptively shut down the pipeline and tear down
 465          the connection. This will speed up HTTP/1.0 servers a tad
 466          since we don't have to wait for the close sequence to
 467          complete */
 468       if (Server->Persistent == false)
 469          Server->Close();
 470
 471       // Reset the pipeline
 472       if (Server->IsOpen() == false)
 473          QueueBack = Queue;
 474
 475       // Connnect to the host
 476       if (Server->Open() == false)
 477       {
 478          Fail(true);
 479          delete Server;
 480          Server = 0;
 481          continue;
 482       }
 483
 484       // Fill the pipeline.
 485       Fetch(0);
 486
 487       // Fetch the next URL header data from the server.
 488       switch (Server->RunHeaders(File))
 489       {
 490          case ServerState::RUN_HEADERS_OK:
 491          break;
 492
 493          // The header data is bad
 494          case ServerState::RUN_HEADERS_PARSE_ERROR:
 495          {
 496             _error->Error(_("Bad header data"));
 497             Fail(true);
 498             RotateDNS();
 499             continue;
 500          }
 501
 502          // The server closed a connection during the header get..
 503          default:
 504          case ServerState::RUN_HEADERS_IO_ERROR:
 505          {
 506             FailCounter++;
 507             _error->Discard();
 508             Server->Close();
 509             Server->Pipeline = false;
 510
 511             if (FailCounter >= 2)
 512             {
 513                Fail(_("Connection failed"),true);
 514                FailCounter = 0;
 515             }
 516
 517             RotateDNS();
 518             continue;
 519          }
 520       };
 521
 522       // Decide what to do.
 523       FetchResult Res;
 524       Res.Filename = Queue->DestFile;
 525       switch (DealWithHeaders(Res))
 526       {
 527          // Ok, the file is Open
 528          case FILE_IS_OPEN:
 529          {
 530             URIStart(Res);
 531
 532             // Run the data
 533             bool Result = true;
 534             if (Server->HaveContent)
 535                Result = Server->RunData(File);
 536
 537             /* If the server is sending back sizeless responses then fill in
 538                the size now */
 539             if (Res.Size == 0)
 540                Res.Size = File->Size();
 541
 542             // Close the file, destroy the FD object and timestamp it
 543             FailFd = -1;
 544             delete File;
 545             File = 0;
 546
 547             // Timestamp
 548             struct timeval times[2];
 549             times[0].tv_sec = times[1].tv_sec = Server->Date;
 550             times[0].tv_usec = times[1].tv_usec = 0;
 551             utimes(Queue->DestFile.c_str(), times);
 552
 553             // Send status to APT
 554             if (Result == true)
 555             {
 556                Hashes * const resultHashes = Server->GetHashes();
 557                HashStringList const hashList = resultHashes->GetHashStringList();
 558                if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
 559                {
 560                   // we did not get the expected hash… mhhh:
 561                   // could it be that server/proxy messed up pipelining?
 562                   FetchItem * BeforeI = Queue;
 563                   for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
 564                   {
 565                      if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
 566                      {
 567                         // yes, he did! Disable pipelining and rewrite queue
 568                         if (Server->Pipeline == true)
 569                         {
 570                            // FIXME: fake a warning message as we have no proper way of communicating here
 571                            std::string out;
 572                            strprintf(out, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::PipelineDepth");
 573                            std::cerr << "W: " << out << std::endl;
 574                            Server->Pipeline = false;
 575                            // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
 576                         }
 577                         Rename(Res.Filename, I->DestFile);
 578                         Res.Filename = I->DestFile;
 579                         BeforeI->Next = I->Next;
 580                         I->Next = Queue;
 581                         Queue = I;
 582                         break;
 583                      }
 584                      BeforeI = I;
 585                   }
 586                }
 587                Res.TakeHashes(*resultHashes);
 588                URIDone(Res);
 589             }
 590             else
 591             {
 592                if (Server->IsOpen() == false)
 593                {
 594                   FailCounter++;
 595                   _error->Discard();
 596                   Server->Close();
 597
 598                   if (FailCounter >= 2)
 599                   {
 600                      Fail(_("Connection failed"),true);
 601                      FailCounter = 0;
 602                   }
 603
 604                   QueueBack = Queue;
 605                }
 606                else
 607                   Fail(true);
 608             }
 609             break;
 610          }
 611
 612          // IMS hit
 613          case IMS_HIT:
 614          {
 615             URIDone(Res);
 616             break;
 617          }
 618
 619          // Hard server error, not found or something
 620          case ERROR_UNRECOVERABLE:
 621          {
 622             Fail();
 623             break;
 624          }
 625
 626          // Hard internal error, kill the connection and fail
 627          case ERROR_NOT_FROM_SERVER:
 628          {
 629             delete File;
 630             File = 0;
 631
 632             Fail();
 633             RotateDNS();
 634             Server->Close();
 635             break;
 636          }
 637
 638          // We need to flush the data, the header is like a 404 w/ error text
 639          case ERROR_WITH_CONTENT_PAGE:
 640          {
 641             Fail();
 642
 643             // Send to content to dev/null
 644             File = new FileFd("/dev/null",FileFd::WriteExists);
 645             Server->RunData(File);
 646             delete File;
 647             File = 0;
 648             break;
 649          }
 650
 651          // Try again with a new URL
 652          case TRY_AGAIN_OR_REDIRECT:
 653          {
 654             // Clear rest of response if there is content
 655             if (Server->HaveContent)
 656             {
 657                File = new FileFd("/dev/null",FileFd::WriteExists);
 658                Server->RunData(File);
 659                delete File;
 660                File = 0;
 661             }
 662
 663             /* Detect redirect loops.  No more redirects are allowed
 664                after the same URI is seen twice in a queue item. */
 665             StringVector &R = Redirected[Queue->DestFile];
 666             bool StopRedirects = false;
 667             if (R.empty() == true)
 668                R.push_back(Queue->Uri);
 669             else if (R[0] == "STOP" || R.size() > 10)
 670                StopRedirects = true;
 671             else
 672             {
 673                for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
 674                   if (Queue->Uri == *I)
 675                   {
 676                      R[0] = "STOP";
 677                      break;
 678                   }
 679
 680                R.push_back(Queue->Uri);
 681             }
 682
 683             if (StopRedirects == false)
 684                Redirect(NextURI);
 685             else
 686                Fail();
 687
 688             break;
 689          }
 690
 691          default:
 692          Fail(_("Internal error"));
 693          break;
 694       }
 695
 696       FailCounter = 0;
 697    }
 698
 699    return 0;
 700 }
 701                                                                         /*}}}*/