methods/server.cc

   1 // -*- mode: cpp; mode: fold -*-
   2 // Description                                                          /*{{{*/
   3 /* ######################################################################
   4
   5    HTTP and HTTPS share a lot of common code and these classes are
   6    exactly the dumping ground for this common code
   7
   8    ##################################################################### */
   9                                                                         /*}}}*/
  10 // Include Files                                                        /*{{{*/
  11 #include <config.h>
  12
  13 #include <apt-pkg/fileutl.h>
  14 #include <apt-pkg/acquire-method.h>
  15 #include <apt-pkg/configuration.h>
  16 #include <apt-pkg/error.h>
  17 #include <apt-pkg/hashes.h>
  18 #include <apt-pkg/netrc.h>
  19
  20 #include <fcntl.h>
  21 #include <sys/stat.h>
  22 #include <sys/time.h>
  23 #include <unistd.h>
  24 #include <signal.h>
  25 #include <stdio.h>
  26 #include <errno.h>
  27 #include <string.h>
  28 #include <climits>
  29 #include <iostream>
  30 #include <map>
  31
  32 // Internet stuff
  33 #include <netdb.h>
  34
  35 #include "config.h"
  36 #include "connect.h"
  37 #include "rfc2553emu.h"
  38 #include "http.h"
  39
  40 #include <apti18n.h>
  41                                                                         /*}}}*/
  42 using namespace std;
  43
  44 string ServerMethod::FailFile;
  45 int ServerMethod::FailFd = -1;
  46 time_t ServerMethod::FailTime = 0;
  47
  48 // ServerState::RunHeaders - Get the headers before the data            /*{{{*/
  49 // ---------------------------------------------------------------------
  50 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
  51    parse error occurred */
  52 ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File)
  53 {
  54    State = Header;
  55
  56    Owner->Status(_("Waiting for headers"));
  57
  58    Major = 0;
  59    Minor = 0;
  60    Result = 0;
  61    Size = 0;
  62    StartPos = 0;
  63    Encoding = Closes;
  64    HaveContent = false;
  65    time(&Date);
  66
  67    do
  68    {
  69       string Data;
  70       if (ReadHeaderLines(Data) == false)
  71          continue;
  72
  73       if (Owner->Debug == true)
  74          clog << Data;
  75
  76       for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
  77       {
  78          string::const_iterator J = I;
  79          for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
  80          if (HeaderLine(string(I,J)) == false)
  81             return RUN_HEADERS_PARSE_ERROR;
  82          I = J;
  83       }
  84
  85       // 100 Continue is a Nop...
  86       if (Result == 100)
  87          continue;
  88
  89       // Tidy up the connection persistence state.
  90       if (Encoding == Closes && HaveContent == true)
  91          Persistent = false;
  92
  93       return RUN_HEADERS_OK;
  94    }
  95    while (LoadNextResponse(false, File) == true);
  96
  97    return RUN_HEADERS_IO_ERROR;
  98 }
  99                                                                         /*}}}*/
 100 // ServerState::HeaderLine - Process a header line                      /*{{{*/
 101 // ---------------------------------------------------------------------
 102 /* */
 103 bool ServerState::HeaderLine(string Line)
 104 {
 105    if (Line.empty() == true)
 106       return true;
 107
 108    string::size_type Pos = Line.find(' ');
 109    if (Pos == string::npos || Pos+1 > Line.length())
 110    {
 111       // Blah, some servers use "connection:closes", evil.
 112       Pos = Line.find(':');
 113       if (Pos == string::npos || Pos + 2 > Line.length())
 114          return _error->Error(_("Bad header line"));
 115       Pos++;
 116    }
 117
 118    // Parse off any trailing spaces between the : and the next word.
 119    string::size_type Pos2 = Pos;
 120    while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
 121       Pos2++;
 122
 123    string Tag = string(Line,0,Pos);
 124    string Val = string(Line,Pos2);
 125
 126    if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
 127    {
 128       // Evil servers return no version
 129       if (Line[4] == '/')
 130       {
 131          int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
 132          if (elements == 3)
 133          {
 134             Code[0] = '\0';
 135             if (Owner->Debug == true)
 136                clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
 137          }
 138          else if (elements != 4)
 139             return _error->Error(_("The HTTP server sent an invalid reply header"));
 140       }
 141       else
 142       {
 143          Major = 0;
 144          Minor = 9;
 145          if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
 146             return _error->Error(_("The HTTP server sent an invalid reply header"));
 147       }
 148
 149       /* Check the HTTP response header to get the default persistence
 150          state. */
 151       if (Major < 1)
 152          Persistent = false;
 153       else
 154       {
 155          if (Major == 1 && Minor == 0)
 156             Persistent = false;
 157          else
 158             Persistent = true;
 159       }
 160
 161       return true;
 162    }
 163
 164    if (stringcasecmp(Tag,"Content-Length:") == 0)
 165    {
 166       if (Encoding == Closes)
 167          Encoding = Stream;
 168       HaveContent = true;
 169
 170       // The length is already set from the Content-Range header
 171       if (StartPos != 0)
 172          return true;
 173
 174       Size = strtoull(Val.c_str(), NULL, 10);
 175       if (Size >= std::numeric_limits<unsigned long long>::max())
 176          return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
 177       else if (Size == 0)
 178          HaveContent = false;
 179       return true;
 180    }
 181
 182    if (stringcasecmp(Tag,"Content-Type:") == 0)
 183    {
 184       HaveContent = true;
 185       return true;
 186    }
 187
 188    if (stringcasecmp(Tag,"Content-Range:") == 0)
 189    {
 190       HaveContent = true;
 191
 192       // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
 193       if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
 194       {
 195          StartPos = 1; // ignore Content-Length, it would override Size
 196          HaveContent = false;
 197       }
 198       else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
 199          return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
 200       if ((unsigned long long)StartPos > Size)
 201          return _error->Error(_("This HTTP server has broken range support"));
 202       return true;
 203    }
 204
 205    if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
 206    {
 207       HaveContent = true;
 208       if (stringcasecmp(Val,"chunked") == 0)
 209          Encoding = Chunked;
 210       return true;
 211    }
 212
 213    if (stringcasecmp(Tag,"Connection:") == 0)
 214    {
 215       if (stringcasecmp(Val,"close") == 0)
 216          Persistent = false;
 217       if (stringcasecmp(Val,"keep-alive") == 0)
 218          Persistent = true;
 219       return true;
 220    }
 221
 222    if (stringcasecmp(Tag,"Last-Modified:") == 0)
 223    {
 224       if (RFC1123StrToTime(Val.c_str(), Date) == false)
 225          return _error->Error(_("Unknown date format"));
 226       return true;
 227    }
 228
 229    if (stringcasecmp(Tag,"Location:") == 0)
 230    {
 231       Location = Val;
 232       return true;
 233    }
 234
 235    return true;
 236 }
 237                                                                         /*}}}*/
 238 // ServerState::ServerState - Constructor                               /*{{{*/
 239 ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOut(120), Owner(Owner)
 240 {
 241    Reset();
 242 }
 243                                                                         /*}}}*/
 244
 245 bool ServerMethod::Configuration(string Message)                        /*{{{*/
 246 {
 247    return pkgAcqMethod::Configuration(Message);
 248 }
 249                                                                         /*}}}*/
 250
 251 // ServerMethod::DealWithHeaders - Handle the retrieved header data     /*{{{*/
 252 // ---------------------------------------------------------------------
 253 /* We look at the header data we got back from the server and decide what
 254    to do. Returns DealWithHeadersResult (see http.h for details).
 255  */
 256 ServerMethod::DealWithHeadersResult
 257 ServerMethod::DealWithHeaders(FetchResult &Res)
 258 {
 259    // Not Modified
 260    if (Server->Result == 304)
 261    {
 262       unlink(Queue->DestFile.c_str());
 263       Res.IMSHit = true;
 264       Res.LastModified = Queue->LastModified;
 265       return IMS_HIT;
 266    }
 267
 268    /* Redirect
 269     *
 270     * Note that it is only OK for us to treat all redirection the same
 271     * because we *always* use GET, not other HTTP methods.  There are
 272     * three redirection codes for which it is not appropriate that we
 273     * redirect.  Pass on those codes so the error handling kicks in.
 274     */
 275    if (AllowRedirect
 276        && (Server->Result > 300 && Server->Result < 400)
 277        && (Server->Result != 300       // Multiple Choices
 278            && Server->Result != 304    // Not Modified
 279            && Server->Result != 306))  // (Not part of HTTP/1.1, reserved)
 280    {
 281       if (Server->Location.empty() == true);
 282       else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
 283       {
 284          URI Uri = Queue->Uri;
 285          if (Uri.Host.empty() == false)
 286             NextURI = URI::SiteOnly(Uri);
 287          else
 288             NextURI.clear();
 289          NextURI.append(DeQuoteString(Server->Location));
 290          return TRY_AGAIN_OR_REDIRECT;
 291       }
 292       else
 293       {
 294          NextURI = DeQuoteString(Server->Location);
 295          URI tmpURI = NextURI;
 296          URI Uri = Queue->Uri;
 297          // same protocol redirects are okay
 298          if (tmpURI.Access == Uri.Access)
 299             return TRY_AGAIN_OR_REDIRECT;
 300          // as well as http to https
 301          else if (Uri.Access == "http" && tmpURI.Access == "https")
 302             return TRY_AGAIN_OR_REDIRECT;
 303       }
 304       /* else pass through for error message */
 305    }
 306    // retry after an invalid range response without partial data
 307    else if (Server->Result == 416)
 308    {
 309       struct stat SBuf;
 310       if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
 311       {
 312          if ((unsigned long long)SBuf.st_size == Server->Size)
 313          {
 314             // the file is completely downloaded, but was not moved
 315             Server->StartPos = Server->Size;
 316             Server->Result = 200;
 317             Server->HaveContent = false;
 318          }
 319          else if (unlink(Queue->DestFile.c_str()) == 0)
 320          {
 321             NextURI = Queue->Uri;
 322             return TRY_AGAIN_OR_REDIRECT;
 323          }
 324       }
 325    }
 326
 327    /* We have a reply we dont handle. This should indicate a perm server
 328       failure */
 329    if (Server->Result < 200 || Server->Result >= 300)
 330    {
 331       char err[255];
 332       snprintf(err,sizeof(err)-1,"HttpError%i",Server->Result);
 333       SetFailReason(err);
 334       _error->Error("%u %s",Server->Result,Server->Code);
 335       if (Server->HaveContent == true)
 336          return ERROR_WITH_CONTENT_PAGE;
 337       return ERROR_UNRECOVERABLE;
 338    }
 339
 340    // This is some sort of 2xx 'data follows' reply
 341    Res.LastModified = Server->Date;
 342    Res.Size = Server->Size;
 343
 344    // Open the file
 345    delete File;
 346    File = new FileFd(Queue->DestFile,FileFd::WriteAny);
 347    if (_error->PendingError() == true)
 348       return ERROR_NOT_FROM_SERVER;
 349
 350    FailFile = Queue->DestFile;
 351    FailFile.c_str();   // Make sure we dont do a malloc in the signal handler
 352    FailFd = File->Fd();
 353    FailTime = Server->Date;
 354
 355    if (Server->InitHashes(*File) == false)
 356    {
 357       _error->Errno("read",_("Problem hashing file"));
 358       return ERROR_NOT_FROM_SERVER;
 359    }
 360    if (Server->StartPos > 0)
 361       Res.ResumePoint = Server->StartPos;
 362
 363    SetNonBlock(File->Fd(),true);
 364    return FILE_IS_OPEN;
 365 }
 366                                                                         /*}}}*/
 367 // ServerMethod::SigTerm - Handle a fatal signal                        /*{{{*/
 368 // ---------------------------------------------------------------------
 369 /* This closes and timestamps the open file. This is necessary to get
 370    resume behavoir on user abort */
 371 void ServerMethod::SigTerm(int)
 372 {
 373    if (FailFd == -1)
 374       _exit(100);
 375
 376    struct timeval times[2];
 377    times[0].tv_sec = FailTime;
 378    times[1].tv_sec = FailTime;
 379    times[0].tv_usec = times[1].tv_usec = 0;
 380    utimes(FailFile.c_str(), times);
 381    close(FailFd);
 382
 383    _exit(100);
 384 }
 385                                                                         /*}}}*/
 386 // ServerMethod::Fetch - Fetch an item                                  /*{{{*/
 387 // ---------------------------------------------------------------------
 388 /* This adds an item to the pipeline. We keep the pipeline at a fixed
 389    depth. */
 390 bool ServerMethod::Fetch(FetchItem *)
 391 {
 392    if (Server == 0)
 393       return true;
 394
 395    // Queue the requests
 396    int Depth = -1;
 397    for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
 398         I = I->Next, Depth++)
 399    {
 400       // If pipelining is disabled, we only queue 1 request
 401       if (Server->Pipeline == false && Depth >= 0)
 402          break;
 403
 404       // Make sure we stick with the same server
 405       if (Server->Comp(I->Uri) == false)
 406          break;
 407       if (QueueBack == I)
 408       {
 409          QueueBack = I->Next;
 410          SendReq(I);
 411          continue;
 412       }
 413    }
 414
 415    return true;
 416 };
 417                                                                         /*}}}*/
 418 // ServerMethod::Loop - Main loop                                       /*{{{*/
 419 int ServerMethod::Loop()
 420 {
 421    typedef vector<string> StringVector;
 422    typedef vector<string>::iterator StringVectorIterator;
 423    map<string, StringVector> Redirected;
 424
 425    signal(SIGTERM,SigTerm);
 426    signal(SIGINT,SigTerm);
 427
 428    Server = 0;
 429
 430    int FailCounter = 0;
 431    while (1)
 432    {
 433       // We have no commands, wait for some to arrive
 434       if (Queue == 0)
 435       {
 436          if (WaitFd(STDIN_FILENO) == false)
 437             return 0;
 438       }
 439
 440       /* Run messages, we can accept 0 (no message) if we didn't
 441          do a WaitFd above.. Otherwise the FD is closed. */
 442       int Result = Run(true);
 443       if (Result != -1 && (Result != 0 || Queue == 0))
 444       {
 445          if(FailReason.empty() == false ||
 446             _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
 447             return 100;
 448          else
 449             return 0;
 450       }
 451
 452       if (Queue == 0)
 453          continue;
 454
 455       // Connect to the server
 456       if (Server == 0 || Server->Comp(Queue->Uri) == false)
 457       {
 458          delete Server;
 459          Server = CreateServerState(Queue->Uri);
 460       }
 461       /* If the server has explicitly said this is the last connection
 462          then we pre-emptively shut down the pipeline and tear down
 463          the connection. This will speed up HTTP/1.0 servers a tad
 464          since we don't have to wait for the close sequence to
 465          complete */
 466       if (Server->Persistent == false)
 467          Server->Close();
 468
 469       // Reset the pipeline
 470       if (Server->IsOpen() == false)
 471          QueueBack = Queue;
 472
 473       // Connnect to the host
 474       if (Server->Open() == false)
 475       {
 476          Fail(true);
 477          delete Server;
 478          Server = 0;
 479          continue;
 480       }
 481
 482       // Fill the pipeline.
 483       Fetch(0);
 484
 485       // Fetch the next URL header data from the server.
 486       switch (Server->RunHeaders(File))
 487       {
 488          case ServerState::RUN_HEADERS_OK:
 489          break;
 490
 491          // The header data is bad
 492          case ServerState::RUN_HEADERS_PARSE_ERROR:
 493          {
 494             _error->Error(_("Bad header data"));
 495             Fail(true);
 496             RotateDNS();
 497             continue;
 498          }
 499
 500          // The server closed a connection during the header get..
 501          default:
 502          case ServerState::RUN_HEADERS_IO_ERROR:
 503          {
 504             FailCounter++;
 505             _error->Discard();
 506             Server->Close();
 507             Server->Pipeline = false;
 508
 509             if (FailCounter >= 2)
 510             {
 511                Fail(_("Connection failed"),true);
 512                FailCounter = 0;
 513             }
 514
 515             RotateDNS();
 516             continue;
 517          }
 518       };
 519
 520       // Decide what to do.
 521       FetchResult Res;
 522       Res.Filename = Queue->DestFile;
 523       switch (DealWithHeaders(Res))
 524       {
 525          // Ok, the file is Open
 526          case FILE_IS_OPEN:
 527          {
 528             URIStart(Res);
 529
 530             // Run the data
 531             bool Result = true;
 532             if (Server->HaveContent)
 533                Result = Server->RunData(File);
 534
 535             /* If the server is sending back sizeless responses then fill in
 536                the size now */
 537             if (Res.Size == 0)
 538                Res.Size = File->Size();
 539
 540             // Close the file, destroy the FD object and timestamp it
 541             FailFd = -1;
 542             delete File;
 543             File = 0;
 544
 545             // Timestamp
 546             struct timeval times[2];
 547             times[0].tv_sec = times[1].tv_sec = Server->Date;
 548             times[0].tv_usec = times[1].tv_usec = 0;
 549             utimes(Queue->DestFile.c_str(), times);
 550
 551             // Send status to APT
 552             if (Result == true)
 553             {
 554                Res.TakeHashes(*Server->GetHashes());
 555                URIDone(Res);
 556             }
 557             else
 558             {
 559                if (Server->IsOpen() == false)
 560                {
 561                   FailCounter++;
 562                   _error->Discard();
 563                   Server->Close();
 564
 565                   if (FailCounter >= 2)
 566                   {
 567                      Fail(_("Connection failed"),true);
 568                      FailCounter = 0;
 569                   }
 570
 571                   QueueBack = Queue;
 572                }
 573                else
 574                   Fail(true);
 575             }
 576             break;
 577          }
 578
 579          // IMS hit
 580          case IMS_HIT:
 581          {
 582             URIDone(Res);
 583             break;
 584          }
 585
 586          // Hard server error, not found or something
 587          case ERROR_UNRECOVERABLE:
 588          {
 589             Fail();
 590             break;
 591          }
 592
 593          // Hard internal error, kill the connection and fail
 594          case ERROR_NOT_FROM_SERVER:
 595          {
 596             delete File;
 597             File = 0;
 598
 599             Fail();
 600             RotateDNS();
 601             Server->Close();
 602             break;
 603          }
 604
 605          // We need to flush the data, the header is like a 404 w/ error text
 606          case ERROR_WITH_CONTENT_PAGE:
 607          {
 608             Fail();
 609
 610             // Send to content to dev/null
 611             File = new FileFd("/dev/null",FileFd::WriteExists);
 612             Server->RunData(File);
 613             delete File;
 614             File = 0;
 615             break;
 616          }
 617
 618          // Try again with a new URL
 619          case TRY_AGAIN_OR_REDIRECT:
 620          {
 621             // Clear rest of response if there is content
 622             if (Server->HaveContent)
 623             {
 624                File = new FileFd("/dev/null",FileFd::WriteExists);
 625                Server->RunData(File);
 626                delete File;
 627                File = 0;
 628             }
 629
 630             /* Detect redirect loops.  No more redirects are allowed
 631                after the same URI is seen twice in a queue item. */
 632             StringVector &R = Redirected[Queue->DestFile];
 633             bool StopRedirects = false;
 634             if (R.empty() == true)
 635                R.push_back(Queue->Uri);
 636             else if (R[0] == "STOP" || R.size() > 10)
 637                StopRedirects = true;
 638             else
 639             {
 640                for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
 641                   if (Queue->Uri == *I)
 642                   {
 643                      R[0] = "STOP";
 644                      break;
 645                   }
 646
 647                R.push_back(Queue->Uri);
 648             }
 649
 650             if (StopRedirects == false)
 651                Redirect(NextURI);
 652             else
 653                Fail();
 654
 655             break;
 656          }
 657
 658          default:
 659          Fail(_("Internal error"));
 660          break;
 661       }
 662
 663       FailCounter = 0;
 664    }
 665
 666    return 0;
 667 }
 668                                                                         /*}}}*/