X-Git-Url: https://git.saurik.com/apt.git/blobdiff_plain/15847d263c371cadcb8664021b8eb2444dd70998..a9e4fd6534a2ae6a5a983df77e5b1efa32f3a756:/methods/server.cc diff --git a/methods/server.cc b/methods/server.cc index a2128441c..9db45eb8c 100644 --- a/methods/server.cc +++ b/methods/server.cc @@ -10,32 +10,26 @@ // Include Files /*{{{*/ #include -#include -#include #include #include -#include -#include +#include +#include +#include +#include +#include +#include #include #include -#include +#include #include -#include -#include -#include -#include -#include #include +#include #include +#include +#include -// Internet stuff -#include - -#include "config.h" -#include "connect.h" -#include "rfc2553emu.h" -#include "http.h" +#include "server.h" #include /*}}}*/ @@ -49,7 +43,8 @@ time_t ServerMethod::FailTime = 0; // --------------------------------------------------------------------- /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header parse error occurred */ -ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File) +ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File, + const std::string &Uri) { State = Header; @@ -58,7 +53,8 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File) Major = 0; Minor = 0; Result = 0; - Size = 0; + TotalFileSize = 0; + JunkSize = 0; StartPos = 0; Encoding = Closes; HaveContent = false; @@ -71,7 +67,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File) continue; if (Owner->Debug == true) - clog << Data; + clog << "Answer for: " << Uri << endl << Data; for (string::const_iterator I = Data.begin(); I < Data.end(); ++I) { @@ -86,7 +82,7 @@ ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File) if (Result == 100) continue; - // Tidy up the connection persistance state. + // Tidy up the connection persistence state. if (Encoding == Closes && HaveContent == true) Persistent = false; @@ -117,12 +113,12 @@ bool ServerState::HeaderLine(string Line) // Parse off any trailing spaces between the : and the next word. string::size_type Pos2 = Pos; - while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0) + while (Pos2 < Line.length() && isspace_ascii(Line[Pos2]) != 0) Pos2++; - + string Tag = string(Line,0,Pos); string Val = string(Line,Pos2); - + if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0) { // Evil servers return no version @@ -132,8 +128,8 @@ bool ServerState::HeaderLine(string Line) if (elements == 3) { Code[0] = '\0'; - if (Owner->Debug == true) - clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl; + if (Owner != NULL && Owner->Debug == true) + clog << "HTTP server doesn't give Reason-Phrase for " << std::to_string(Result) << std::endl; } else if (elements != 4) return _error->Error(_("The HTTP server sent an invalid reply header")); @@ -146,36 +142,49 @@ bool ServerState::HeaderLine(string Line) return _error->Error(_("The HTTP server sent an invalid reply header")); } - /* Check the HTTP response header to get the default persistance + /* Check the HTTP response header to get the default persistence state. */ if (Major < 1) Persistent = false; else { if (Major == 1 && Minor == 0) + { Persistent = false; + } else + { Persistent = true; + if (PipelineAllowed) + Pipeline = true; + } } return true; - } - + } + if (stringcasecmp(Tag,"Content-Length:") == 0) { if (Encoding == Closes) Encoding = Stream; HaveContent = true; - - // The length is already set from the Content-Range header - if (StartPos != 0) - return true; - Size = strtoull(Val.c_str(), NULL, 10); - if (Size >= std::numeric_limits::max()) + unsigned long long * DownloadSizePtr = &DownloadSize; + if (Result == 416) + DownloadSizePtr = &JunkSize; + + *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10); + if (*DownloadSizePtr >= std::numeric_limits::max()) return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header")); - else if (Size == 0) + else if (*DownloadSizePtr == 0) HaveContent = false; + + // On partial content (206) the Content-Length less than the real + // size, so do not set it here but leave that to the Content-Range + // header instead + if(Result != 206 && TotalFileSize == 0) + TotalFileSize = DownloadSize; + return true; } @@ -184,29 +193,29 @@ bool ServerState::HeaderLine(string Line) HaveContent = true; return true; } - + if (stringcasecmp(Tag,"Content-Range:") == 0) { HaveContent = true; // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416 - if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1) - { - StartPos = 1; // ignore Content-Length, it would override Size - HaveContent = false; - } - else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2) + if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1) + ; // we got the expected filesize which is all we wanted + else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2) return _error->Error(_("The HTTP server sent an invalid Content-Range header")); - if ((unsigned long long)StartPos > Size) + if ((unsigned long long)StartPos > TotalFileSize) return _error->Error(_("This HTTP server has broken range support")); + + // figure out what we will download + DownloadSize = TotalFileSize - StartPos; return true; } - + if (stringcasecmp(Tag,"Transfer-Encoding:") == 0) { HaveContent = true; if (stringcasecmp(Val,"chunked") == 0) - Encoding = Chunked; + Encoding = Chunked; return true; } @@ -218,7 +227,7 @@ bool ServerState::HeaderLine(string Line) Persistent = true; return true; } - + if (stringcasecmp(Tag,"Last-Modified:") == 0) { if (RFC1123StrToTime(Val.c_str(), Date) == false) @@ -236,15 +245,16 @@ bool ServerState::HeaderLine(string Line) } /*}}}*/ // ServerState::ServerState - Constructor /*{{{*/ -ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOut(120), Owner(Owner) +ServerState::ServerState(URI Srv, ServerMethod *Owner) : + DownloadSize(0), ServerName(Srv), TimeOut(120), Owner(Owner) { Reset(); } /*}}}*/ - -bool ServerMethod::Configuration(string Message) /*{{{*/ +bool ServerState::AddPartialFileToHashes(FileFd &File) /*{{{*/ { - return pkgAcqMethod::Configuration(Message); + File.Truncate(StartPos); + return GetHashes()->AddFD(File, StartPos); } /*}}}*/ @@ -259,12 +269,12 @@ ServerMethod::DealWithHeaders(FetchResult &Res) // Not Modified if (Server->Result == 304) { - unlink(Queue->DestFile.c_str()); + RemoveFile("server", Queue->DestFile); Res.IMSHit = true; Res.LastModified = Queue->LastModified; return IMS_HIT; } - + /* Redirect * * Note that it is only OK for us to treat all redirection the same @@ -291,11 +301,15 @@ ServerMethod::DealWithHeaders(FetchResult &Res) } else { - NextURI = DeQuoteString(Server->Location); - URI tmpURI = NextURI; - // Do not allow a redirection to switch protocol - if (tmpURI.Access == "http") - return TRY_AGAIN_OR_REDIRECT; + NextURI = DeQuoteString(Server->Location); + URI tmpURI = NextURI; + URI Uri = Queue->Uri; + // same protocol redirects are okay + if (tmpURI.Access == Uri.Access) + return TRY_AGAIN_OR_REDIRECT; + // as well as http to https + else if (Uri.Access == "http" && tmpURI.Access == "https") + return TRY_AGAIN_OR_REDIRECT; } /* else pass through for error message */ } @@ -305,14 +319,33 @@ ServerMethod::DealWithHeaders(FetchResult &Res) struct stat SBuf; if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0) { - if ((unsigned long long)SBuf.st_size == Server->Size) + bool partialHit = false; + if (Queue->ExpectedHashes.usable() == true) + { + Hashes resultHashes(Queue->ExpectedHashes); + FileFd file(Queue->DestFile, FileFd::ReadOnly); + Server->TotalFileSize = file.FileSize(); + Server->Date = file.ModificationTime(); + resultHashes.AddFD(file); + HashStringList const hashList = resultHashes.GetHashStringList(); + partialHit = (Queue->ExpectedHashes == hashList); + } + else if ((unsigned long long)SBuf.st_size == Server->TotalFileSize) + partialHit = true; + if (partialHit == true) { // the file is completely downloaded, but was not moved - Server->StartPos = Server->Size; - Server->Result = 200; + if (Server->HaveContent == true) + { + // Send to error page to dev/null + FileFd DevNull("/dev/null",FileFd::WriteExists); + Server->RunData(&DevNull); + } Server->HaveContent = false; + Server->StartPos = Server->TotalFileSize; + Server->Result = 200; } - else if (unlink(Queue->DestFile.c_str()) == 0) + else if (RemoveFile("server", Queue->DestFile)) { NextURI = Queue->Uri; return TRY_AGAIN_OR_REDIRECT; @@ -320,14 +353,14 @@ ServerMethod::DealWithHeaders(FetchResult &Res) } } - /* We have a reply we dont handle. This should indicate a perm server + /* We have a reply we don't handle. This should indicate a perm server failure */ if (Server->Result < 200 || Server->Result >= 300) { - char err[255]; - snprintf(err,sizeof(err)-1,"HttpError%i",Server->Result); + std::string err; + strprintf(err, "HttpError%u", Server->Result); SetFailReason(err); - _error->Error("%u %s",Server->Result,Server->Code); + _error->Error("%u %s", Server->Result, Server->Code); if (Server->HaveContent == true) return ERROR_WITH_CONTENT_PAGE; return ERROR_UNRECOVERABLE; @@ -335,7 +368,7 @@ ServerMethod::DealWithHeaders(FetchResult &Res) // This is some sort of 2xx 'data follows' reply Res.LastModified = Server->Date; - Res.Size = Server->Size; + Res.Size = Server->TotalFileSize; // Open the file delete File; @@ -344,11 +377,11 @@ ServerMethod::DealWithHeaders(FetchResult &Res) return ERROR_NOT_FROM_SERVER; FailFile = Queue->DestFile; - FailFile.c_str(); // Make sure we dont do a malloc in the signal handler + FailFile.c_str(); // Make sure we don't do a malloc in the signal handler FailFd = File->Fd(); FailTime = Server->Date; - if (Server->InitHashes(*File) == false) + if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false) { _error->Errno("read",_("Problem hashing file")); return ERROR_NOT_FROM_SERVER; @@ -362,20 +395,20 @@ ServerMethod::DealWithHeaders(FetchResult &Res) /*}}}*/ // ServerMethod::SigTerm - Handle a fatal signal /*{{{*/ // --------------------------------------------------------------------- -/* This closes and timestamps the open file. This is neccessary to get +/* This closes and timestamps the open file. This is necessary to get resume behavoir on user abort */ void ServerMethod::SigTerm(int) { if (FailFd == -1) _exit(100); + + struct timeval times[2]; + times[0].tv_sec = FailTime; + times[1].tv_sec = FailTime; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(FailFile.c_str(), times); close(FailFd); - - // Timestamp - struct utimbuf UBuf; - UBuf.actime = FailTime; - UBuf.modtime = FailTime; - utime(FailFile.c_str(),&UBuf); - + _exit(100); } /*}}}*/ @@ -385,31 +418,68 @@ void ServerMethod::SigTerm(int) depth. */ bool ServerMethod::Fetch(FetchItem *) { - if (Server == 0) + if (Server == nullptr || QueueBack == nullptr) return true; - // Queue the requests - int Depth = -1; - for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth; - I = I->Next, Depth++) - { - // If pipelining is disabled, we only queue 1 request - if (Server->Pipeline == false && Depth >= 0) - break; - + // If pipelining is disabled, we only queue 1 request + auto const AllowedDepth = Server->Pipeline ? PipelineDepth : 0; + // how deep is our pipeline currently? + decltype(PipelineDepth) CurrentDepth = 0; + for (FetchItem const *I = Queue; I != QueueBack; I = I->Next) + ++CurrentDepth; + + do { // Make sure we stick with the same server - if (Server->Comp(I->Uri) == false) + if (Server->Comp(QueueBack->Uri) == false) break; - if (QueueBack == I) + + bool const UsableHashes = QueueBack->ExpectedHashes.usable(); + // if we have no hashes, do at most one such request + // as we can't fixup pipeling misbehaviors otherwise + if (CurrentDepth != 0 && UsableHashes == false) + break; + + if (UsableHashes && FileExists(QueueBack->DestFile)) { - QueueBack = I->Next; - SendReq(I); - continue; + FileFd partial(QueueBack->DestFile, FileFd::ReadOnly); + Hashes wehave(QueueBack->ExpectedHashes); + if (QueueBack->ExpectedHashes.FileSize() == partial.FileSize()) + { + if (wehave.AddFD(partial) && + wehave.GetHashStringList() == QueueBack->ExpectedHashes) + { + FetchResult Res; + Res.Filename = QueueBack->DestFile; + Res.ResumePoint = QueueBack->ExpectedHashes.FileSize(); + URIStart(Res); + // move item to the start of the queue as URIDone will + // always dequeued the first item in the queue + if (Queue != QueueBack) + { + FetchItem *Prev = Queue; + for (; Prev->Next != QueueBack; Prev = Prev->Next) + /* look for the previous queue item */; + Prev->Next = QueueBack->Next; + QueueBack->Next = Queue; + Queue = QueueBack; + QueueBack = Prev->Next; + } + Res.TakeHashes(wehave); + URIDone(Res); + continue; + } + else + RemoveFile("Fetch-Partial", QueueBack->DestFile); + } } - } - + auto const Tmp = QueueBack; + QueueBack = QueueBack->Next; + SendReq(Tmp); + ++CurrentDepth; + } while (CurrentDepth <= AllowedDepth && QueueBack != nullptr); + return true; -}; +} /*}}}*/ // ServerMethod::Loop - Main loop /*{{{*/ int ServerMethod::Loop() @@ -450,10 +520,8 @@ int ServerMethod::Loop() // Connect to the server if (Server == 0 || Server->Comp(Queue->Uri) == false) - { - delete Server; Server = CreateServerState(Queue->Uri); - } + /* If the server has explicitly said this is the last connection then we pre-emptively shut down the pipeline and tear down the connection. This will speed up HTTP/1.0 servers a tad @@ -470,8 +538,7 @@ int ServerMethod::Loop() if (Server->Open() == false) { Fail(true); - delete Server; - Server = 0; + Server = nullptr; continue; } @@ -479,7 +546,7 @@ int ServerMethod::Loop() Fetch(0); // Fetch the next URL header data from the server. - switch (Server->RunHeaders(File)) + switch (Server->RunHeaders(File, Queue->Uri)) { case ServerState::RUN_HEADERS_OK: break; @@ -501,6 +568,7 @@ int ServerMethod::Loop() _error->Discard(); Server->Close(); Server->Pipeline = false; + Server->PipelineAllowed = false; if (FailCounter >= 2) { @@ -525,6 +593,13 @@ int ServerMethod::Loop() // Run the data bool Result = true; + + // ensure we don't fetch too much + // we could do "Server->MaximumSize = Queue->MaximumSize" here + // but that would break the clever pipeline messup detection + // so instead we use the size of the biggest item in the queue + Server->MaximumSize = FindMaximumObjectSizeInQueue(); + if (Server->HaveContent) Result = Server->RunData(File); @@ -539,16 +614,47 @@ int ServerMethod::Loop() File = 0; // Timestamp - struct utimbuf UBuf; - time(&UBuf.actime); - UBuf.actime = Server->Date; - UBuf.modtime = Server->Date; - utime(Queue->DestFile.c_str(),&UBuf); + struct timeval times[2]; + times[0].tv_sec = times[1].tv_sec = Server->Date; + times[0].tv_usec = times[1].tv_usec = 0; + utimes(Queue->DestFile.c_str(), times); // Send status to APT if (Result == true) { - Res.TakeHashes(*Server->GetHashes()); + Hashes * const resultHashes = Server->GetHashes(); + HashStringList const hashList = resultHashes->GetHashStringList(); + if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList) + { + // we did not get the expected hash… mhhh: + // could it be that server/proxy messed up pipelining? + FetchItem * BeforeI = Queue; + for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next) + { + if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList) + { + // yes, he did! Disable pipelining and rewrite queue + if (Server->Pipeline == true) + { + // FIXME: fake a warning message as we have no proper way of communicating here + std::string out; + strprintf(out, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::PipelineDepth"); + std::cerr << "W: " << out << std::endl; + Server->Pipeline = false; + Server->PipelineAllowed = false; + // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well + } + Rename(Res.Filename, I->DestFile); + Res.Filename = I->DestFile; + BeforeI->Next = I->Next; + I->Next = Queue; + Queue = I; + break; + } + BeforeI = I; + } + } + Res.TakeHashes(*resultHashes); URIDone(Res); } else @@ -568,7 +674,10 @@ int ServerMethod::Loop() QueueBack = Queue; } else + { + Server->Close(); Fail(true); + } } break; } @@ -663,3 +772,17 @@ int ServerMethod::Loop() return 0; } /*}}}*/ +unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/ +{ + unsigned long long MaxSizeInQueue = 0; + for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next) + MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize); + return MaxSizeInQueue; +} + /*}}}*/ +ServerMethod::ServerMethod(char const * const Binary, char const * const Ver,unsigned long const Flags) :/*{{{*/ + aptMethod(Binary, Ver, Flags), Server(nullptr), File(NULL), PipelineDepth(10), + AllowRedirect(false), Debug(false) +{ +} + /*}}}*/