// Include Files /*{{{*/
#include <config.h>
-#include <apt-pkg/fileutl.h>
#include <apt-pkg/acquire-method.h>
#include <apt-pkg/configuration.h>
#include <apt-pkg/error.h>
-#include <apt-pkg/hashes.h>
-#include <apt-pkg/netrc.h>
+#include <apt-pkg/fileutl.h>
+#include <apt-pkg/strutl.h>
-#include <fcntl.h>
+#include <ctype.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
#include <sys/stat.h>
#include <sys/time.h>
+#include <time.h>
#include <unistd.h>
-#include <signal.h>
-#include <stdio.h>
-#include <errno.h>
-#include <string.h>
-#include <climits>
#include <iostream>
+#include <limits>
#include <map>
+#include <string>
+#include <vector>
-// Internet stuff
-#include <netdb.h>
-
-#include "config.h"
-#include "connect.h"
-#include "rfc2553emu.h"
-#include "http.h"
+#include "server.h"
#include <apti18n.h>
/*}}}*/
// ---------------------------------------------------------------------
/* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
parse error occurred */
-ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File)
+ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
+ const std::string &Uri)
{
State = Header;
Major = 0;
Minor = 0;
Result = 0;
- Size = 0;
+ TotalFileSize = 0;
+ JunkSize = 0;
StartPos = 0;
Encoding = Closes;
HaveContent = false;
continue;
if (Owner->Debug == true)
- clog << Data;
+ clog << "Answer for: " << Uri << endl << Data;
for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
{
if (Result == 100)
continue;
- // Tidy up the connection persistance state.
+ // Tidy up the connection persistence state.
if (Encoding == Closes && HaveContent == true)
Persistent = false;
string::size_type Pos2 = Pos;
while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
Pos2++;
-
+
string Tag = string(Line,0,Pos);
string Val = string(Line,Pos2);
-
+
if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
{
// Evil servers return no version
if (elements == 3)
{
Code[0] = '\0';
- if (Owner->Debug == true)
+ if (Owner != NULL && Owner->Debug == true)
clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
}
else if (elements != 4)
return _error->Error(_("The HTTP server sent an invalid reply header"));
}
- /* Check the HTTP response header to get the default persistance
+ /* Check the HTTP response header to get the default persistence
state. */
if (Major < 1)
Persistent = false;
}
return true;
- }
-
+ }
+
if (stringcasecmp(Tag,"Content-Length:") == 0)
{
if (Encoding == Closes)
Encoding = Stream;
HaveContent = true;
-
- // The length is already set from the Content-Range header
- if (StartPos != 0)
- return true;
- Size = strtoull(Val.c_str(), NULL, 10);
- if (Size >= std::numeric_limits<unsigned long long>::max())
+ unsigned long long * DownloadSizePtr = &DownloadSize;
+ if (Result == 416)
+ DownloadSizePtr = &JunkSize;
+
+ *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10);
+ if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max())
return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
- else if (Size == 0)
+ else if (*DownloadSizePtr == 0)
HaveContent = false;
+
+ // On partial content (206) the Content-Length less than the real
+ // size, so do not set it here but leave that to the Content-Range
+ // header instead
+ if(Result != 206 && TotalFileSize == 0)
+ TotalFileSize = DownloadSize;
+
return true;
}
HaveContent = true;
return true;
}
-
+
if (stringcasecmp(Tag,"Content-Range:") == 0)
{
HaveContent = true;
// §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
- if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
- {
- StartPos = 1; // ignore Content-Length, it would override Size
- HaveContent = false;
- }
- else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
+ if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
+ ; // we got the expected filesize which is all we wanted
+ else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
- if ((unsigned long long)StartPos > Size)
+ if ((unsigned long long)StartPos > TotalFileSize)
return _error->Error(_("This HTTP server has broken range support"));
+
+ // figure out what we will download
+ DownloadSize = TotalFileSize - StartPos;
return true;
}
-
+
if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
{
HaveContent = true;
if (stringcasecmp(Val,"chunked") == 0)
- Encoding = Chunked;
+ Encoding = Chunked;
return true;
}
Persistent = true;
return true;
}
-
+
if (stringcasecmp(Tag,"Last-Modified:") == 0)
{
if (RFC1123StrToTime(Val.c_str(), Date) == false)
Reset();
}
/*}}}*/
-
-bool ServerMethod::Configuration(string Message) /*{{{*/
+bool ServerState::AddPartialFileToHashes(FileFd &File) /*{{{*/
{
- return pkgAcqMethod::Configuration(Message);
+ File.Truncate(StartPos);
+ return GetHashes()->AddFD(File, StartPos);
}
/*}}}*/
// Not Modified
if (Server->Result == 304)
{
- unlink(Queue->DestFile.c_str());
+ RemoveFile("server", Queue->DestFile);
Res.IMSHit = true;
Res.LastModified = Queue->LastModified;
return IMS_HIT;
}
-
+
/* Redirect
*
* Note that it is only OK for us to treat all redirection the same
}
else
{
- NextURI = DeQuoteString(Server->Location);
- URI tmpURI = NextURI;
- // Do not allow a redirection to switch protocol
- if (tmpURI.Access == "http")
- return TRY_AGAIN_OR_REDIRECT;
+ NextURI = DeQuoteString(Server->Location);
+ URI tmpURI = NextURI;
+ URI Uri = Queue->Uri;
+ // same protocol redirects are okay
+ if (tmpURI.Access == Uri.Access)
+ return TRY_AGAIN_OR_REDIRECT;
+ // as well as http to https
+ else if (Uri.Access == "http" && tmpURI.Access == "https")
+ return TRY_AGAIN_OR_REDIRECT;
}
/* else pass through for error message */
}
struct stat SBuf;
if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
{
- if ((unsigned long long)SBuf.st_size == Server->Size)
+ bool partialHit = false;
+ if (Queue->ExpectedHashes.usable() == true)
+ {
+ Hashes resultHashes(Queue->ExpectedHashes);
+ FileFd file(Queue->DestFile, FileFd::ReadOnly);
+ Server->TotalFileSize = file.FileSize();
+ Server->Date = file.ModificationTime();
+ resultHashes.AddFD(file);
+ HashStringList const hashList = resultHashes.GetHashStringList();
+ partialHit = (Queue->ExpectedHashes == hashList);
+ }
+ else if ((unsigned long long)SBuf.st_size == Server->TotalFileSize)
+ partialHit = true;
+ if (partialHit == true)
{
// the file is completely downloaded, but was not moved
- Server->StartPos = Server->Size;
- Server->Result = 200;
+ if (Server->HaveContent == true)
+ {
+ // Send to error page to dev/null
+ FileFd DevNull("/dev/null",FileFd::WriteExists);
+ Server->RunData(&DevNull);
+ }
Server->HaveContent = false;
+ Server->StartPos = Server->TotalFileSize;
+ Server->Result = 200;
}
- else if (unlink(Queue->DestFile.c_str()) == 0)
+ else if (RemoveFile("server", Queue->DestFile))
{
NextURI = Queue->Uri;
return TRY_AGAIN_OR_REDIRECT;
}
}
- /* We have a reply we dont handle. This should indicate a perm server
+ /* We have a reply we don't handle. This should indicate a perm server
failure */
if (Server->Result < 200 || Server->Result >= 300)
{
- char err[255];
- snprintf(err,sizeof(err)-1,"HttpError%i",Server->Result);
+ std::string err;
+ strprintf(err, "HttpError%u", Server->Result);
SetFailReason(err);
- _error->Error("%u %s",Server->Result,Server->Code);
+ _error->Error("%u %s", Server->Result, Server->Code);
if (Server->HaveContent == true)
return ERROR_WITH_CONTENT_PAGE;
return ERROR_UNRECOVERABLE;
// This is some sort of 2xx 'data follows' reply
Res.LastModified = Server->Date;
- Res.Size = Server->Size;
+ Res.Size = Server->TotalFileSize;
// Open the file
delete File;
return ERROR_NOT_FROM_SERVER;
FailFile = Queue->DestFile;
- FailFile.c_str(); // Make sure we dont do a malloc in the signal handler
+ FailFile.c_str(); // Make sure we don't do a malloc in the signal handler
FailFd = File->Fd();
FailTime = Server->Date;
- if (Server->InitHashes(*File) == false)
+ if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
{
_error->Errno("read",_("Problem hashing file"));
return ERROR_NOT_FROM_SERVER;
/*}}}*/
// ServerMethod::SigTerm - Handle a fatal signal /*{{{*/
// ---------------------------------------------------------------------
-/* This closes and timestamps the open file. This is neccessary to get
+/* This closes and timestamps the open file. This is necessary to get
resume behavoir on user abort */
void ServerMethod::SigTerm(int)
{
if (FailFd == -1)
_exit(100);
- struct timespec times[2];
+ struct timeval times[2];
times[0].tv_sec = FailTime;
times[1].tv_sec = FailTime;
- times[0].tv_nsec = times[1].tv_nsec = 0;
- futimens(FailFd, times);
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(FailFile.c_str(), times);
close(FailFd);
_exit(100);
for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
I = I->Next, Depth++)
{
- // If pipelining is disabled, we only queue 1 request
- if (Server->Pipeline == false && Depth >= 0)
- break;
+ if (Depth >= 0)
+ {
+ // If pipelining is disabled, we only queue 1 request
+ if (Server->Pipeline == false)
+ break;
+ // if we have no hashes, do at most one such request
+ // as we can't fixup pipeling misbehaviors otherwise
+ else if (I->ExpectedHashes.usable() == false)
+ break;
+ }
// Make sure we stick with the same server
if (Server->Comp(I->Uri) == false)
}
return true;
-};
+}
/*}}}*/
// ServerMethod::Loop - Main loop /*{{{*/
int ServerMethod::Loop()
// Connect to the server
if (Server == 0 || Server->Comp(Queue->Uri) == false)
- {
- delete Server;
Server = CreateServerState(Queue->Uri);
- }
+
/* If the server has explicitly said this is the last connection
then we pre-emptively shut down the pipeline and tear down
the connection. This will speed up HTTP/1.0 servers a tad
if (Server->Open() == false)
{
Fail(true);
- delete Server;
- Server = 0;
+ Server = nullptr;
continue;
}
Fetch(0);
// Fetch the next URL header data from the server.
- switch (Server->RunHeaders(File))
+ switch (Server->RunHeaders(File, Queue->Uri))
{
case ServerState::RUN_HEADERS_OK:
break;
// Run the data
bool Result = true;
+
+ // ensure we don't fetch too much
+ // we could do "Server->MaximumSize = Queue->MaximumSize" here
+ // but that would break the clever pipeline messup detection
+ // so instead we use the size of the biggest item in the queue
+ Server->MaximumSize = FindMaximumObjectSizeInQueue();
+
if (Server->HaveContent)
Result = Server->RunData(File);
File = 0;
// Timestamp
- struct timespec times[2];
+ struct timeval times[2];
times[0].tv_sec = times[1].tv_sec = Server->Date;
- times[0].tv_nsec = times[1].tv_nsec = 0;
- utimensat(AT_FDCWD, Queue->DestFile.c_str(), times, AT_SYMLINK_NOFOLLOW);
+ times[0].tv_usec = times[1].tv_usec = 0;
+ utimes(Queue->DestFile.c_str(), times);
// Send status to APT
if (Result == true)
{
- Res.TakeHashes(*Server->GetHashes());
+ Hashes * const resultHashes = Server->GetHashes();
+ HashStringList const hashList = resultHashes->GetHashStringList();
+ if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
+ {
+ // we did not get the expected hash… mhhh:
+ // could it be that server/proxy messed up pipelining?
+ FetchItem * BeforeI = Queue;
+ for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
+ {
+ if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
+ {
+ // yes, he did! Disable pipelining and rewrite queue
+ if (Server->Pipeline == true)
+ {
+ // FIXME: fake a warning message as we have no proper way of communicating here
+ std::string out;
+ strprintf(out, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::PipelineDepth");
+ std::cerr << "W: " << out << std::endl;
+ Server->Pipeline = false;
+ // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
+ }
+ Rename(Res.Filename, I->DestFile);
+ Res.Filename = I->DestFile;
+ BeforeI->Next = I->Next;
+ I->Next = Queue;
+ Queue = I;
+ break;
+ }
+ BeforeI = I;
+ }
+ }
+ Res.TakeHashes(*resultHashes);
URIDone(Res);
}
else
QueueBack = Queue;
}
else
+ {
+ Server->Close();
Fail(true);
+ }
}
break;
}
return 0;
}
/*}}}*/
+unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/
+{
+ unsigned long long MaxSizeInQueue = 0;
+ for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
+ MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
+ return MaxSizeInQueue;
+}
+ /*}}}*/
+ServerMethod::ServerMethod(char const * const Binary, char const * const Ver,unsigned long const Flags) :/*{{{*/
+ aptMethod(Binary, Ver, Flags), Server(nullptr), File(NULL), PipelineDepth(10),
+ AllowRedirect(false), Debug(false)
+{
+}
+ /*}}}*/
projects / apt.git / blobdiff