1 // -*- mode: cpp; mode: fold -*-
3 /* ######################################################################
5 HTTP and HTTPS share a lot of common code and these classes are
6 exactly the dumping ground for this common code
8 ##################################################################### */
10 // Include Files /*{{{*/
13 #include <apt-pkg/acquire-method.h>
14 #include <apt-pkg/configuration.h>
15 #include <apt-pkg/error.h>
16 #include <apt-pkg/fileutl.h>
17 #include <apt-pkg/strutl.h>
39 string
ServerMethod::FailFile
;
40 int ServerMethod::FailFd
= -1;
41 time_t ServerMethod::FailTime
= 0;
43 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
44 // ---------------------------------------------------------------------
45 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
46 parse error occurred */
47 ServerState::RunHeadersResult
ServerState::RunHeaders(FileFd
* const File
,
48 const std::string
&Uri
)
52 Owner
->Status(_("Waiting for headers"));
67 if (ReadHeaderLines(Data
) == false)
70 if (Owner
->Debug
== true)
71 clog
<< "Answer for: " << Uri
<< endl
<< Data
;
73 for (string::const_iterator I
= Data
.begin(); I
< Data
.end(); ++I
)
75 string::const_iterator J
= I
;
76 for (; J
!= Data
.end() && *J
!= '\n' && *J
!= '\r'; ++J
);
77 if (HeaderLine(string(I
,J
)) == false)
78 return RUN_HEADERS_PARSE_ERROR
;
82 // 100 Continue is a Nop...
86 // Tidy up the connection persistence state.
87 if (Encoding
== Closes
&& HaveContent
== true)
90 return RUN_HEADERS_OK
;
92 while (LoadNextResponse(false, File
) == true);
94 return RUN_HEADERS_IO_ERROR
;
97 // ServerState::HeaderLine - Process a header line /*{{{*/
98 // ---------------------------------------------------------------------
100 bool ServerState::HeaderLine(string Line
)
102 if (Line
.empty() == true)
105 string::size_type Pos
= Line
.find(' ');
106 if (Pos
== string::npos
|| Pos
+1 > Line
.length())
108 // Blah, some servers use "connection:closes", evil.
109 Pos
= Line
.find(':');
110 if (Pos
== string::npos
|| Pos
+ 2 > Line
.length())
111 return _error
->Error(_("Bad header line"));
115 // Parse off any trailing spaces between the : and the next word.
116 string::size_type Pos2
= Pos
;
117 while (Pos2
< Line
.length() && isspace(Line
[Pos2
]) != 0)
120 string Tag
= string(Line
,0,Pos
);
121 string Val
= string(Line
,Pos2
);
123 if (stringcasecmp(Tag
.c_str(),Tag
.c_str()+4,"HTTP") == 0)
125 // Evil servers return no version
128 int const elements
= sscanf(Line
.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major
,&Minor
,&Result
,Code
);
132 if (Owner
!= NULL
&& Owner
->Debug
== true)
133 clog
<< "HTTP server doesn't give Reason-Phrase for " << Result
<< std::endl
;
135 else if (elements
!= 4)
136 return _error
->Error(_("The HTTP server sent an invalid reply header"));
142 if (sscanf(Line
.c_str(),"HTTP %3u%359[^\n]",&Result
,Code
) != 2)
143 return _error
->Error(_("The HTTP server sent an invalid reply header"));
146 /* Check the HTTP response header to get the default persistence
152 if (Major
== 1 && Minor
== 0)
161 if (stringcasecmp(Tag
,"Content-Length:") == 0)
163 if (Encoding
== Closes
)
167 unsigned long long * DownloadSizePtr
= &DownloadSize
;
169 DownloadSizePtr
= &JunkSize
;
171 *DownloadSizePtr
= strtoull(Val
.c_str(), NULL
, 10);
172 if (*DownloadSizePtr
>= std::numeric_limits
<unsigned long long>::max())
173 return _error
->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
174 else if (*DownloadSizePtr
== 0)
177 // On partial content (206) the Content-Length less than the real
178 // size, so do not set it here but leave that to the Content-Range
180 if(Result
!= 206 && TotalFileSize
== 0)
181 TotalFileSize
= DownloadSize
;
186 if (stringcasecmp(Tag
,"Content-Type:") == 0)
192 if (stringcasecmp(Tag
,"Content-Range:") == 0)
196 // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
197 if (Result
== 416 && sscanf(Val
.c_str(), "bytes */%llu",&TotalFileSize
) == 1)
198 ; // we got the expected filesize which is all we wanted
199 else if (sscanf(Val
.c_str(),"bytes %llu-%*u/%llu",&StartPos
,&TotalFileSize
) != 2)
200 return _error
->Error(_("The HTTP server sent an invalid Content-Range header"));
201 if ((unsigned long long)StartPos
> TotalFileSize
)
202 return _error
->Error(_("This HTTP server has broken range support"));
204 // figure out what we will download
205 DownloadSize
= TotalFileSize
- StartPos
;
209 if (stringcasecmp(Tag
,"Transfer-Encoding:") == 0)
212 if (stringcasecmp(Val
,"chunked") == 0)
217 if (stringcasecmp(Tag
,"Connection:") == 0)
219 if (stringcasecmp(Val
,"close") == 0)
221 if (stringcasecmp(Val
,"keep-alive") == 0)
226 if (stringcasecmp(Tag
,"Last-Modified:") == 0)
228 if (RFC1123StrToTime(Val
.c_str(), Date
) == false)
229 return _error
->Error(_("Unknown date format"));
233 if (stringcasecmp(Tag
,"Location:") == 0)
242 // ServerState::ServerState - Constructor /*{{{*/
243 ServerState::ServerState(URI Srv
, ServerMethod
*Owner
) : ServerName(Srv
), TimeOut(120), Owner(Owner
)
248 bool ServerState::AddPartialFileToHashes(FileFd
&File
) /*{{{*/
250 File
.Truncate(StartPos
);
251 return GetHashes()->AddFD(File
, StartPos
);
255 bool ServerMethod::Configuration(string Message
) /*{{{*/
257 if (pkgAcqMethod::Configuration(Message
) == false)
266 // ServerMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
267 // ---------------------------------------------------------------------
268 /* We look at the header data we got back from the server and decide what
269 to do. Returns DealWithHeadersResult (see http.h for details).
271 ServerMethod::DealWithHeadersResult
272 ServerMethod::DealWithHeaders(FetchResult
&Res
)
275 if (Server
->Result
== 304)
277 unlink(Queue
->DestFile
.c_str());
279 Res
.LastModified
= Queue
->LastModified
;
285 * Note that it is only OK for us to treat all redirection the same
286 * because we *always* use GET, not other HTTP methods. There are
287 * three redirection codes for which it is not appropriate that we
288 * redirect. Pass on those codes so the error handling kicks in.
291 && (Server
->Result
> 300 && Server
->Result
< 400)
292 && (Server
->Result
!= 300 // Multiple Choices
293 && Server
->Result
!= 304 // Not Modified
294 && Server
->Result
!= 306)) // (Not part of HTTP/1.1, reserved)
296 if (Server
->Location
.empty() == true);
297 else if (Server
->Location
[0] == '/' && Queue
->Uri
.empty() == false)
299 URI Uri
= Queue
->Uri
;
300 if (Uri
.Host
.empty() == false)
301 NextURI
= URI::SiteOnly(Uri
);
304 NextURI
.append(DeQuoteString(Server
->Location
));
305 return TRY_AGAIN_OR_REDIRECT
;
309 NextURI
= DeQuoteString(Server
->Location
);
310 URI tmpURI
= NextURI
;
311 URI Uri
= Queue
->Uri
;
312 // same protocol redirects are okay
313 if (tmpURI
.Access
== Uri
.Access
)
314 return TRY_AGAIN_OR_REDIRECT
;
315 // as well as http to https
316 else if (Uri
.Access
== "http" && tmpURI
.Access
== "https")
317 return TRY_AGAIN_OR_REDIRECT
;
319 /* else pass through for error message */
321 // retry after an invalid range response without partial data
322 else if (Server
->Result
== 416)
325 if (stat(Queue
->DestFile
.c_str(),&SBuf
) >= 0 && SBuf
.st_size
> 0)
327 bool partialHit
= false;
328 if (Queue
->ExpectedHashes
.usable() == true)
330 Hashes
resultHashes(Queue
->ExpectedHashes
);
331 FileFd
file(Queue
->DestFile
, FileFd::ReadOnly
);
332 Server
->TotalFileSize
= file
.FileSize();
333 Server
->Date
= file
.ModificationTime();
334 resultHashes
.AddFD(file
);
335 HashStringList
const hashList
= resultHashes
.GetHashStringList();
336 partialHit
= (Queue
->ExpectedHashes
== hashList
);
338 else if ((unsigned long long)SBuf
.st_size
== Server
->TotalFileSize
)
340 if (partialHit
== true)
342 // the file is completely downloaded, but was not moved
343 if (Server
->HaveContent
== true)
345 // Send to error page to dev/null
346 FileFd
DevNull("/dev/null",FileFd::WriteExists
);
347 Server
->RunData(&DevNull
);
349 Server
->HaveContent
= false;
350 Server
->StartPos
= Server
->TotalFileSize
;
351 Server
->Result
= 200;
353 else if (unlink(Queue
->DestFile
.c_str()) == 0)
355 NextURI
= Queue
->Uri
;
356 return TRY_AGAIN_OR_REDIRECT
;
361 /* We have a reply we don't handle. This should indicate a perm server
363 if (Server
->Result
< 200 || Server
->Result
>= 300)
366 strprintf(err
, "HttpError%u", Server
->Result
);
368 _error
->Error("%u %s", Server
->Result
, Server
->Code
);
369 if (Server
->HaveContent
== true)
370 return ERROR_WITH_CONTENT_PAGE
;
371 return ERROR_UNRECOVERABLE
;
374 // This is some sort of 2xx 'data follows' reply
375 Res
.LastModified
= Server
->Date
;
376 Res
.Size
= Server
->TotalFileSize
;
380 File
= new FileFd(Queue
->DestFile
,FileFd::WriteAny
);
381 if (_error
->PendingError() == true)
382 return ERROR_NOT_FROM_SERVER
;
384 FailFile
= Queue
->DestFile
;
385 FailFile
.c_str(); // Make sure we don't do a malloc in the signal handler
387 FailTime
= Server
->Date
;
389 if (Server
->InitHashes(Queue
->ExpectedHashes
) == false || Server
->AddPartialFileToHashes(*File
) == false)
391 _error
->Errno("read",_("Problem hashing file"));
392 return ERROR_NOT_FROM_SERVER
;
394 if (Server
->StartPos
> 0)
395 Res
.ResumePoint
= Server
->StartPos
;
397 SetNonBlock(File
->Fd(),true);
401 // ServerMethod::SigTerm - Handle a fatal signal /*{{{*/
402 // ---------------------------------------------------------------------
403 /* This closes and timestamps the open file. This is necessary to get
404 resume behavoir on user abort */
405 void ServerMethod::SigTerm(int)
410 struct timeval times
[2];
411 times
[0].tv_sec
= FailTime
;
412 times
[1].tv_sec
= FailTime
;
413 times
[0].tv_usec
= times
[1].tv_usec
= 0;
414 utimes(FailFile
.c_str(), times
);
420 // ServerMethod::Fetch - Fetch an item /*{{{*/
421 // ---------------------------------------------------------------------
422 /* This adds an item to the pipeline. We keep the pipeline at a fixed
424 bool ServerMethod::Fetch(FetchItem
*)
429 // Queue the requests
431 for (FetchItem
*I
= Queue
; I
!= 0 && Depth
< (signed)PipelineDepth
;
432 I
= I
->Next
, Depth
++)
436 // If pipelining is disabled, we only queue 1 request
437 if (Server
->Pipeline
== false)
439 // if we have no hashes, do at most one such request
440 // as we can't fixup pipeling misbehaviors otherwise
441 else if (I
->ExpectedHashes
.usable() == false)
445 // Make sure we stick with the same server
446 if (Server
->Comp(I
->Uri
) == false)
459 // ServerMethod::Loop - Main loop /*{{{*/
460 int ServerMethod::Loop()
462 typedef vector
<string
> StringVector
;
463 typedef vector
<string
>::iterator StringVectorIterator
;
464 map
<string
, StringVector
> Redirected
;
466 signal(SIGTERM
,SigTerm
);
467 signal(SIGINT
,SigTerm
);
474 // We have no commands, wait for some to arrive
477 if (WaitFd(STDIN_FILENO
) == false)
481 /* Run messages, we can accept 0 (no message) if we didn't
482 do a WaitFd above.. Otherwise the FD is closed. */
483 int Result
= Run(true);
484 if (Result
!= -1 && (Result
!= 0 || Queue
== 0))
486 if(FailReason
.empty() == false ||
487 _config
->FindB("Acquire::http::DependOnSTDIN", true) == true)
496 // Connect to the server
497 if (Server
== 0 || Server
->Comp(Queue
->Uri
) == false)
498 Server
= CreateServerState(Queue
->Uri
);
500 /* If the server has explicitly said this is the last connection
501 then we pre-emptively shut down the pipeline and tear down
502 the connection. This will speed up HTTP/1.0 servers a tad
503 since we don't have to wait for the close sequence to
505 if (Server
->Persistent
== false)
508 // Reset the pipeline
509 if (Server
->IsOpen() == false)
512 // Connnect to the host
513 if (Server
->Open() == false)
520 // Fill the pipeline.
523 // Fetch the next URL header data from the server.
524 switch (Server
->RunHeaders(File
, Queue
->Uri
))
526 case ServerState::RUN_HEADERS_OK
:
529 // The header data is bad
530 case ServerState::RUN_HEADERS_PARSE_ERROR
:
532 _error
->Error(_("Bad header data"));
538 // The server closed a connection during the header get..
540 case ServerState::RUN_HEADERS_IO_ERROR
:
545 Server
->Pipeline
= false;
547 if (FailCounter
>= 2)
549 Fail(_("Connection failed"),true);
558 // Decide what to do.
560 Res
.Filename
= Queue
->DestFile
;
561 switch (DealWithHeaders(Res
))
563 // Ok, the file is Open
571 // ensure we don't fetch too much
572 // we could do "Server->MaximumSize = Queue->MaximumSize" here
573 // but that would break the clever pipeline messup detection
574 // so instead we use the size of the biggest item in the queue
575 Server
->MaximumSize
= FindMaximumObjectSizeInQueue();
577 if (Server
->HaveContent
)
578 Result
= Server
->RunData(File
);
580 /* If the server is sending back sizeless responses then fill in
583 Res
.Size
= File
->Size();
585 // Close the file, destroy the FD object and timestamp it
591 struct timeval times
[2];
592 times
[0].tv_sec
= times
[1].tv_sec
= Server
->Date
;
593 times
[0].tv_usec
= times
[1].tv_usec
= 0;
594 utimes(Queue
->DestFile
.c_str(), times
);
596 // Send status to APT
599 Hashes
* const resultHashes
= Server
->GetHashes();
600 HashStringList
const hashList
= resultHashes
->GetHashStringList();
601 if (PipelineDepth
!= 0 && Queue
->ExpectedHashes
.usable() == true && Queue
->ExpectedHashes
!= hashList
)
603 // we did not get the expected hash… mhhh:
604 // could it be that server/proxy messed up pipelining?
605 FetchItem
* BeforeI
= Queue
;
606 for (FetchItem
*I
= Queue
->Next
; I
!= 0 && I
!= QueueBack
; I
= I
->Next
)
608 if (I
->ExpectedHashes
.usable() == true && I
->ExpectedHashes
== hashList
)
610 // yes, he did! Disable pipelining and rewrite queue
611 if (Server
->Pipeline
== true)
613 // FIXME: fake a warning message as we have no proper way of communicating here
615 strprintf(out
, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::PipelineDepth");
616 std::cerr
<< "W: " << out
<< std::endl
;
617 Server
->Pipeline
= false;
618 // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
620 Rename(Res
.Filename
, I
->DestFile
);
621 Res
.Filename
= I
->DestFile
;
622 BeforeI
->Next
= I
->Next
;
630 Res
.TakeHashes(*resultHashes
);
635 if (Server
->IsOpen() == false)
641 if (FailCounter
>= 2)
643 Fail(_("Connection failed"),true);
665 // Hard server error, not found or something
666 case ERROR_UNRECOVERABLE
:
672 // Hard internal error, kill the connection and fail
673 case ERROR_NOT_FROM_SERVER
:
684 // We need to flush the data, the header is like a 404 w/ error text
685 case ERROR_WITH_CONTENT_PAGE
:
689 // Send to content to dev/null
690 File
= new FileFd("/dev/null",FileFd::WriteExists
);
691 Server
->RunData(File
);
697 // Try again with a new URL
698 case TRY_AGAIN_OR_REDIRECT
:
700 // Clear rest of response if there is content
701 if (Server
->HaveContent
)
703 File
= new FileFd("/dev/null",FileFd::WriteExists
);
704 Server
->RunData(File
);
709 /* Detect redirect loops. No more redirects are allowed
710 after the same URI is seen twice in a queue item. */
711 StringVector
&R
= Redirected
[Queue
->DestFile
];
712 bool StopRedirects
= false;
713 if (R
.empty() == true)
714 R
.push_back(Queue
->Uri
);
715 else if (R
[0] == "STOP" || R
.size() > 10)
716 StopRedirects
= true;
719 for (StringVectorIterator I
= R
.begin(); I
!= R
.end(); ++I
)
720 if (Queue
->Uri
== *I
)
726 R
.push_back(Queue
->Uri
);
729 if (StopRedirects
== false)
738 Fail(_("Internal error"));
748 unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/
750 unsigned long long MaxSizeInQueue
= 0;
751 for (FetchItem
*I
= Queue
; I
!= 0 && I
!= QueueBack
; I
= I
->Next
)
752 MaxSizeInQueue
= std::max(MaxSizeInQueue
, I
->MaximumSize
);
753 return MaxSizeInQueue
;
756 ServerMethod::ServerMethod(const char *Ver
,unsigned long Flags
) : /*{{{*/
757 pkgAcqMethod(Ver
, Flags
), Server(nullptr), File(NULL
), PipelineDepth(10),
758 AllowRedirect(false), Debug(false)