]>
git.saurik.com Git - apt.git/blob - methods/server.cc
1 // -*- mode: cpp; mode: fold -*-
3 /* ######################################################################
5 HTTP and HTTPS share a lot of common code and these classes are
6 exactly the dumping ground for this common code
8 ##################################################################### */
10 // Include Files /*{{{*/
13 #include <apt-pkg/configuration.h>
14 #include <apt-pkg/error.h>
15 #include <apt-pkg/fileutl.h>
16 #include <apt-pkg/strutl.h>
38 string
ServerMethod::FailFile
;
39 int ServerMethod::FailFd
= -1;
40 time_t ServerMethod::FailTime
= 0;
42 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
43 // ---------------------------------------------------------------------
44 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
45 parse error occurred */
46 ServerState::RunHeadersResult
ServerState::RunHeaders(FileFd
* const File
,
47 const std::string
&Uri
)
51 Owner
->Status(_("Waiting for headers"));
66 if (ReadHeaderLines(Data
) == false)
69 if (Owner
->Debug
== true)
70 clog
<< "Answer for: " << Uri
<< endl
<< Data
;
72 for (string::const_iterator I
= Data
.begin(); I
< Data
.end(); ++I
)
74 string::const_iterator J
= I
;
75 for (; J
!= Data
.end() && *J
!= '\n' && *J
!= '\r'; ++J
);
76 if (HeaderLine(string(I
,J
)) == false)
77 return RUN_HEADERS_PARSE_ERROR
;
81 // 100 Continue is a Nop...
85 // Tidy up the connection persistence state.
86 if (Encoding
== Closes
&& HaveContent
== true)
89 return RUN_HEADERS_OK
;
91 while (LoadNextResponse(false, File
) == true);
93 return RUN_HEADERS_IO_ERROR
;
96 // ServerState::HeaderLine - Process a header line /*{{{*/
97 // ---------------------------------------------------------------------
99 bool ServerState::HeaderLine(string Line
)
101 if (Line
.empty() == true)
104 string::size_type Pos
= Line
.find(' ');
105 if (Pos
== string::npos
|| Pos
+1 > Line
.length())
107 // Blah, some servers use "connection:closes", evil.
108 Pos
= Line
.find(':');
109 if (Pos
== string::npos
|| Pos
+ 2 > Line
.length())
110 return _error
->Error(_("Bad header line"));
114 // Parse off any trailing spaces between the : and the next word.
115 string::size_type Pos2
= Pos
;
116 while (Pos2
< Line
.length() && isspace_ascii(Line
[Pos2
]) != 0)
119 string Tag
= string(Line
,0,Pos
);
120 string Val
= string(Line
,Pos2
);
122 if (stringcasecmp(Tag
.c_str(),Tag
.c_str()+4,"HTTP") == 0)
124 // Evil servers return no version
127 int const elements
= sscanf(Line
.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major
,&Minor
,&Result
,Code
);
131 if (Owner
!= NULL
&& Owner
->Debug
== true)
132 clog
<< "HTTP server doesn't give Reason-Phrase for " << std::to_string(Result
) << std::endl
;
134 else if (elements
!= 4)
135 return _error
->Error(_("The HTTP server sent an invalid reply header"));
141 if (sscanf(Line
.c_str(),"HTTP %3u%359[^\n]",&Result
,Code
) != 2)
142 return _error
->Error(_("The HTTP server sent an invalid reply header"));
145 /* Check the HTTP response header to get the default persistence
151 if (Major
== 1 && Minor
== 0)
166 if (stringcasecmp(Tag
,"Content-Length:") == 0)
168 if (Encoding
== Closes
)
172 unsigned long long * DownloadSizePtr
= &DownloadSize
;
173 if (Result
== 416 || (Result
>= 300 && Result
< 400))
174 DownloadSizePtr
= &JunkSize
;
176 *DownloadSizePtr
= strtoull(Val
.c_str(), NULL
, 10);
177 if (*DownloadSizePtr
>= std::numeric_limits
<unsigned long long>::max())
178 return _error
->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
179 else if (*DownloadSizePtr
== 0)
182 // On partial content (206) the Content-Length less than the real
183 // size, so do not set it here but leave that to the Content-Range
185 if(Result
!= 206 && TotalFileSize
== 0)
186 TotalFileSize
= DownloadSize
;
191 if (stringcasecmp(Tag
,"Content-Type:") == 0)
197 if (stringcasecmp(Tag
,"Content-Range:") == 0)
201 // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
202 if (Result
== 416 && sscanf(Val
.c_str(), "bytes */%llu",&TotalFileSize
) == 1)
203 ; // we got the expected filesize which is all we wanted
204 else if (sscanf(Val
.c_str(),"bytes %llu-%*u/%llu",&StartPos
,&TotalFileSize
) != 2)
205 return _error
->Error(_("The HTTP server sent an invalid Content-Range header"));
206 if ((unsigned long long)StartPos
> TotalFileSize
)
207 return _error
->Error(_("This HTTP server has broken range support"));
209 // figure out what we will download
210 DownloadSize
= TotalFileSize
- StartPos
;
214 if (stringcasecmp(Tag
,"Transfer-Encoding:") == 0)
217 if (stringcasecmp(Val
,"chunked") == 0)
222 if (stringcasecmp(Tag
,"Connection:") == 0)
224 if (stringcasecmp(Val
,"close") == 0)
226 if (stringcasecmp(Val
,"keep-alive") == 0)
231 if (stringcasecmp(Tag
,"Last-Modified:") == 0)
233 if (RFC1123StrToTime(Val
.c_str(), Date
) == false)
234 return _error
->Error(_("Unknown date format"));
238 if (stringcasecmp(Tag
,"Location:") == 0)
247 // ServerState::ServerState - Constructor /*{{{*/
248 ServerState::ServerState(URI Srv
, ServerMethod
*Owner
) :
249 DownloadSize(0), ServerName(Srv
), TimeOut(120), Owner(Owner
)
254 bool ServerState::AddPartialFileToHashes(FileFd
&File
) /*{{{*/
256 File
.Truncate(StartPos
);
257 return GetHashes()->AddFD(File
, StartPos
);
261 // ServerMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
262 // ---------------------------------------------------------------------
263 /* We look at the header data we got back from the server and decide what
264 to do. Returns DealWithHeadersResult (see http.h for details).
266 ServerMethod::DealWithHeadersResult
267 ServerMethod::DealWithHeaders(FetchResult
&Res
)
270 if (Server
->Result
== 304)
272 RemoveFile("server", Queue
->DestFile
);
274 Res
.LastModified
= Queue
->LastModified
;
281 * Note that it is only OK for us to treat all redirection the same
282 * because we *always* use GET, not other HTTP methods. There are
283 * three redirection codes for which it is not appropriate that we
284 * redirect. Pass on those codes so the error handling kicks in.
287 && (Server
->Result
> 300 && Server
->Result
< 400)
288 && (Server
->Result
!= 300 // Multiple Choices
289 && Server
->Result
!= 304 // Not Modified
290 && Server
->Result
!= 306)) // (Not part of HTTP/1.1, reserved)
292 if (Server
->Location
.empty() == true)
294 else if (Server
->Location
[0] == '/' && Queue
->Uri
.empty() == false)
296 URI Uri
= Queue
->Uri
;
297 if (Uri
.Host
.empty() == false)
298 NextURI
= URI::SiteOnly(Uri
);
301 NextURI
.append(DeQuoteString(Server
->Location
));
302 if (Queue
->Uri
== NextURI
)
304 SetFailReason("RedirectionLoop");
305 _error
->Error("Redirection loop encountered");
306 if (Server
->HaveContent
== true)
307 return ERROR_WITH_CONTENT_PAGE
;
308 return ERROR_UNRECOVERABLE
;
310 return TRY_AGAIN_OR_REDIRECT
;
314 NextURI
= DeQuoteString(Server
->Location
);
315 URI tmpURI
= NextURI
;
316 if (tmpURI
.Access
== "http" && Binary
== "https+http")
318 tmpURI
.Access
= "https+http";
321 if (Queue
->Uri
== NextURI
)
323 SetFailReason("RedirectionLoop");
324 _error
->Error("Redirection loop encountered");
325 if (Server
->HaveContent
== true)
326 return ERROR_WITH_CONTENT_PAGE
;
327 return ERROR_UNRECOVERABLE
;
329 URI Uri
= Queue
->Uri
;
330 // same protocol redirects are okay
331 if (tmpURI
.Access
== Uri
.Access
)
332 return TRY_AGAIN_OR_REDIRECT
;
333 // as well as http to https
334 else if ((Uri
.Access
== "http" || Uri
.Access
== "https+http") && tmpURI
.Access
== "https")
335 return TRY_AGAIN_OR_REDIRECT
;
337 _error
->Error("Redirection from %s to '%s' is forbidden", Uri
.Access
.c_str(), NextURI
.c_str());
339 /* else pass through for error message */
341 // retry after an invalid range response without partial data
342 else if (Server
->Result
== 416)
345 if (stat(Queue
->DestFile
.c_str(),&SBuf
) >= 0 && SBuf
.st_size
> 0)
347 bool partialHit
= false;
348 if (Queue
->ExpectedHashes
.usable() == true)
350 Hashes
resultHashes(Queue
->ExpectedHashes
);
351 FileFd
file(Queue
->DestFile
, FileFd::ReadOnly
);
352 Server
->TotalFileSize
= file
.FileSize();
353 Server
->Date
= file
.ModificationTime();
354 resultHashes
.AddFD(file
);
355 HashStringList
const hashList
= resultHashes
.GetHashStringList();
356 partialHit
= (Queue
->ExpectedHashes
== hashList
);
358 else if ((unsigned long long)SBuf
.st_size
== Server
->TotalFileSize
)
360 if (partialHit
== true)
362 // the file is completely downloaded, but was not moved
363 if (Server
->HaveContent
== true)
365 // nuke the sent error page
366 Server
->RunDataToDevNull();
367 Server
->HaveContent
= false;
369 Server
->StartPos
= Server
->TotalFileSize
;
370 Server
->Result
= 200;
372 else if (RemoveFile("server", Queue
->DestFile
))
374 NextURI
= Queue
->Uri
;
375 return TRY_AGAIN_OR_REDIRECT
;
380 /* We have a reply we don't handle. This should indicate a perm server
382 if (Server
->Result
< 200 || Server
->Result
>= 300)
384 if (_error
->PendingError() == false)
387 strprintf(err
, "HttpError%u", Server
->Result
);
389 _error
->Error("%u %s", Server
->Result
, Server
->Code
);
391 if (Server
->HaveContent
== true)
392 return ERROR_WITH_CONTENT_PAGE
;
393 return ERROR_UNRECOVERABLE
;
396 // This is some sort of 2xx 'data follows' reply
397 Res
.LastModified
= Server
->Date
;
398 Res
.Size
= Server
->TotalFileSize
;
402 // ServerMethod::SigTerm - Handle a fatal signal /*{{{*/
403 // ---------------------------------------------------------------------
404 /* This closes and timestamps the open file. This is necessary to get
405 resume behavoir on user abort */
406 void ServerMethod::SigTerm(int)
411 struct timeval times
[2];
412 times
[0].tv_sec
= FailTime
;
413 times
[1].tv_sec
= FailTime
;
414 times
[0].tv_usec
= times
[1].tv_usec
= 0;
415 utimes(FailFile
.c_str(), times
);
421 // ServerMethod::Fetch - Fetch an item /*{{{*/
422 // ---------------------------------------------------------------------
423 /* This adds an item to the pipeline. We keep the pipeline at a fixed
425 bool ServerMethod::Fetch(FetchItem
*)
427 if (Server
== nullptr || QueueBack
== nullptr)
430 // If pipelining is disabled, we only queue 1 request
431 auto const AllowedDepth
= Server
->Pipeline
? PipelineDepth
: 0;
432 // how deep is our pipeline currently?
433 decltype(PipelineDepth
) CurrentDepth
= 0;
434 for (FetchItem
const *I
= Queue
; I
!= QueueBack
; I
= I
->Next
)
436 if (CurrentDepth
> AllowedDepth
)
440 // Make sure we stick with the same server
441 if (Server
->Comp(QueueBack
->Uri
) == false)
444 bool const UsableHashes
= QueueBack
->ExpectedHashes
.usable();
445 // if we have no hashes, do at most one such request
446 // as we can't fixup pipeling misbehaviors otherwise
447 if (CurrentDepth
!= 0 && UsableHashes
== false)
450 if (UsableHashes
&& FileExists(QueueBack
->DestFile
))
452 FileFd
partial(QueueBack
->DestFile
, FileFd::ReadOnly
);
453 Hashes
wehave(QueueBack
->ExpectedHashes
);
454 if (QueueBack
->ExpectedHashes
.FileSize() == partial
.FileSize())
456 if (wehave
.AddFD(partial
) &&
457 wehave
.GetHashStringList() == QueueBack
->ExpectedHashes
)
460 Res
.Filename
= QueueBack
->DestFile
;
461 Res
.ResumePoint
= QueueBack
->ExpectedHashes
.FileSize();
463 // move item to the start of the queue as URIDone will
464 // always dequeued the first item in the queue
465 if (Queue
!= QueueBack
)
467 FetchItem
*Prev
= Queue
;
468 for (; Prev
->Next
!= QueueBack
; Prev
= Prev
->Next
)
469 /* look for the previous queue item */;
470 Prev
->Next
= QueueBack
->Next
;
471 QueueBack
->Next
= Queue
;
473 QueueBack
= Prev
->Next
;
475 Res
.TakeHashes(wehave
);
480 RemoveFile("Fetch-Partial", QueueBack
->DestFile
);
483 auto const Tmp
= QueueBack
;
484 QueueBack
= QueueBack
->Next
;
487 } while (CurrentDepth
<= AllowedDepth
&& QueueBack
!= nullptr);
492 // ServerMethod::Loop - Main loop /*{{{*/
493 int ServerMethod::Loop()
495 signal(SIGTERM
,SigTerm
);
496 signal(SIGINT
,SigTerm
);
503 // We have no commands, wait for some to arrive
506 if (WaitFd(STDIN_FILENO
) == false)
510 /* Run messages, we can accept 0 (no message) if we didn't
511 do a WaitFd above.. Otherwise the FD is closed. */
512 int Result
= Run(true);
513 if (Result
!= -1 && (Result
!= 0 || Queue
== 0))
515 if(FailReason
.empty() == false ||
516 _config
->FindB("Acquire::http::DependOnSTDIN", true) == true)
525 // Connect to the server
526 if (Server
== 0 || Server
->Comp(Queue
->Uri
) == false)
527 Server
= CreateServerState(Queue
->Uri
);
529 /* If the server has explicitly said this is the last connection
530 then we pre-emptively shut down the pipeline and tear down
531 the connection. This will speed up HTTP/1.0 servers a tad
532 since we don't have to wait for the close sequence to
534 if (Server
->Persistent
== false)
537 // Reset the pipeline
538 if (Server
->IsOpen() == false)
541 // Connnect to the host
542 if (Server
->Open() == false)
549 // Fill the pipeline.
552 // Fetch the next URL header data from the server.
553 switch (Server
->RunHeaders(File
, Queue
->Uri
))
555 case ServerState::RUN_HEADERS_OK
:
558 // The header data is bad
559 case ServerState::RUN_HEADERS_PARSE_ERROR
:
561 _error
->Error(_("Bad header data"));
568 // The server closed a connection during the header get..
570 case ServerState::RUN_HEADERS_IO_ERROR
:
575 Server
->Pipeline
= false;
576 Server
->PipelineAllowed
= false;
578 if (FailCounter
>= 2)
580 Fail(_("Connection failed"),true);
589 // Decide what to do.
591 Res
.Filename
= Queue
->DestFile
;
592 switch (DealWithHeaders(Res
))
594 // Ok, the file is Open
602 // ensure we don't fetch too much
603 // we could do "Server->MaximumSize = Queue->MaximumSize" here
604 // but that would break the clever pipeline messup detection
605 // so instead we use the size of the biggest item in the queue
606 Server
->MaximumSize
= FindMaximumObjectSizeInQueue();
608 if (Server
->HaveContent
)
609 Result
= Server
->RunData(File
);
611 /* If the server is sending back sizeless responses then fill in
614 Res
.Size
= File
->Size();
616 // Close the file, destroy the FD object and timestamp it
622 struct timeval times
[2];
623 times
[0].tv_sec
= times
[1].tv_sec
= Server
->Date
;
624 times
[0].tv_usec
= times
[1].tv_usec
= 0;
625 utimes(Queue
->DestFile
.c_str(), times
);
627 // Send status to APT
630 Hashes
* const resultHashes
= Server
->GetHashes();
631 HashStringList
const hashList
= resultHashes
->GetHashStringList();
632 if (PipelineDepth
!= 0 && Queue
->ExpectedHashes
.usable() == true && Queue
->ExpectedHashes
!= hashList
)
634 // we did not get the expected hash… mhhh:
635 // could it be that server/proxy messed up pipelining?
636 FetchItem
* BeforeI
= Queue
;
637 for (FetchItem
*I
= Queue
->Next
; I
!= 0 && I
!= QueueBack
; I
= I
->Next
)
639 if (I
->ExpectedHashes
.usable() == true && I
->ExpectedHashes
== hashList
)
641 // yes, he did! Disable pipelining and rewrite queue
642 if (Server
->Pipeline
== true)
644 Warning(_("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::Pipeline-Depth");
645 Server
->Pipeline
= false;
646 Server
->PipelineAllowed
= false;
647 // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
649 Rename(Res
.Filename
, I
->DestFile
);
650 Res
.Filename
= I
->DestFile
;
651 BeforeI
->Next
= I
->Next
;
659 Res
.TakeHashes(*resultHashes
);
664 if (Server
->IsOpen() == false)
670 if (FailCounter
>= 2)
672 Fail(_("Connection failed"),true);
694 // Hard server error, not found or something
695 case ERROR_UNRECOVERABLE
:
701 // Hard internal error, kill the connection and fail
702 case ERROR_NOT_FROM_SERVER
:
713 // We need to flush the data, the header is like a 404 w/ error text
714 case ERROR_WITH_CONTENT_PAGE
:
717 Server
->RunDataToDevNull();
721 // Try again with a new URL
722 case TRY_AGAIN_OR_REDIRECT
:
724 // Clear rest of response if there is content
725 if (Server
->HaveContent
)
726 Server
->RunDataToDevNull();
732 Fail(_("Internal error"));
742 unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/
744 unsigned long long MaxSizeInQueue
= 0;
745 for (FetchItem
*I
= Queue
; I
!= 0 && I
!= QueueBack
; I
= I
->Next
)
746 MaxSizeInQueue
= std::max(MaxSizeInQueue
, I
->MaximumSize
);
747 return MaxSizeInQueue
;
750 ServerMethod::ServerMethod(char const * const Binary
, char const * const Ver
,unsigned long const Flags
) :/*{{{*/
751 aptMethod(Binary
, Ver
, Flags
), Server(nullptr), File(NULL
), PipelineDepth(10),
752 AllowRedirect(false), Debug(false)