]> git.saurik.com Git - apt.git/blob - methods/server.cc
don't try pipelining if server closes connections
[apt.git] / methods / server.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 /* ######################################################################
4
5 HTTP and HTTPS share a lot of common code and these classes are
6 exactly the dumping ground for this common code
7
8 ##################################################################### */
9 /*}}}*/
10 // Include Files /*{{{*/
11 #include <config.h>
12
13 #include <apt-pkg/acquire-method.h>
14 #include <apt-pkg/configuration.h>
15 #include <apt-pkg/error.h>
16 #include <apt-pkg/fileutl.h>
17 #include <apt-pkg/strutl.h>
18
19 #include <ctype.h>
20 #include <signal.h>
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <sys/stat.h>
24 #include <sys/time.h>
25 #include <time.h>
26 #include <unistd.h>
27 #include <iostream>
28 #include <limits>
29 #include <map>
30 #include <string>
31 #include <vector>
32
33 #include "server.h"
34
35 #include <apti18n.h>
36 /*}}}*/
37 using namespace std;
38
39 string ServerMethod::FailFile;
40 int ServerMethod::FailFd = -1;
41 time_t ServerMethod::FailTime = 0;
42
43 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
44 // ---------------------------------------------------------------------
45 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
46 parse error occurred */
47 ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
48 const std::string &Uri)
49 {
50 State = Header;
51
52 Owner->Status(_("Waiting for headers"));
53
54 Major = 0;
55 Minor = 0;
56 Result = 0;
57 TotalFileSize = 0;
58 JunkSize = 0;
59 StartPos = 0;
60 Encoding = Closes;
61 HaveContent = false;
62 time(&Date);
63
64 do
65 {
66 string Data;
67 if (ReadHeaderLines(Data) == false)
68 continue;
69
70 if (Owner->Debug == true)
71 clog << "Answer for: " << Uri << endl << Data;
72
73 for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
74 {
75 string::const_iterator J = I;
76 for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
77 if (HeaderLine(string(I,J)) == false)
78 return RUN_HEADERS_PARSE_ERROR;
79 I = J;
80 }
81
82 // 100 Continue is a Nop...
83 if (Result == 100)
84 continue;
85
86 // Tidy up the connection persistence state.
87 if (Encoding == Closes && HaveContent == true)
88 Persistent = false;
89
90 return RUN_HEADERS_OK;
91 }
92 while (LoadNextResponse(false, File) == true);
93
94 return RUN_HEADERS_IO_ERROR;
95 }
96 /*}}}*/
97 // ServerState::HeaderLine - Process a header line /*{{{*/
98 // ---------------------------------------------------------------------
99 /* */
100 bool ServerState::HeaderLine(string Line)
101 {
102 if (Line.empty() == true)
103 return true;
104
105 if (Line.size() > 4 && stringcasecmp(Line.data(), Line.data()+4, "HTTP") == 0)
106 {
107 // Evil servers return no version
108 if (Line[4] == '/')
109 {
110 int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
111 if (elements == 3)
112 {
113 Code[0] = '\0';
114 if (Owner != NULL && Owner->Debug == true)
115 clog << "HTTP server doesn't give Reason-Phrase for " << std::to_string(Result) << std::endl;
116 }
117 else if (elements != 4)
118 return _error->Error(_("The HTTP server sent an invalid reply header"));
119 }
120 else
121 {
122 Major = 0;
123 Minor = 9;
124 if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
125 return _error->Error(_("The HTTP server sent an invalid reply header"));
126 }
127
128 /* Check the HTTP response header to get the default persistence
129 state. */
130 if (Major < 1)
131 Persistent = false;
132 else
133 {
134 if (Major == 1 && Minor == 0)
135 {
136 Persistent = false;
137 }
138 else
139 {
140 Persistent = true;
141 if (PipelineAllowed)
142 Pipeline = true;
143 }
144 }
145
146 return true;
147 }
148
149 // Blah, some servers use "connection:closes", evil.
150 // and some even send empty header fields…
151 string::size_type Pos = Line.find(':');
152 if (Pos == string::npos)
153 return _error->Error(_("Bad header line"));
154 ++Pos;
155
156 // Parse off any trailing spaces between the : and the next word.
157 string::size_type Pos2 = Pos;
158 while (Pos2 < Line.length() && isspace_ascii(Line[Pos2]) != 0)
159 Pos2++;
160
161 string const Tag(Line,0,Pos);
162 string const Val(Line,Pos2);
163
164 if (stringcasecmp(Tag,"Content-Length:") == 0)
165 {
166 if (Encoding == Closes)
167 Encoding = Stream;
168 HaveContent = true;
169
170 unsigned long long * DownloadSizePtr = &DownloadSize;
171 if (Result == 416)
172 DownloadSizePtr = &JunkSize;
173
174 *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10);
175 if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max())
176 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
177 else if (*DownloadSizePtr == 0)
178 HaveContent = false;
179
180 // On partial content (206) the Content-Length less than the real
181 // size, so do not set it here but leave that to the Content-Range
182 // header instead
183 if(Result != 206 && TotalFileSize == 0)
184 TotalFileSize = DownloadSize;
185
186 return true;
187 }
188
189 if (stringcasecmp(Tag,"Content-Type:") == 0)
190 {
191 HaveContent = true;
192 return true;
193 }
194
195 if (stringcasecmp(Tag,"Content-Range:") == 0)
196 {
197 HaveContent = true;
198
199 // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
200 if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
201 ; // we got the expected filesize which is all we wanted
202 else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
203 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
204 if ((unsigned long long)StartPos > TotalFileSize)
205 return _error->Error(_("This HTTP server has broken range support"));
206
207 // figure out what we will download
208 DownloadSize = TotalFileSize - StartPos;
209 return true;
210 }
211
212 if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
213 {
214 HaveContent = true;
215 if (stringcasecmp(Val,"chunked") == 0)
216 Encoding = Chunked;
217 return true;
218 }
219
220 if (stringcasecmp(Tag,"Connection:") == 0)
221 {
222 if (stringcasecmp(Val,"close") == 0)
223 {
224 Persistent = false;
225 Pipeline = false;
226 /* Some servers send error pages (as they are dynamically generated)
227 for simplicity via a connection close instead of e.g. chunked,
228 so assuming an always closing server only if we get a file + close */
229 if (Result >= 200 && Result < 300)
230 PipelineAllowed = false;
231 }
232 else if (stringcasecmp(Val,"keep-alive") == 0)
233 Persistent = true;
234 return true;
235 }
236
237 if (stringcasecmp(Tag,"Last-Modified:") == 0)
238 {
239 if (RFC1123StrToTime(Val.c_str(), Date) == false)
240 return _error->Error(_("Unknown date format"));
241 return true;
242 }
243
244 if (stringcasecmp(Tag,"Location:") == 0)
245 {
246 Location = Val;
247 return true;
248 }
249
250 return true;
251 }
252 /*}}}*/
253 // ServerState::ServerState - Constructor /*{{{*/
254 ServerState::ServerState(URI Srv, ServerMethod *Owner) :
255 DownloadSize(0), ServerName(Srv), TimeOut(120), Owner(Owner)
256 {
257 Reset();
258 }
259 /*}}}*/
260 bool ServerState::AddPartialFileToHashes(FileFd &File) /*{{{*/
261 {
262 File.Truncate(StartPos);
263 return GetHashes()->AddFD(File, StartPos);
264 }
265 /*}}}*/
266
267 // ServerMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
268 // ---------------------------------------------------------------------
269 /* We look at the header data we got back from the server and decide what
270 to do. Returns DealWithHeadersResult (see http.h for details).
271 */
272 ServerMethod::DealWithHeadersResult
273 ServerMethod::DealWithHeaders(FetchResult &Res)
274 {
275 // Not Modified
276 if (Server->Result == 304)
277 {
278 RemoveFile("server", Queue->DestFile);
279 Res.IMSHit = true;
280 Res.LastModified = Queue->LastModified;
281 return IMS_HIT;
282 }
283
284 /* Redirect
285 *
286 * Note that it is only OK for us to treat all redirection the same
287 * because we *always* use GET, not other HTTP methods. There are
288 * three redirection codes for which it is not appropriate that we
289 * redirect. Pass on those codes so the error handling kicks in.
290 */
291 if (AllowRedirect
292 && (Server->Result > 300 && Server->Result < 400)
293 && (Server->Result != 300 // Multiple Choices
294 && Server->Result != 304 // Not Modified
295 && Server->Result != 306)) // (Not part of HTTP/1.1, reserved)
296 {
297 if (Server->Location.empty() == true);
298 else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
299 {
300 URI Uri = Queue->Uri;
301 if (Uri.Host.empty() == false)
302 NextURI = URI::SiteOnly(Uri);
303 else
304 NextURI.clear();
305 NextURI.append(DeQuoteString(Server->Location));
306 return TRY_AGAIN_OR_REDIRECT;
307 }
308 else
309 {
310 NextURI = DeQuoteString(Server->Location);
311 URI tmpURI = NextURI;
312 URI Uri = Queue->Uri;
313 // same protocol redirects are okay
314 if (tmpURI.Access == Uri.Access)
315 return TRY_AGAIN_OR_REDIRECT;
316 // as well as http to https
317 else if (Uri.Access == "http" && tmpURI.Access == "https")
318 return TRY_AGAIN_OR_REDIRECT;
319 }
320 /* else pass through for error message */
321 }
322 // retry after an invalid range response without partial data
323 else if (Server->Result == 416)
324 {
325 struct stat SBuf;
326 if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
327 {
328 bool partialHit = false;
329 if (Queue->ExpectedHashes.usable() == true)
330 {
331 Hashes resultHashes(Queue->ExpectedHashes);
332 FileFd file(Queue->DestFile, FileFd::ReadOnly);
333 Server->TotalFileSize = file.FileSize();
334 Server->Date = file.ModificationTime();
335 resultHashes.AddFD(file);
336 HashStringList const hashList = resultHashes.GetHashStringList();
337 partialHit = (Queue->ExpectedHashes == hashList);
338 }
339 else if ((unsigned long long)SBuf.st_size == Server->TotalFileSize)
340 partialHit = true;
341 if (partialHit == true)
342 {
343 // the file is completely downloaded, but was not moved
344 if (Server->HaveContent == true)
345 {
346 // Send to error page to dev/null
347 FileFd DevNull("/dev/null",FileFd::WriteExists);
348 Server->RunData(&DevNull);
349 }
350 Server->HaveContent = false;
351 Server->StartPos = Server->TotalFileSize;
352 Server->Result = 200;
353 }
354 else if (RemoveFile("server", Queue->DestFile))
355 {
356 NextURI = Queue->Uri;
357 return TRY_AGAIN_OR_REDIRECT;
358 }
359 }
360 }
361
362 /* We have a reply we don't handle. This should indicate a perm server
363 failure */
364 if (Server->Result < 200 || Server->Result >= 300)
365 {
366 std::string err;
367 strprintf(err, "HttpError%u", Server->Result);
368 SetFailReason(err);
369 _error->Error("%u %s", Server->Result, Server->Code);
370 if (Server->HaveContent == true)
371 return ERROR_WITH_CONTENT_PAGE;
372 return ERROR_UNRECOVERABLE;
373 }
374
375 // This is some sort of 2xx 'data follows' reply
376 Res.LastModified = Server->Date;
377 Res.Size = Server->TotalFileSize;
378
379 // Open the file
380 delete File;
381 File = new FileFd(Queue->DestFile,FileFd::WriteAny);
382 if (_error->PendingError() == true)
383 return ERROR_NOT_FROM_SERVER;
384
385 FailFile = Queue->DestFile;
386 FailFile.c_str(); // Make sure we don't do a malloc in the signal handler
387 FailFd = File->Fd();
388 FailTime = Server->Date;
389
390 if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
391 {
392 _error->Errno("read",_("Problem hashing file"));
393 return ERROR_NOT_FROM_SERVER;
394 }
395 if (Server->StartPos > 0)
396 Res.ResumePoint = Server->StartPos;
397
398 SetNonBlock(File->Fd(),true);
399 return FILE_IS_OPEN;
400 }
401 /*}}}*/
402 // ServerMethod::SigTerm - Handle a fatal signal /*{{{*/
403 // ---------------------------------------------------------------------
404 /* This closes and timestamps the open file. This is necessary to get
405 resume behavoir on user abort */
406 void ServerMethod::SigTerm(int)
407 {
408 if (FailFd == -1)
409 _exit(100);
410
411 struct timeval times[2];
412 times[0].tv_sec = FailTime;
413 times[1].tv_sec = FailTime;
414 times[0].tv_usec = times[1].tv_usec = 0;
415 utimes(FailFile.c_str(), times);
416 close(FailFd);
417
418 _exit(100);
419 }
420 /*}}}*/
421 // ServerMethod::Fetch - Fetch an item /*{{{*/
422 // ---------------------------------------------------------------------
423 /* This adds an item to the pipeline. We keep the pipeline at a fixed
424 depth. */
425 bool ServerMethod::Fetch(FetchItem *)
426 {
427 if (Server == 0)
428 return true;
429
430 // Queue the requests
431 int Depth = -1;
432 for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
433 I = I->Next, Depth++)
434 {
435 if (Depth >= 0)
436 {
437 // If pipelining is disabled, we only queue 1 request
438 if (Server->Pipeline == false)
439 break;
440 // if we have no hashes, do at most one such request
441 // as we can't fixup pipeling misbehaviors otherwise
442 else if (I->ExpectedHashes.usable() == false)
443 break;
444 }
445
446 // Make sure we stick with the same server
447 if (Server->Comp(I->Uri) == false)
448 break;
449 if (QueueBack == I)
450 {
451 QueueBack = I->Next;
452 SendReq(I);
453 continue;
454 }
455 }
456
457 return true;
458 }
459 /*}}}*/
460 // ServerMethod::Loop - Main loop /*{{{*/
461 int ServerMethod::Loop()
462 {
463 typedef vector<string> StringVector;
464 typedef vector<string>::iterator StringVectorIterator;
465 map<string, StringVector> Redirected;
466
467 signal(SIGTERM,SigTerm);
468 signal(SIGINT,SigTerm);
469
470 Server = 0;
471
472 int FailCounter = 0;
473 while (1)
474 {
475 // We have no commands, wait for some to arrive
476 if (Queue == 0)
477 {
478 if (WaitFd(STDIN_FILENO) == false)
479 return 0;
480 }
481
482 /* Run messages, we can accept 0 (no message) if we didn't
483 do a WaitFd above.. Otherwise the FD is closed. */
484 int Result = Run(true);
485 if (Result != -1 && (Result != 0 || Queue == 0))
486 {
487 if(FailReason.empty() == false ||
488 _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
489 return 100;
490 else
491 return 0;
492 }
493
494 if (Queue == 0)
495 continue;
496
497 // Connect to the server
498 if (Server == 0 || Server->Comp(Queue->Uri) == false)
499 Server = CreateServerState(Queue->Uri);
500
501 /* If the server has explicitly said this is the last connection
502 then we pre-emptively shut down the pipeline and tear down
503 the connection. This will speed up HTTP/1.0 servers a tad
504 since we don't have to wait for the close sequence to
505 complete */
506 if (Server->Persistent == false)
507 Server->Close();
508
509 // Reset the pipeline
510 if (Server->IsOpen() == false)
511 QueueBack = Queue;
512
513 // Connnect to the host
514 if (Server->Open() == false)
515 {
516 Fail(true);
517 Server = nullptr;
518 continue;
519 }
520
521 // Fill the pipeline.
522 Fetch(0);
523
524 // Fetch the next URL header data from the server.
525 switch (Server->RunHeaders(File, Queue->Uri))
526 {
527 case ServerState::RUN_HEADERS_OK:
528 break;
529
530 // The header data is bad
531 case ServerState::RUN_HEADERS_PARSE_ERROR:
532 {
533 _error->Error(_("Bad header data"));
534 Fail(true);
535 Server->Close();
536 RotateDNS();
537 continue;
538 }
539
540 // The server closed a connection during the header get..
541 default:
542 case ServerState::RUN_HEADERS_IO_ERROR:
543 {
544 FailCounter++;
545 _error->Discard();
546 Server->Close();
547 Server->Pipeline = false;
548 Server->PipelineAllowed = false;
549
550 if (FailCounter >= 2)
551 {
552 Fail(_("Connection failed"),true);
553 FailCounter = 0;
554 }
555
556 RotateDNS();
557 continue;
558 }
559 };
560
561 // Decide what to do.
562 FetchResult Res;
563 Res.Filename = Queue->DestFile;
564 switch (DealWithHeaders(Res))
565 {
566 // Ok, the file is Open
567 case FILE_IS_OPEN:
568 {
569 URIStart(Res);
570
571 // Run the data
572 bool Result = true;
573
574 // ensure we don't fetch too much
575 // we could do "Server->MaximumSize = Queue->MaximumSize" here
576 // but that would break the clever pipeline messup detection
577 // so instead we use the size of the biggest item in the queue
578 Server->MaximumSize = FindMaximumObjectSizeInQueue();
579
580 if (Server->HaveContent)
581 Result = Server->RunData(File);
582
583 /* If the server is sending back sizeless responses then fill in
584 the size now */
585 if (Res.Size == 0)
586 Res.Size = File->Size();
587
588 // Close the file, destroy the FD object and timestamp it
589 FailFd = -1;
590 delete File;
591 File = 0;
592
593 // Timestamp
594 struct timeval times[2];
595 times[0].tv_sec = times[1].tv_sec = Server->Date;
596 times[0].tv_usec = times[1].tv_usec = 0;
597 utimes(Queue->DestFile.c_str(), times);
598
599 // Send status to APT
600 if (Result == true)
601 {
602 Hashes * const resultHashes = Server->GetHashes();
603 HashStringList const hashList = resultHashes->GetHashStringList();
604 if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
605 {
606 // we did not get the expected hash… mhhh:
607 // could it be that server/proxy messed up pipelining?
608 FetchItem * BeforeI = Queue;
609 for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
610 {
611 if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
612 {
613 // yes, he did! Disable pipelining and rewrite queue
614 if (Server->Pipeline == true)
615 {
616 Warning(_("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::Pipeline-Depth");
617 Server->Pipeline = false;
618 Server->PipelineAllowed = false;
619 // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
620 }
621 Rename(Res.Filename, I->DestFile);
622 Res.Filename = I->DestFile;
623 BeforeI->Next = I->Next;
624 I->Next = Queue;
625 Queue = I;
626 break;
627 }
628 BeforeI = I;
629 }
630 }
631 Res.TakeHashes(*resultHashes);
632 URIDone(Res);
633 }
634 else
635 {
636 if (Server->IsOpen() == false)
637 {
638 FailCounter++;
639 _error->Discard();
640 Server->Close();
641
642 if (FailCounter >= 2)
643 {
644 Fail(_("Connection failed"),true);
645 FailCounter = 0;
646 }
647
648 QueueBack = Queue;
649 }
650 else
651 {
652 Server->Close();
653 Fail(true);
654 }
655 }
656 break;
657 }
658
659 // IMS hit
660 case IMS_HIT:
661 {
662 URIDone(Res);
663 break;
664 }
665
666 // Hard server error, not found or something
667 case ERROR_UNRECOVERABLE:
668 {
669 Fail();
670 break;
671 }
672
673 // Hard internal error, kill the connection and fail
674 case ERROR_NOT_FROM_SERVER:
675 {
676 delete File;
677 File = 0;
678
679 Fail();
680 RotateDNS();
681 Server->Close();
682 break;
683 }
684
685 // We need to flush the data, the header is like a 404 w/ error text
686 case ERROR_WITH_CONTENT_PAGE:
687 {
688 Fail();
689
690 // Send to content to dev/null
691 File = new FileFd("/dev/null",FileFd::WriteExists);
692 Server->RunData(File);
693 delete File;
694 File = 0;
695 break;
696 }
697
698 // Try again with a new URL
699 case TRY_AGAIN_OR_REDIRECT:
700 {
701 // Clear rest of response if there is content
702 if (Server->HaveContent)
703 {
704 File = new FileFd("/dev/null",FileFd::WriteExists);
705 Server->RunData(File);
706 delete File;
707 File = 0;
708 }
709
710 /* Detect redirect loops. No more redirects are allowed
711 after the same URI is seen twice in a queue item. */
712 StringVector &R = Redirected[Queue->DestFile];
713 bool StopRedirects = false;
714 if (R.empty() == true)
715 R.push_back(Queue->Uri);
716 else if (R[0] == "STOP" || R.size() > 10)
717 StopRedirects = true;
718 else
719 {
720 for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
721 if (Queue->Uri == *I)
722 {
723 R[0] = "STOP";
724 break;
725 }
726
727 R.push_back(Queue->Uri);
728 }
729
730 if (StopRedirects == false)
731 Redirect(NextURI);
732 else
733 Fail();
734
735 break;
736 }
737
738 default:
739 Fail(_("Internal error"));
740 break;
741 }
742
743 FailCounter = 0;
744 }
745
746 return 0;
747 }
748 /*}}}*/
749 unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/
750 {
751 unsigned long long MaxSizeInQueue = 0;
752 for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
753 MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
754 return MaxSizeInQueue;
755 }
756 /*}}}*/
757 ServerMethod::ServerMethod(char const * const Binary, char const * const Ver,unsigned long const Flags) :/*{{{*/
758 aptMethod(Binary, Ver, Flags), Server(nullptr), File(NULL), PipelineDepth(10),
759 AllowRedirect(false), Debug(false)
760 {
761 }
762 /*}}}*/