]> git.saurik.com Git - apt.git/blob - methods/server.cc
adc242538b970372f61b3a1b45e96eb01c633283
[apt.git] / methods / server.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 /* ######################################################################
4
5 HTTP and HTTPS share a lot of common code and these classes are
6 exactly the dumping ground for this common code
7
8 ##################################################################### */
9 /*}}}*/
10 // Include Files /*{{{*/
11 #include <config.h>
12
13 #include <apt-pkg/acquire-method.h>
14 #include <apt-pkg/configuration.h>
15 #include <apt-pkg/error.h>
16 #include <apt-pkg/fileutl.h>
17 #include <apt-pkg/strutl.h>
18
19 #include <ctype.h>
20 #include <signal.h>
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <sys/stat.h>
24 #include <sys/time.h>
25 #include <time.h>
26 #include <unistd.h>
27 #include <iostream>
28 #include <limits>
29 #include <map>
30 #include <string>
31 #include <vector>
32
33 #include "server.h"
34
35 #include <apti18n.h>
36 /*}}}*/
37 using namespace std;
38
39 string ServerMethod::FailFile;
40 int ServerMethod::FailFd = -1;
41 time_t ServerMethod::FailTime = 0;
42
43 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
44 // ---------------------------------------------------------------------
45 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
46 parse error occurred */
47 ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
48 const std::string &Uri)
49 {
50 State = Header;
51
52 Owner->Status(_("Waiting for headers"));
53
54 Major = 0;
55 Minor = 0;
56 Result = 0;
57 TotalFileSize = 0;
58 JunkSize = 0;
59 StartPos = 0;
60 Encoding = Closes;
61 HaveContent = false;
62 time(&Date);
63
64 do
65 {
66 string Data;
67 if (ReadHeaderLines(Data) == false)
68 continue;
69
70 if (Owner->Debug == true)
71 clog << "Answer for: " << Uri << endl << Data;
72
73 for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
74 {
75 string::const_iterator J = I;
76 for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
77 if (HeaderLine(string(I,J)) == false)
78 return RUN_HEADERS_PARSE_ERROR;
79 I = J;
80 }
81
82 // 100 Continue is a Nop...
83 if (Result == 100)
84 continue;
85
86 // Tidy up the connection persistence state.
87 if (Encoding == Closes && HaveContent == true)
88 Persistent = false;
89
90 return RUN_HEADERS_OK;
91 }
92 while (LoadNextResponse(false, File) == true);
93
94 return RUN_HEADERS_IO_ERROR;
95 }
96 /*}}}*/
97 // ServerState::HeaderLine - Process a header line /*{{{*/
98 // ---------------------------------------------------------------------
99 /* */
100 bool ServerState::HeaderLine(string Line)
101 {
102 if (Line.empty() == true)
103 return true;
104
105 if (Line.size() > 4 && stringcasecmp(Line.data(), Line.data()+4, "HTTP") == 0)
106 {
107 // Evil servers return no version
108 if (Line[4] == '/')
109 {
110 int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
111 if (elements == 3)
112 {
113 Code[0] = '\0';
114 if (Owner != NULL && Owner->Debug == true)
115 clog << "HTTP server doesn't give Reason-Phrase for " << std::to_string(Result) << std::endl;
116 }
117 else if (elements != 4)
118 return _error->Error(_("The HTTP server sent an invalid reply header"));
119 }
120 else
121 {
122 Major = 0;
123 Minor = 9;
124 if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
125 return _error->Error(_("The HTTP server sent an invalid reply header"));
126 }
127
128 /* Check the HTTP response header to get the default persistence
129 state. */
130 if (Major < 1)
131 Persistent = false;
132 else
133 {
134 if (Major == 1 && Minor == 0)
135 {
136 Persistent = false;
137 }
138 else
139 {
140 Persistent = true;
141 if (PipelineAllowed)
142 Pipeline = true;
143 }
144 }
145
146 return true;
147 }
148
149 // Blah, some servers use "connection:closes", evil.
150 // and some even send empty header fields…
151 string::size_type Pos = Line.find(':');
152 if (Pos == string::npos)
153 return _error->Error(_("Bad header line"));
154 ++Pos;
155
156 // Parse off any trailing spaces between the : and the next word.
157 string::size_type Pos2 = Pos;
158 while (Pos2 < Line.length() && isspace_ascii(Line[Pos2]) != 0)
159 Pos2++;
160
161 string const Tag(Line,0,Pos);
162 string const Val(Line,Pos2);
163
164 if (stringcasecmp(Tag,"Content-Length:") == 0)
165 {
166 if (Encoding == Closes)
167 Encoding = Stream;
168 HaveContent = true;
169
170 unsigned long long * DownloadSizePtr = &DownloadSize;
171 if (Result == 416)
172 DownloadSizePtr = &JunkSize;
173
174 *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10);
175 if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max())
176 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
177 else if (*DownloadSizePtr == 0)
178 HaveContent = false;
179
180 // On partial content (206) the Content-Length less than the real
181 // size, so do not set it here but leave that to the Content-Range
182 // header instead
183 if(Result != 206 && TotalFileSize == 0)
184 TotalFileSize = DownloadSize;
185
186 return true;
187 }
188
189 if (stringcasecmp(Tag,"Content-Type:") == 0)
190 {
191 HaveContent = true;
192 return true;
193 }
194
195 if (stringcasecmp(Tag,"Content-Range:") == 0)
196 {
197 HaveContent = true;
198
199 // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
200 if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
201 ; // we got the expected filesize which is all we wanted
202 else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
203 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
204 if ((unsigned long long)StartPos > TotalFileSize)
205 return _error->Error(_("This HTTP server has broken range support"));
206
207 // figure out what we will download
208 DownloadSize = TotalFileSize - StartPos;
209 return true;
210 }
211
212 if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
213 {
214 HaveContent = true;
215 if (stringcasecmp(Val,"chunked") == 0)
216 Encoding = Chunked;
217 return true;
218 }
219
220 if (stringcasecmp(Tag,"Connection:") == 0)
221 {
222 if (stringcasecmp(Val,"close") == 0)
223 Persistent = false;
224 if (stringcasecmp(Val,"keep-alive") == 0)
225 Persistent = true;
226 return true;
227 }
228
229 if (stringcasecmp(Tag,"Last-Modified:") == 0)
230 {
231 if (RFC1123StrToTime(Val.c_str(), Date) == false)
232 return _error->Error(_("Unknown date format"));
233 return true;
234 }
235
236 if (stringcasecmp(Tag,"Location:") == 0)
237 {
238 Location = Val;
239 return true;
240 }
241
242 return true;
243 }
244 /*}}}*/
245 // ServerState::ServerState - Constructor /*{{{*/
246 ServerState::ServerState(URI Srv, ServerMethod *Owner) :
247 DownloadSize(0), ServerName(Srv), TimeOut(120), Owner(Owner)
248 {
249 Reset();
250 }
251 /*}}}*/
252 bool ServerState::AddPartialFileToHashes(FileFd &File) /*{{{*/
253 {
254 File.Truncate(StartPos);
255 return GetHashes()->AddFD(File, StartPos);
256 }
257 /*}}}*/
258
259 // ServerMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
260 // ---------------------------------------------------------------------
261 /* We look at the header data we got back from the server and decide what
262 to do. Returns DealWithHeadersResult (see http.h for details).
263 */
264 ServerMethod::DealWithHeadersResult
265 ServerMethod::DealWithHeaders(FetchResult &Res)
266 {
267 // Not Modified
268 if (Server->Result == 304)
269 {
270 RemoveFile("server", Queue->DestFile);
271 Res.IMSHit = true;
272 Res.LastModified = Queue->LastModified;
273 return IMS_HIT;
274 }
275
276 /* Redirect
277 *
278 * Note that it is only OK for us to treat all redirection the same
279 * because we *always* use GET, not other HTTP methods. There are
280 * three redirection codes for which it is not appropriate that we
281 * redirect. Pass on those codes so the error handling kicks in.
282 */
283 if (AllowRedirect
284 && (Server->Result > 300 && Server->Result < 400)
285 && (Server->Result != 300 // Multiple Choices
286 && Server->Result != 304 // Not Modified
287 && Server->Result != 306)) // (Not part of HTTP/1.1, reserved)
288 {
289 if (Server->Location.empty() == true);
290 else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
291 {
292 URI Uri = Queue->Uri;
293 if (Uri.Host.empty() == false)
294 NextURI = URI::SiteOnly(Uri);
295 else
296 NextURI.clear();
297 NextURI.append(DeQuoteString(Server->Location));
298 return TRY_AGAIN_OR_REDIRECT;
299 }
300 else
301 {
302 NextURI = DeQuoteString(Server->Location);
303 URI tmpURI = NextURI;
304 URI Uri = Queue->Uri;
305 // same protocol redirects are okay
306 if (tmpURI.Access == Uri.Access)
307 return TRY_AGAIN_OR_REDIRECT;
308 // as well as http to https
309 else if (Uri.Access == "http" && tmpURI.Access == "https")
310 return TRY_AGAIN_OR_REDIRECT;
311 }
312 /* else pass through for error message */
313 }
314 // retry after an invalid range response without partial data
315 else if (Server->Result == 416)
316 {
317 struct stat SBuf;
318 if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
319 {
320 bool partialHit = false;
321 if (Queue->ExpectedHashes.usable() == true)
322 {
323 Hashes resultHashes(Queue->ExpectedHashes);
324 FileFd file(Queue->DestFile, FileFd::ReadOnly);
325 Server->TotalFileSize = file.FileSize();
326 Server->Date = file.ModificationTime();
327 resultHashes.AddFD(file);
328 HashStringList const hashList = resultHashes.GetHashStringList();
329 partialHit = (Queue->ExpectedHashes == hashList);
330 }
331 else if ((unsigned long long)SBuf.st_size == Server->TotalFileSize)
332 partialHit = true;
333 if (partialHit == true)
334 {
335 // the file is completely downloaded, but was not moved
336 if (Server->HaveContent == true)
337 {
338 // Send to error page to dev/null
339 FileFd DevNull("/dev/null",FileFd::WriteExists);
340 Server->RunData(&DevNull);
341 }
342 Server->HaveContent = false;
343 Server->StartPos = Server->TotalFileSize;
344 Server->Result = 200;
345 }
346 else if (RemoveFile("server", Queue->DestFile))
347 {
348 NextURI = Queue->Uri;
349 return TRY_AGAIN_OR_REDIRECT;
350 }
351 }
352 }
353
354 /* We have a reply we don't handle. This should indicate a perm server
355 failure */
356 if (Server->Result < 200 || Server->Result >= 300)
357 {
358 std::string err;
359 strprintf(err, "HttpError%u", Server->Result);
360 SetFailReason(err);
361 _error->Error("%u %s", Server->Result, Server->Code);
362 if (Server->HaveContent == true)
363 return ERROR_WITH_CONTENT_PAGE;
364 return ERROR_UNRECOVERABLE;
365 }
366
367 // This is some sort of 2xx 'data follows' reply
368 Res.LastModified = Server->Date;
369 Res.Size = Server->TotalFileSize;
370
371 // Open the file
372 delete File;
373 File = new FileFd(Queue->DestFile,FileFd::WriteAny);
374 if (_error->PendingError() == true)
375 return ERROR_NOT_FROM_SERVER;
376
377 FailFile = Queue->DestFile;
378 FailFile.c_str(); // Make sure we don't do a malloc in the signal handler
379 FailFd = File->Fd();
380 FailTime = Server->Date;
381
382 if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
383 {
384 _error->Errno("read",_("Problem hashing file"));
385 return ERROR_NOT_FROM_SERVER;
386 }
387 if (Server->StartPos > 0)
388 Res.ResumePoint = Server->StartPos;
389
390 SetNonBlock(File->Fd(),true);
391 return FILE_IS_OPEN;
392 }
393 /*}}}*/
394 // ServerMethod::SigTerm - Handle a fatal signal /*{{{*/
395 // ---------------------------------------------------------------------
396 /* This closes and timestamps the open file. This is necessary to get
397 resume behavoir on user abort */
398 void ServerMethod::SigTerm(int)
399 {
400 if (FailFd == -1)
401 _exit(100);
402
403 struct timeval times[2];
404 times[0].tv_sec = FailTime;
405 times[1].tv_sec = FailTime;
406 times[0].tv_usec = times[1].tv_usec = 0;
407 utimes(FailFile.c_str(), times);
408 close(FailFd);
409
410 _exit(100);
411 }
412 /*}}}*/
413 // ServerMethod::Fetch - Fetch an item /*{{{*/
414 // ---------------------------------------------------------------------
415 /* This adds an item to the pipeline. We keep the pipeline at a fixed
416 depth. */
417 bool ServerMethod::Fetch(FetchItem *)
418 {
419 if (Server == 0)
420 return true;
421
422 // Queue the requests
423 int Depth = -1;
424 for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
425 I = I->Next, Depth++)
426 {
427 if (Depth >= 0)
428 {
429 // If pipelining is disabled, we only queue 1 request
430 if (Server->Pipeline == false)
431 break;
432 // if we have no hashes, do at most one such request
433 // as we can't fixup pipeling misbehaviors otherwise
434 else if (I->ExpectedHashes.usable() == false)
435 break;
436 }
437
438 // Make sure we stick with the same server
439 if (Server->Comp(I->Uri) == false)
440 break;
441 if (QueueBack == I)
442 {
443 QueueBack = I->Next;
444 SendReq(I);
445 continue;
446 }
447 }
448
449 return true;
450 }
451 /*}}}*/
452 // ServerMethod::Loop - Main loop /*{{{*/
453 int ServerMethod::Loop()
454 {
455 typedef vector<string> StringVector;
456 typedef vector<string>::iterator StringVectorIterator;
457 map<string, StringVector> Redirected;
458
459 signal(SIGTERM,SigTerm);
460 signal(SIGINT,SigTerm);
461
462 Server = 0;
463
464 int FailCounter = 0;
465 while (1)
466 {
467 // We have no commands, wait for some to arrive
468 if (Queue == 0)
469 {
470 if (WaitFd(STDIN_FILENO) == false)
471 return 0;
472 }
473
474 /* Run messages, we can accept 0 (no message) if we didn't
475 do a WaitFd above.. Otherwise the FD is closed. */
476 int Result = Run(true);
477 if (Result != -1 && (Result != 0 || Queue == 0))
478 {
479 if(FailReason.empty() == false ||
480 _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
481 return 100;
482 else
483 return 0;
484 }
485
486 if (Queue == 0)
487 continue;
488
489 // Connect to the server
490 if (Server == 0 || Server->Comp(Queue->Uri) == false)
491 Server = CreateServerState(Queue->Uri);
492
493 /* If the server has explicitly said this is the last connection
494 then we pre-emptively shut down the pipeline and tear down
495 the connection. This will speed up HTTP/1.0 servers a tad
496 since we don't have to wait for the close sequence to
497 complete */
498 if (Server->Persistent == false)
499 Server->Close();
500
501 // Reset the pipeline
502 if (Server->IsOpen() == false)
503 QueueBack = Queue;
504
505 // Connnect to the host
506 if (Server->Open() == false)
507 {
508 Fail(true);
509 Server = nullptr;
510 continue;
511 }
512
513 // Fill the pipeline.
514 Fetch(0);
515
516 // Fetch the next URL header data from the server.
517 switch (Server->RunHeaders(File, Queue->Uri))
518 {
519 case ServerState::RUN_HEADERS_OK:
520 break;
521
522 // The header data is bad
523 case ServerState::RUN_HEADERS_PARSE_ERROR:
524 {
525 _error->Error(_("Bad header data"));
526 Fail(true);
527 Server->Close();
528 RotateDNS();
529 continue;
530 }
531
532 // The server closed a connection during the header get..
533 default:
534 case ServerState::RUN_HEADERS_IO_ERROR:
535 {
536 FailCounter++;
537 _error->Discard();
538 Server->Close();
539 Server->Pipeline = false;
540 Server->PipelineAllowed = false;
541
542 if (FailCounter >= 2)
543 {
544 Fail(_("Connection failed"),true);
545 FailCounter = 0;
546 }
547
548 RotateDNS();
549 continue;
550 }
551 };
552
553 // Decide what to do.
554 FetchResult Res;
555 Res.Filename = Queue->DestFile;
556 switch (DealWithHeaders(Res))
557 {
558 // Ok, the file is Open
559 case FILE_IS_OPEN:
560 {
561 URIStart(Res);
562
563 // Run the data
564 bool Result = true;
565
566 // ensure we don't fetch too much
567 // we could do "Server->MaximumSize = Queue->MaximumSize" here
568 // but that would break the clever pipeline messup detection
569 // so instead we use the size of the biggest item in the queue
570 Server->MaximumSize = FindMaximumObjectSizeInQueue();
571
572 if (Server->HaveContent)
573 Result = Server->RunData(File);
574
575 /* If the server is sending back sizeless responses then fill in
576 the size now */
577 if (Res.Size == 0)
578 Res.Size = File->Size();
579
580 // Close the file, destroy the FD object and timestamp it
581 FailFd = -1;
582 delete File;
583 File = 0;
584
585 // Timestamp
586 struct timeval times[2];
587 times[0].tv_sec = times[1].tv_sec = Server->Date;
588 times[0].tv_usec = times[1].tv_usec = 0;
589 utimes(Queue->DestFile.c_str(), times);
590
591 // Send status to APT
592 if (Result == true)
593 {
594 Hashes * const resultHashes = Server->GetHashes();
595 HashStringList const hashList = resultHashes->GetHashStringList();
596 if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
597 {
598 // we did not get the expected hash… mhhh:
599 // could it be that server/proxy messed up pipelining?
600 FetchItem * BeforeI = Queue;
601 for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
602 {
603 if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
604 {
605 // yes, he did! Disable pipelining and rewrite queue
606 if (Server->Pipeline == true)
607 {
608 Warning(_("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::Pipeline-Depth");
609 Server->Pipeline = false;
610 Server->PipelineAllowed = false;
611 // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
612 }
613 Rename(Res.Filename, I->DestFile);
614 Res.Filename = I->DestFile;
615 BeforeI->Next = I->Next;
616 I->Next = Queue;
617 Queue = I;
618 break;
619 }
620 BeforeI = I;
621 }
622 }
623 Res.TakeHashes(*resultHashes);
624 URIDone(Res);
625 }
626 else
627 {
628 if (Server->IsOpen() == false)
629 {
630 FailCounter++;
631 _error->Discard();
632 Server->Close();
633
634 if (FailCounter >= 2)
635 {
636 Fail(_("Connection failed"),true);
637 FailCounter = 0;
638 }
639
640 QueueBack = Queue;
641 }
642 else
643 {
644 Server->Close();
645 Fail(true);
646 }
647 }
648 break;
649 }
650
651 // IMS hit
652 case IMS_HIT:
653 {
654 URIDone(Res);
655 break;
656 }
657
658 // Hard server error, not found or something
659 case ERROR_UNRECOVERABLE:
660 {
661 Fail();
662 break;
663 }
664
665 // Hard internal error, kill the connection and fail
666 case ERROR_NOT_FROM_SERVER:
667 {
668 delete File;
669 File = 0;
670
671 Fail();
672 RotateDNS();
673 Server->Close();
674 break;
675 }
676
677 // We need to flush the data, the header is like a 404 w/ error text
678 case ERROR_WITH_CONTENT_PAGE:
679 {
680 Fail();
681
682 // Send to content to dev/null
683 File = new FileFd("/dev/null",FileFd::WriteExists);
684 Server->RunData(File);
685 delete File;
686 File = 0;
687 break;
688 }
689
690 // Try again with a new URL
691 case TRY_AGAIN_OR_REDIRECT:
692 {
693 // Clear rest of response if there is content
694 if (Server->HaveContent)
695 {
696 File = new FileFd("/dev/null",FileFd::WriteExists);
697 Server->RunData(File);
698 delete File;
699 File = 0;
700 }
701
702 /* Detect redirect loops. No more redirects are allowed
703 after the same URI is seen twice in a queue item. */
704 StringVector &R = Redirected[Queue->DestFile];
705 bool StopRedirects = false;
706 if (R.empty() == true)
707 R.push_back(Queue->Uri);
708 else if (R[0] == "STOP" || R.size() > 10)
709 StopRedirects = true;
710 else
711 {
712 for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
713 if (Queue->Uri == *I)
714 {
715 R[0] = "STOP";
716 break;
717 }
718
719 R.push_back(Queue->Uri);
720 }
721
722 if (StopRedirects == false)
723 Redirect(NextURI);
724 else
725 Fail();
726
727 break;
728 }
729
730 default:
731 Fail(_("Internal error"));
732 break;
733 }
734
735 FailCounter = 0;
736 }
737
738 return 0;
739 }
740 /*}}}*/
741 unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/
742 {
743 unsigned long long MaxSizeInQueue = 0;
744 for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
745 MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
746 return MaxSizeInQueue;
747 }
748 /*}}}*/
749 ServerMethod::ServerMethod(char const * const Binary, char const * const Ver,unsigned long const Flags) :/*{{{*/
750 aptMethod(Binary, Ver, Flags), Server(nullptr), File(NULL), PipelineDepth(10),
751 AllowRedirect(false), Debug(false)
752 {
753 }
754 /*}}}*/