]> git.saurik.com Git - apt.git/blob - methods/server.cc
Report non-transient errors as errors, not as warnings
[apt.git] / methods / server.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 /* ######################################################################
4
5 HTTP and HTTPS share a lot of common code and these classes are
6 exactly the dumping ground for this common code
7
8 ##################################################################### */
9 /*}}}*/
10 // Include Files /*{{{*/
11 #include <config.h>
12
13 #include <apt-pkg/acquire-method.h>
14 #include <apt-pkg/configuration.h>
15 #include <apt-pkg/error.h>
16 #include <apt-pkg/fileutl.h>
17 #include <apt-pkg/strutl.h>
18
19 #include <ctype.h>
20 #include <signal.h>
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <sys/stat.h>
24 #include <sys/time.h>
25 #include <time.h>
26 #include <unistd.h>
27 #include <iostream>
28 #include <limits>
29 #include <map>
30 #include <string>
31 #include <vector>
32
33 #include "server.h"
34
35 #include <apti18n.h>
36 /*}}}*/
37 using namespace std;
38
39 string ServerMethod::FailFile;
40 int ServerMethod::FailFd = -1;
41 time_t ServerMethod::FailTime = 0;
42
43 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
44 // ---------------------------------------------------------------------
45 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
46 parse error occurred */
47 ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
48 const std::string &Uri)
49 {
50 State = Header;
51
52 Owner->Status(_("Waiting for headers"));
53
54 Major = 0;
55 Minor = 0;
56 Result = 0;
57 TotalFileSize = 0;
58 JunkSize = 0;
59 StartPos = 0;
60 Encoding = Closes;
61 HaveContent = false;
62 time(&Date);
63
64 do
65 {
66 string Data;
67 if (ReadHeaderLines(Data) == false)
68 continue;
69
70 if (Owner->Debug == true)
71 clog << "Answer for: " << Uri << endl << Data;
72
73 for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
74 {
75 string::const_iterator J = I;
76 for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
77 if (HeaderLine(string(I,J)) == false)
78 return RUN_HEADERS_PARSE_ERROR;
79 I = J;
80 }
81
82 // 100 Continue is a Nop...
83 if (Result == 100)
84 continue;
85
86 // Tidy up the connection persistence state.
87 if (Encoding == Closes && HaveContent == true)
88 Persistent = false;
89
90 return RUN_HEADERS_OK;
91 }
92 while (LoadNextResponse(false, File) == true);
93
94 return RUN_HEADERS_IO_ERROR;
95 }
96 /*}}}*/
97 // ServerState::HeaderLine - Process a header line /*{{{*/
98 // ---------------------------------------------------------------------
99 /* */
100 bool ServerState::HeaderLine(string Line)
101 {
102 if (Line.empty() == true)
103 return true;
104
105 string::size_type Pos = Line.find(' ');
106 if (Pos == string::npos || Pos+1 > Line.length())
107 {
108 // Blah, some servers use "connection:closes", evil.
109 Pos = Line.find(':');
110 if (Pos == string::npos || Pos + 2 > Line.length())
111 return _error->Error(_("Bad header line"));
112 Pos++;
113 }
114
115 // Parse off any trailing spaces between the : and the next word.
116 string::size_type Pos2 = Pos;
117 while (Pos2 < Line.length() && isspace_ascii(Line[Pos2]) != 0)
118 Pos2++;
119
120 string Tag = string(Line,0,Pos);
121 string Val = string(Line,Pos2);
122
123 if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
124 {
125 // Evil servers return no version
126 if (Line[4] == '/')
127 {
128 int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
129 if (elements == 3)
130 {
131 Code[0] = '\0';
132 if (Owner != NULL && Owner->Debug == true)
133 clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
134 }
135 else if (elements != 4)
136 return _error->Error(_("The HTTP server sent an invalid reply header"));
137 }
138 else
139 {
140 Major = 0;
141 Minor = 9;
142 if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
143 return _error->Error(_("The HTTP server sent an invalid reply header"));
144 }
145
146 /* Check the HTTP response header to get the default persistence
147 state. */
148 if (Major < 1)
149 Persistent = false;
150 else
151 {
152 if (Major == 1 && Minor == 0)
153 {
154 Persistent = false;
155 }
156 else
157 {
158 Persistent = true;
159 if (PipelineAllowed)
160 Pipeline = true;
161 }
162 }
163
164 return true;
165 }
166
167 if (stringcasecmp(Tag,"Content-Length:") == 0)
168 {
169 if (Encoding == Closes)
170 Encoding = Stream;
171 HaveContent = true;
172
173 unsigned long long * DownloadSizePtr = &DownloadSize;
174 if (Result == 416)
175 DownloadSizePtr = &JunkSize;
176
177 *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10);
178 if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max())
179 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
180 else if (*DownloadSizePtr == 0)
181 HaveContent = false;
182
183 // On partial content (206) the Content-Length less than the real
184 // size, so do not set it here but leave that to the Content-Range
185 // header instead
186 if(Result != 206 && TotalFileSize == 0)
187 TotalFileSize = DownloadSize;
188
189 return true;
190 }
191
192 if (stringcasecmp(Tag,"Content-Type:") == 0)
193 {
194 HaveContent = true;
195 return true;
196 }
197
198 if (stringcasecmp(Tag,"Content-Range:") == 0)
199 {
200 HaveContent = true;
201
202 // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
203 if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
204 ; // we got the expected filesize which is all we wanted
205 else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
206 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
207 if ((unsigned long long)StartPos > TotalFileSize)
208 return _error->Error(_("This HTTP server has broken range support"));
209
210 // figure out what we will download
211 DownloadSize = TotalFileSize - StartPos;
212 return true;
213 }
214
215 if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
216 {
217 HaveContent = true;
218 if (stringcasecmp(Val,"chunked") == 0)
219 Encoding = Chunked;
220 return true;
221 }
222
223 if (stringcasecmp(Tag,"Connection:") == 0)
224 {
225 if (stringcasecmp(Val,"close") == 0)
226 Persistent = false;
227 if (stringcasecmp(Val,"keep-alive") == 0)
228 Persistent = true;
229 return true;
230 }
231
232 if (stringcasecmp(Tag,"Last-Modified:") == 0)
233 {
234 if (RFC1123StrToTime(Val.c_str(), Date) == false)
235 return _error->Error(_("Unknown date format"));
236 return true;
237 }
238
239 if (stringcasecmp(Tag,"Location:") == 0)
240 {
241 Location = Val;
242 return true;
243 }
244
245 return true;
246 }
247 /*}}}*/
248 // ServerState::ServerState - Constructor /*{{{*/
249 ServerState::ServerState(URI Srv, ServerMethod *Owner) :
250 DownloadSize(0), ServerName(Srv), TimeOut(120), Owner(Owner)
251 {
252 Reset();
253 }
254 /*}}}*/
255 bool ServerState::AddPartialFileToHashes(FileFd &File) /*{{{*/
256 {
257 File.Truncate(StartPos);
258 return GetHashes()->AddFD(File, StartPos);
259 }
260 /*}}}*/
261
262 // ServerMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
263 // ---------------------------------------------------------------------
264 /* We look at the header data we got back from the server and decide what
265 to do. Returns DealWithHeadersResult (see http.h for details).
266 */
267 ServerMethod::DealWithHeadersResult
268 ServerMethod::DealWithHeaders(FetchResult &Res)
269 {
270 // Not Modified
271 if (Server->Result == 304)
272 {
273 RemoveFile("server", Queue->DestFile);
274 Res.IMSHit = true;
275 Res.LastModified = Queue->LastModified;
276 return IMS_HIT;
277 }
278
279 /* Redirect
280 *
281 * Note that it is only OK for us to treat all redirection the same
282 * because we *always* use GET, not other HTTP methods. There are
283 * three redirection codes for which it is not appropriate that we
284 * redirect. Pass on those codes so the error handling kicks in.
285 */
286 if (AllowRedirect
287 && (Server->Result > 300 && Server->Result < 400)
288 && (Server->Result != 300 // Multiple Choices
289 && Server->Result != 304 // Not Modified
290 && Server->Result != 306)) // (Not part of HTTP/1.1, reserved)
291 {
292 if (Server->Location.empty() == true);
293 else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
294 {
295 URI Uri = Queue->Uri;
296 if (Uri.Host.empty() == false)
297 NextURI = URI::SiteOnly(Uri);
298 else
299 NextURI.clear();
300 NextURI.append(DeQuoteString(Server->Location));
301 return TRY_AGAIN_OR_REDIRECT;
302 }
303 else
304 {
305 NextURI = DeQuoteString(Server->Location);
306 URI tmpURI = NextURI;
307 URI Uri = Queue->Uri;
308 // same protocol redirects are okay
309 if (tmpURI.Access == Uri.Access)
310 return TRY_AGAIN_OR_REDIRECT;
311 // as well as http to https
312 else if (Uri.Access == "http" && tmpURI.Access == "https")
313 return TRY_AGAIN_OR_REDIRECT;
314 }
315 /* else pass through for error message */
316 }
317 // retry after an invalid range response without partial data
318 else if (Server->Result == 416)
319 {
320 struct stat SBuf;
321 if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
322 {
323 bool partialHit = false;
324 if (Queue->ExpectedHashes.usable() == true)
325 {
326 Hashes resultHashes(Queue->ExpectedHashes);
327 FileFd file(Queue->DestFile, FileFd::ReadOnly);
328 Server->TotalFileSize = file.FileSize();
329 Server->Date = file.ModificationTime();
330 resultHashes.AddFD(file);
331 HashStringList const hashList = resultHashes.GetHashStringList();
332 partialHit = (Queue->ExpectedHashes == hashList);
333 }
334 else if ((unsigned long long)SBuf.st_size == Server->TotalFileSize)
335 partialHit = true;
336 if (partialHit == true)
337 {
338 // the file is completely downloaded, but was not moved
339 if (Server->HaveContent == true)
340 {
341 // Send to error page to dev/null
342 FileFd DevNull("/dev/null",FileFd::WriteExists);
343 Server->RunData(&DevNull);
344 }
345 Server->HaveContent = false;
346 Server->StartPos = Server->TotalFileSize;
347 Server->Result = 200;
348 }
349 else if (RemoveFile("server", Queue->DestFile))
350 {
351 NextURI = Queue->Uri;
352 return TRY_AGAIN_OR_REDIRECT;
353 }
354 }
355 }
356
357 /* We have a reply we don't handle. This should indicate a perm server
358 failure */
359 if (Server->Result < 200 || Server->Result >= 300)
360 {
361 std::string err;
362 strprintf(err, "HttpError%u", Server->Result);
363 SetFailReason(err);
364 _error->Error("%u %s", Server->Result, Server->Code);
365 if (Server->HaveContent == true)
366 return ERROR_WITH_CONTENT_PAGE;
367 return ERROR_UNRECOVERABLE;
368 }
369
370 // This is some sort of 2xx 'data follows' reply
371 Res.LastModified = Server->Date;
372 Res.Size = Server->TotalFileSize;
373
374 // Open the file
375 delete File;
376 File = new FileFd(Queue->DestFile,FileFd::WriteAny);
377 if (_error->PendingError() == true)
378 return ERROR_NOT_FROM_SERVER;
379
380 FailFile = Queue->DestFile;
381 FailFile.c_str(); // Make sure we don't do a malloc in the signal handler
382 FailFd = File->Fd();
383 FailTime = Server->Date;
384
385 if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
386 {
387 _error->Errno("read",_("Problem hashing file"));
388 return ERROR_NOT_FROM_SERVER;
389 }
390 if (Server->StartPos > 0)
391 Res.ResumePoint = Server->StartPos;
392
393 SetNonBlock(File->Fd(),true);
394 return FILE_IS_OPEN;
395 }
396 /*}}}*/
397 // ServerMethod::SigTerm - Handle a fatal signal /*{{{*/
398 // ---------------------------------------------------------------------
399 /* This closes and timestamps the open file. This is necessary to get
400 resume behavoir on user abort */
401 void ServerMethod::SigTerm(int)
402 {
403 if (FailFd == -1)
404 _exit(100);
405
406 struct timeval times[2];
407 times[0].tv_sec = FailTime;
408 times[1].tv_sec = FailTime;
409 times[0].tv_usec = times[1].tv_usec = 0;
410 utimes(FailFile.c_str(), times);
411 close(FailFd);
412
413 _exit(100);
414 }
415 /*}}}*/
416 // ServerMethod::Fetch - Fetch an item /*{{{*/
417 // ---------------------------------------------------------------------
418 /* This adds an item to the pipeline. We keep the pipeline at a fixed
419 depth. */
420 bool ServerMethod::Fetch(FetchItem *)
421 {
422 if (Server == 0)
423 return true;
424
425 // Queue the requests
426 int Depth = -1;
427 for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
428 I = I->Next, Depth++)
429 {
430 if (Depth >= 0)
431 {
432 // If pipelining is disabled, we only queue 1 request
433 if (Server->Pipeline == false)
434 break;
435 // if we have no hashes, do at most one such request
436 // as we can't fixup pipeling misbehaviors otherwise
437 else if (I->ExpectedHashes.usable() == false)
438 break;
439 }
440
441 // Make sure we stick with the same server
442 if (Server->Comp(I->Uri) == false)
443 break;
444 if (QueueBack == I)
445 {
446 QueueBack = I->Next;
447 SendReq(I);
448 continue;
449 }
450 }
451
452 return true;
453 }
454 /*}}}*/
455 // ServerMethod::Loop - Main loop /*{{{*/
456 int ServerMethod::Loop()
457 {
458 typedef vector<string> StringVector;
459 typedef vector<string>::iterator StringVectorIterator;
460 map<string, StringVector> Redirected;
461
462 signal(SIGTERM,SigTerm);
463 signal(SIGINT,SigTerm);
464
465 Server = 0;
466
467 int FailCounter = 0;
468 while (1)
469 {
470 // We have no commands, wait for some to arrive
471 if (Queue == 0)
472 {
473 if (WaitFd(STDIN_FILENO) == false)
474 return 0;
475 }
476
477 /* Run messages, we can accept 0 (no message) if we didn't
478 do a WaitFd above.. Otherwise the FD is closed. */
479 int Result = Run(true);
480 if (Result != -1 && (Result != 0 || Queue == 0))
481 {
482 if(FailReason.empty() == false ||
483 _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
484 return 100;
485 else
486 return 0;
487 }
488
489 if (Queue == 0)
490 continue;
491
492 // Connect to the server
493 if (Server == 0 || Server->Comp(Queue->Uri) == false)
494 Server = CreateServerState(Queue->Uri);
495
496 /* If the server has explicitly said this is the last connection
497 then we pre-emptively shut down the pipeline and tear down
498 the connection. This will speed up HTTP/1.0 servers a tad
499 since we don't have to wait for the close sequence to
500 complete */
501 if (Server->Persistent == false)
502 Server->Close();
503
504 // Reset the pipeline
505 if (Server->IsOpen() == false)
506 QueueBack = Queue;
507
508 // Connnect to the host
509 if (Server->Open() == false)
510 {
511 Fail(true);
512 Server = nullptr;
513 continue;
514 }
515
516 // Fill the pipeline.
517 Fetch(0);
518
519 // Fetch the next URL header data from the server.
520 switch (Server->RunHeaders(File, Queue->Uri))
521 {
522 case ServerState::RUN_HEADERS_OK:
523 break;
524
525 // The header data is bad
526 case ServerState::RUN_HEADERS_PARSE_ERROR:
527 {
528 _error->Error(_("Bad header data"));
529 Fail(true);
530 RotateDNS();
531 continue;
532 }
533
534 // The server closed a connection during the header get..
535 default:
536 case ServerState::RUN_HEADERS_IO_ERROR:
537 {
538 FailCounter++;
539 _error->Discard();
540 Server->Close();
541 Server->Pipeline = false;
542 Server->PipelineAllowed = false;
543
544 if (FailCounter >= 2)
545 {
546 Fail(_("Connection failed"),true);
547 FailCounter = 0;
548 }
549
550 RotateDNS();
551 continue;
552 }
553 };
554
555 // Decide what to do.
556 FetchResult Res;
557 Res.Filename = Queue->DestFile;
558 switch (DealWithHeaders(Res))
559 {
560 // Ok, the file is Open
561 case FILE_IS_OPEN:
562 {
563 URIStart(Res);
564
565 // Run the data
566 bool Result = true;
567
568 // ensure we don't fetch too much
569 // we could do "Server->MaximumSize = Queue->MaximumSize" here
570 // but that would break the clever pipeline messup detection
571 // so instead we use the size of the biggest item in the queue
572 Server->MaximumSize = FindMaximumObjectSizeInQueue();
573
574 if (Server->HaveContent)
575 Result = Server->RunData(File);
576
577 /* If the server is sending back sizeless responses then fill in
578 the size now */
579 if (Res.Size == 0)
580 Res.Size = File->Size();
581
582 // Close the file, destroy the FD object and timestamp it
583 FailFd = -1;
584 delete File;
585 File = 0;
586
587 // Timestamp
588 struct timeval times[2];
589 times[0].tv_sec = times[1].tv_sec = Server->Date;
590 times[0].tv_usec = times[1].tv_usec = 0;
591 utimes(Queue->DestFile.c_str(), times);
592
593 // Send status to APT
594 if (Result == true)
595 {
596 Hashes * const resultHashes = Server->GetHashes();
597 HashStringList const hashList = resultHashes->GetHashStringList();
598 if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
599 {
600 // we did not get the expected hash… mhhh:
601 // could it be that server/proxy messed up pipelining?
602 FetchItem * BeforeI = Queue;
603 for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
604 {
605 if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
606 {
607 // yes, he did! Disable pipelining and rewrite queue
608 if (Server->Pipeline == true)
609 {
610 // FIXME: fake a warning message as we have no proper way of communicating here
611 std::string out;
612 strprintf(out, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::PipelineDepth");
613 std::cerr << "W: " << out << std::endl;
614 Server->Pipeline = false;
615 Server->PipelineAllowed = false;
616 // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
617 }
618 Rename(Res.Filename, I->DestFile);
619 Res.Filename = I->DestFile;
620 BeforeI->Next = I->Next;
621 I->Next = Queue;
622 Queue = I;
623 break;
624 }
625 BeforeI = I;
626 }
627 }
628 Res.TakeHashes(*resultHashes);
629 URIDone(Res);
630 }
631 else
632 {
633 if (Server->IsOpen() == false)
634 {
635 FailCounter++;
636 _error->Discard();
637 Server->Close();
638
639 if (FailCounter >= 2)
640 {
641 Fail(_("Connection failed"),true);
642 FailCounter = 0;
643 }
644
645 QueueBack = Queue;
646 }
647 else
648 {
649 Server->Close();
650 Fail(true);
651 }
652 }
653 break;
654 }
655
656 // IMS hit
657 case IMS_HIT:
658 {
659 URIDone(Res);
660 break;
661 }
662
663 // Hard server error, not found or something
664 case ERROR_UNRECOVERABLE:
665 {
666 Fail();
667 break;
668 }
669
670 // Hard internal error, kill the connection and fail
671 case ERROR_NOT_FROM_SERVER:
672 {
673 delete File;
674 File = 0;
675
676 Fail();
677 RotateDNS();
678 Server->Close();
679 break;
680 }
681
682 // We need to flush the data, the header is like a 404 w/ error text
683 case ERROR_WITH_CONTENT_PAGE:
684 {
685 Fail();
686
687 // Send to content to dev/null
688 File = new FileFd("/dev/null",FileFd::WriteExists);
689 Server->RunData(File);
690 delete File;
691 File = 0;
692 break;
693 }
694
695 // Try again with a new URL
696 case TRY_AGAIN_OR_REDIRECT:
697 {
698 // Clear rest of response if there is content
699 if (Server->HaveContent)
700 {
701 File = new FileFd("/dev/null",FileFd::WriteExists);
702 Server->RunData(File);
703 delete File;
704 File = 0;
705 }
706
707 /* Detect redirect loops. No more redirects are allowed
708 after the same URI is seen twice in a queue item. */
709 StringVector &R = Redirected[Queue->DestFile];
710 bool StopRedirects = false;
711 if (R.empty() == true)
712 R.push_back(Queue->Uri);
713 else if (R[0] == "STOP" || R.size() > 10)
714 StopRedirects = true;
715 else
716 {
717 for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
718 if (Queue->Uri == *I)
719 {
720 R[0] = "STOP";
721 break;
722 }
723
724 R.push_back(Queue->Uri);
725 }
726
727 if (StopRedirects == false)
728 Redirect(NextURI);
729 else
730 Fail();
731
732 break;
733 }
734
735 default:
736 Fail(_("Internal error"));
737 break;
738 }
739
740 FailCounter = 0;
741 }
742
743 return 0;
744 }
745 /*}}}*/
746 unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/
747 {
748 unsigned long long MaxSizeInQueue = 0;
749 for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
750 MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
751 return MaxSizeInQueue;
752 }
753 /*}}}*/
754 ServerMethod::ServerMethod(char const * const Binary, char const * const Ver,unsigned long const Flags) :/*{{{*/
755 aptMethod(Binary, Ver, Flags), Server(nullptr), File(NULL), PipelineDepth(10),
756 AllowRedirect(false), Debug(false)
757 {
758 }
759 /*}}}*/