]> git.saurik.com Git - apt.git/blob - methods/server.cc
Run "./prepare-release pre-export"
[apt.git] / methods / server.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 /* ######################################################################
4
5 HTTP and HTTPS share a lot of common code and these classes are
6 exactly the dumping ground for this common code
7
8 ##################################################################### */
9 /*}}}*/
10 // Include Files /*{{{*/
11 #include <config.h>
12
13 #include <apt-pkg/acquire-method.h>
14 #include <apt-pkg/configuration.h>
15 #include <apt-pkg/error.h>
16 #include <apt-pkg/fileutl.h>
17 #include <apt-pkg/strutl.h>
18
19 #include <ctype.h>
20 #include <signal.h>
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <sys/stat.h>
24 #include <sys/time.h>
25 #include <time.h>
26 #include <unistd.h>
27 #include <iostream>
28 #include <limits>
29 #include <map>
30 #include <string>
31 #include <vector>
32
33 #include "server.h"
34
35 #include <apti18n.h>
36 /*}}}*/
37 using namespace std;
38
39 string ServerMethod::FailFile;
40 int ServerMethod::FailFd = -1;
41 time_t ServerMethod::FailTime = 0;
42
43 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
44 // ---------------------------------------------------------------------
45 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
46 parse error occurred */
47 ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
48 const std::string &Uri)
49 {
50 State = Header;
51
52 Owner->Status(_("Waiting for headers"));
53
54 Major = 0;
55 Minor = 0;
56 Result = 0;
57 TotalFileSize = 0;
58 JunkSize = 0;
59 StartPos = 0;
60 Encoding = Closes;
61 HaveContent = false;
62 time(&Date);
63
64 do
65 {
66 string Data;
67 if (ReadHeaderLines(Data) == false)
68 continue;
69
70 if (Owner->Debug == true)
71 clog << "Answer for: " << Uri << endl << Data;
72
73 for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
74 {
75 string::const_iterator J = I;
76 for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
77 if (HeaderLine(string(I,J)) == false)
78 return RUN_HEADERS_PARSE_ERROR;
79 I = J;
80 }
81
82 // 100 Continue is a Nop...
83 if (Result == 100)
84 continue;
85
86 // Tidy up the connection persistence state.
87 if (Encoding == Closes && HaveContent == true)
88 Persistent = false;
89
90 return RUN_HEADERS_OK;
91 }
92 while (LoadNextResponse(false, File) == true);
93
94 return RUN_HEADERS_IO_ERROR;
95 }
96 /*}}}*/
97 // ServerState::HeaderLine - Process a header line /*{{{*/
98 // ---------------------------------------------------------------------
99 /* */
100 bool ServerState::HeaderLine(string Line)
101 {
102 if (Line.empty() == true)
103 return true;
104
105 string::size_type Pos = Line.find(' ');
106 if (Pos == string::npos || Pos+1 > Line.length())
107 {
108 // Blah, some servers use "connection:closes", evil.
109 Pos = Line.find(':');
110 if (Pos == string::npos || Pos + 2 > Line.length())
111 return _error->Error(_("Bad header line"));
112 Pos++;
113 }
114
115 // Parse off any trailing spaces between the : and the next word.
116 string::size_type Pos2 = Pos;
117 while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
118 Pos2++;
119
120 string Tag = string(Line,0,Pos);
121 string Val = string(Line,Pos2);
122
123 if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
124 {
125 // Evil servers return no version
126 if (Line[4] == '/')
127 {
128 int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
129 if (elements == 3)
130 {
131 Code[0] = '\0';
132 if (Owner != NULL && Owner->Debug == true)
133 clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
134 }
135 else if (elements != 4)
136 return _error->Error(_("The HTTP server sent an invalid reply header"));
137 }
138 else
139 {
140 Major = 0;
141 Minor = 9;
142 if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
143 return _error->Error(_("The HTTP server sent an invalid reply header"));
144 }
145
146 /* Check the HTTP response header to get the default persistence
147 state. */
148 if (Major < 1)
149 Persistent = false;
150 else
151 {
152 if (Major == 1 && Minor == 0)
153 Persistent = false;
154 else
155 Persistent = true;
156 }
157
158 return true;
159 }
160
161 if (stringcasecmp(Tag,"Content-Length:") == 0)
162 {
163 if (Encoding == Closes)
164 Encoding = Stream;
165 HaveContent = true;
166
167 unsigned long long * DownloadSizePtr = &DownloadSize;
168 if (Result == 416)
169 DownloadSizePtr = &JunkSize;
170
171 *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10);
172 if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max())
173 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
174 else if (*DownloadSizePtr == 0)
175 HaveContent = false;
176
177 // On partial content (206) the Content-Length less than the real
178 // size, so do not set it here but leave that to the Content-Range
179 // header instead
180 if(Result != 206 && TotalFileSize == 0)
181 TotalFileSize = DownloadSize;
182
183 return true;
184 }
185
186 if (stringcasecmp(Tag,"Content-Type:") == 0)
187 {
188 HaveContent = true;
189 return true;
190 }
191
192 if (stringcasecmp(Tag,"Content-Range:") == 0)
193 {
194 HaveContent = true;
195
196 // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
197 if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
198 ; // we got the expected filesize which is all we wanted
199 else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
200 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
201 if ((unsigned long long)StartPos > TotalFileSize)
202 return _error->Error(_("This HTTP server has broken range support"));
203
204 // figure out what we will download
205 DownloadSize = TotalFileSize - StartPos;
206 return true;
207 }
208
209 if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
210 {
211 HaveContent = true;
212 if (stringcasecmp(Val,"chunked") == 0)
213 Encoding = Chunked;
214 return true;
215 }
216
217 if (stringcasecmp(Tag,"Connection:") == 0)
218 {
219 if (stringcasecmp(Val,"close") == 0)
220 Persistent = false;
221 if (stringcasecmp(Val,"keep-alive") == 0)
222 Persistent = true;
223 return true;
224 }
225
226 if (stringcasecmp(Tag,"Last-Modified:") == 0)
227 {
228 if (RFC1123StrToTime(Val.c_str(), Date) == false)
229 return _error->Error(_("Unknown date format"));
230 return true;
231 }
232
233 if (stringcasecmp(Tag,"Location:") == 0)
234 {
235 Location = Val;
236 return true;
237 }
238
239 return true;
240 }
241 /*}}}*/
242 // ServerState::ServerState - Constructor /*{{{*/
243 ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOut(120), Owner(Owner)
244 {
245 Reset();
246 }
247 /*}}}*/
248 bool ServerState::AddPartialFileToHashes(FileFd &File) /*{{{*/
249 {
250 File.Truncate(StartPos);
251 return GetHashes()->AddFD(File, StartPos);
252 }
253 /*}}}*/
254
255 bool ServerMethod::Configuration(string Message) /*{{{*/
256 {
257 if (pkgAcqMethod::Configuration(Message) == false)
258 return false;
259
260 DropPrivsOrDie();
261
262 return true;
263 }
264 /*}}}*/
265
266 // ServerMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
267 // ---------------------------------------------------------------------
268 /* We look at the header data we got back from the server and decide what
269 to do. Returns DealWithHeadersResult (see http.h for details).
270 */
271 ServerMethod::DealWithHeadersResult
272 ServerMethod::DealWithHeaders(FetchResult &Res)
273 {
274 // Not Modified
275 if (Server->Result == 304)
276 {
277 unlink(Queue->DestFile.c_str());
278 Res.IMSHit = true;
279 Res.LastModified = Queue->LastModified;
280 return IMS_HIT;
281 }
282
283 /* Redirect
284 *
285 * Note that it is only OK for us to treat all redirection the same
286 * because we *always* use GET, not other HTTP methods. There are
287 * three redirection codes for which it is not appropriate that we
288 * redirect. Pass on those codes so the error handling kicks in.
289 */
290 if (AllowRedirect
291 && (Server->Result > 300 && Server->Result < 400)
292 && (Server->Result != 300 // Multiple Choices
293 && Server->Result != 304 // Not Modified
294 && Server->Result != 306)) // (Not part of HTTP/1.1, reserved)
295 {
296 if (Server->Location.empty() == true);
297 else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
298 {
299 URI Uri = Queue->Uri;
300 if (Uri.Host.empty() == false)
301 NextURI = URI::SiteOnly(Uri);
302 else
303 NextURI.clear();
304 NextURI.append(DeQuoteString(Server->Location));
305 return TRY_AGAIN_OR_REDIRECT;
306 }
307 else
308 {
309 NextURI = DeQuoteString(Server->Location);
310 URI tmpURI = NextURI;
311 URI Uri = Queue->Uri;
312 // same protocol redirects are okay
313 if (tmpURI.Access == Uri.Access)
314 return TRY_AGAIN_OR_REDIRECT;
315 // as well as http to https
316 else if (Uri.Access == "http" && tmpURI.Access == "https")
317 return TRY_AGAIN_OR_REDIRECT;
318 }
319 /* else pass through for error message */
320 }
321 // retry after an invalid range response without partial data
322 else if (Server->Result == 416)
323 {
324 struct stat SBuf;
325 if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
326 {
327 bool partialHit = false;
328 if (Queue->ExpectedHashes.usable() == true)
329 {
330 Hashes resultHashes(Queue->ExpectedHashes);
331 FileFd file(Queue->DestFile, FileFd::ReadOnly);
332 Server->TotalFileSize = file.FileSize();
333 Server->Date = file.ModificationTime();
334 resultHashes.AddFD(file);
335 HashStringList const hashList = resultHashes.GetHashStringList();
336 partialHit = (Queue->ExpectedHashes == hashList);
337 }
338 else if ((unsigned long long)SBuf.st_size == Server->TotalFileSize)
339 partialHit = true;
340 if (partialHit == true)
341 {
342 // the file is completely downloaded, but was not moved
343 if (Server->HaveContent == true)
344 {
345 // Send to error page to dev/null
346 FileFd DevNull("/dev/null",FileFd::WriteExists);
347 Server->RunData(&DevNull);
348 }
349 Server->HaveContent = false;
350 Server->StartPos = Server->TotalFileSize;
351 Server->Result = 200;
352 }
353 else if (unlink(Queue->DestFile.c_str()) == 0)
354 {
355 NextURI = Queue->Uri;
356 return TRY_AGAIN_OR_REDIRECT;
357 }
358 }
359 }
360
361 /* We have a reply we don't handle. This should indicate a perm server
362 failure */
363 if (Server->Result < 200 || Server->Result >= 300)
364 {
365 std::string err;
366 strprintf(err, "HttpError%u", Server->Result);
367 SetFailReason(err);
368 _error->Error("%u %s", Server->Result, Server->Code);
369 if (Server->HaveContent == true)
370 return ERROR_WITH_CONTENT_PAGE;
371 return ERROR_UNRECOVERABLE;
372 }
373
374 // This is some sort of 2xx 'data follows' reply
375 Res.LastModified = Server->Date;
376 Res.Size = Server->TotalFileSize;
377
378 // Open the file
379 delete File;
380 File = new FileFd(Queue->DestFile,FileFd::WriteAny);
381 if (_error->PendingError() == true)
382 return ERROR_NOT_FROM_SERVER;
383
384 FailFile = Queue->DestFile;
385 FailFile.c_str(); // Make sure we don't do a malloc in the signal handler
386 FailFd = File->Fd();
387 FailTime = Server->Date;
388
389 if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
390 {
391 _error->Errno("read",_("Problem hashing file"));
392 return ERROR_NOT_FROM_SERVER;
393 }
394 if (Server->StartPos > 0)
395 Res.ResumePoint = Server->StartPos;
396
397 SetNonBlock(File->Fd(),true);
398 return FILE_IS_OPEN;
399 }
400 /*}}}*/
401 // ServerMethod::SigTerm - Handle a fatal signal /*{{{*/
402 // ---------------------------------------------------------------------
403 /* This closes and timestamps the open file. This is necessary to get
404 resume behavoir on user abort */
405 void ServerMethod::SigTerm(int)
406 {
407 if (FailFd == -1)
408 _exit(100);
409
410 struct timeval times[2];
411 times[0].tv_sec = FailTime;
412 times[1].tv_sec = FailTime;
413 times[0].tv_usec = times[1].tv_usec = 0;
414 utimes(FailFile.c_str(), times);
415 close(FailFd);
416
417 _exit(100);
418 }
419 /*}}}*/
420 // ServerMethod::Fetch - Fetch an item /*{{{*/
421 // ---------------------------------------------------------------------
422 /* This adds an item to the pipeline. We keep the pipeline at a fixed
423 depth. */
424 bool ServerMethod::Fetch(FetchItem *)
425 {
426 if (Server == 0)
427 return true;
428
429 // Queue the requests
430 int Depth = -1;
431 for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
432 I = I->Next, Depth++)
433 {
434 if (Depth >= 0)
435 {
436 // If pipelining is disabled, we only queue 1 request
437 if (Server->Pipeline == false)
438 break;
439 // if we have no hashes, do at most one such request
440 // as we can't fixup pipeling misbehaviors otherwise
441 else if (I->ExpectedHashes.usable() == false)
442 break;
443 }
444
445 // Make sure we stick with the same server
446 if (Server->Comp(I->Uri) == false)
447 break;
448 if (QueueBack == I)
449 {
450 QueueBack = I->Next;
451 SendReq(I);
452 continue;
453 }
454 }
455
456 return true;
457 }
458 /*}}}*/
459 // ServerMethod::Loop - Main loop /*{{{*/
460 int ServerMethod::Loop()
461 {
462 typedef vector<string> StringVector;
463 typedef vector<string>::iterator StringVectorIterator;
464 map<string, StringVector> Redirected;
465
466 signal(SIGTERM,SigTerm);
467 signal(SIGINT,SigTerm);
468
469 Server = 0;
470
471 int FailCounter = 0;
472 while (1)
473 {
474 // We have no commands, wait for some to arrive
475 if (Queue == 0)
476 {
477 if (WaitFd(STDIN_FILENO) == false)
478 return 0;
479 }
480
481 /* Run messages, we can accept 0 (no message) if we didn't
482 do a WaitFd above.. Otherwise the FD is closed. */
483 int Result = Run(true);
484 if (Result != -1 && (Result != 0 || Queue == 0))
485 {
486 if(FailReason.empty() == false ||
487 _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
488 return 100;
489 else
490 return 0;
491 }
492
493 if (Queue == 0)
494 continue;
495
496 // Connect to the server
497 if (Server == 0 || Server->Comp(Queue->Uri) == false)
498 Server = CreateServerState(Queue->Uri);
499
500 /* If the server has explicitly said this is the last connection
501 then we pre-emptively shut down the pipeline and tear down
502 the connection. This will speed up HTTP/1.0 servers a tad
503 since we don't have to wait for the close sequence to
504 complete */
505 if (Server->Persistent == false)
506 Server->Close();
507
508 // Reset the pipeline
509 if (Server->IsOpen() == false)
510 QueueBack = Queue;
511
512 // Connnect to the host
513 if (Server->Open() == false)
514 {
515 Fail(true);
516 Server = nullptr;
517 continue;
518 }
519
520 // Fill the pipeline.
521 Fetch(0);
522
523 // Fetch the next URL header data from the server.
524 switch (Server->RunHeaders(File, Queue->Uri))
525 {
526 case ServerState::RUN_HEADERS_OK:
527 break;
528
529 // The header data is bad
530 case ServerState::RUN_HEADERS_PARSE_ERROR:
531 {
532 _error->Error(_("Bad header data"));
533 Fail(true);
534 RotateDNS();
535 continue;
536 }
537
538 // The server closed a connection during the header get..
539 default:
540 case ServerState::RUN_HEADERS_IO_ERROR:
541 {
542 FailCounter++;
543 _error->Discard();
544 Server->Close();
545 Server->Pipeline = false;
546
547 if (FailCounter >= 2)
548 {
549 Fail(_("Connection failed"),true);
550 FailCounter = 0;
551 }
552
553 RotateDNS();
554 continue;
555 }
556 };
557
558 // Decide what to do.
559 FetchResult Res;
560 Res.Filename = Queue->DestFile;
561 switch (DealWithHeaders(Res))
562 {
563 // Ok, the file is Open
564 case FILE_IS_OPEN:
565 {
566 URIStart(Res);
567
568 // Run the data
569 bool Result = true;
570
571 // ensure we don't fetch too much
572 // we could do "Server->MaximumSize = Queue->MaximumSize" here
573 // but that would break the clever pipeline messup detection
574 // so instead we use the size of the biggest item in the queue
575 Server->MaximumSize = FindMaximumObjectSizeInQueue();
576
577 if (Server->HaveContent)
578 Result = Server->RunData(File);
579
580 /* If the server is sending back sizeless responses then fill in
581 the size now */
582 if (Res.Size == 0)
583 Res.Size = File->Size();
584
585 // Close the file, destroy the FD object and timestamp it
586 FailFd = -1;
587 delete File;
588 File = 0;
589
590 // Timestamp
591 struct timeval times[2];
592 times[0].tv_sec = times[1].tv_sec = Server->Date;
593 times[0].tv_usec = times[1].tv_usec = 0;
594 utimes(Queue->DestFile.c_str(), times);
595
596 // Send status to APT
597 if (Result == true)
598 {
599 Hashes * const resultHashes = Server->GetHashes();
600 HashStringList const hashList = resultHashes->GetHashStringList();
601 if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
602 {
603 // we did not get the expected hash… mhhh:
604 // could it be that server/proxy messed up pipelining?
605 FetchItem * BeforeI = Queue;
606 for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
607 {
608 if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
609 {
610 // yes, he did! Disable pipelining and rewrite queue
611 if (Server->Pipeline == true)
612 {
613 // FIXME: fake a warning message as we have no proper way of communicating here
614 std::string out;
615 strprintf(out, _("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::PipelineDepth");
616 std::cerr << "W: " << out << std::endl;
617 Server->Pipeline = false;
618 // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
619 }
620 Rename(Res.Filename, I->DestFile);
621 Res.Filename = I->DestFile;
622 BeforeI->Next = I->Next;
623 I->Next = Queue;
624 Queue = I;
625 break;
626 }
627 BeforeI = I;
628 }
629 }
630 Res.TakeHashes(*resultHashes);
631 URIDone(Res);
632 }
633 else
634 {
635 if (Server->IsOpen() == false)
636 {
637 FailCounter++;
638 _error->Discard();
639 Server->Close();
640
641 if (FailCounter >= 2)
642 {
643 Fail(_("Connection failed"),true);
644 FailCounter = 0;
645 }
646
647 QueueBack = Queue;
648 }
649 else
650 {
651 Server->Close();
652 Fail(true);
653 }
654 }
655 break;
656 }
657
658 // IMS hit
659 case IMS_HIT:
660 {
661 URIDone(Res);
662 break;
663 }
664
665 // Hard server error, not found or something
666 case ERROR_UNRECOVERABLE:
667 {
668 Fail();
669 break;
670 }
671
672 // Hard internal error, kill the connection and fail
673 case ERROR_NOT_FROM_SERVER:
674 {
675 delete File;
676 File = 0;
677
678 Fail();
679 RotateDNS();
680 Server->Close();
681 break;
682 }
683
684 // We need to flush the data, the header is like a 404 w/ error text
685 case ERROR_WITH_CONTENT_PAGE:
686 {
687 Fail();
688
689 // Send to content to dev/null
690 File = new FileFd("/dev/null",FileFd::WriteExists);
691 Server->RunData(File);
692 delete File;
693 File = 0;
694 break;
695 }
696
697 // Try again with a new URL
698 case TRY_AGAIN_OR_REDIRECT:
699 {
700 // Clear rest of response if there is content
701 if (Server->HaveContent)
702 {
703 File = new FileFd("/dev/null",FileFd::WriteExists);
704 Server->RunData(File);
705 delete File;
706 File = 0;
707 }
708
709 /* Detect redirect loops. No more redirects are allowed
710 after the same URI is seen twice in a queue item. */
711 StringVector &R = Redirected[Queue->DestFile];
712 bool StopRedirects = false;
713 if (R.empty() == true)
714 R.push_back(Queue->Uri);
715 else if (R[0] == "STOP" || R.size() > 10)
716 StopRedirects = true;
717 else
718 {
719 for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
720 if (Queue->Uri == *I)
721 {
722 R[0] = "STOP";
723 break;
724 }
725
726 R.push_back(Queue->Uri);
727 }
728
729 if (StopRedirects == false)
730 Redirect(NextURI);
731 else
732 Fail();
733
734 break;
735 }
736
737 default:
738 Fail(_("Internal error"));
739 break;
740 }
741
742 FailCounter = 0;
743 }
744
745 return 0;
746 }
747 /*}}}*/
748 unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/
749 {
750 unsigned long long MaxSizeInQueue = 0;
751 for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
752 MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
753 return MaxSizeInQueue;
754 }
755 /*}}}*/
756 ServerMethod::ServerMethod(const char *Ver,unsigned long Flags) : /*{{{*/
757 pkgAcqMethod(Ver, Flags), Server(nullptr), File(NULL), PipelineDepth(10),
758 AllowRedirect(false), Debug(false)
759 {
760 }
761 /*}}}*/