]> git.saurik.com Git - apt.git/blob - methods/server.cc
use the same redirection handling for http and https
[apt.git] / methods / server.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 /* ######################################################################
4
5 HTTP and HTTPS share a lot of common code and these classes are
6 exactly the dumping ground for this common code
7
8 ##################################################################### */
9 /*}}}*/
10 // Include Files /*{{{*/
11 #include <config.h>
12
13 #include <apt-pkg/configuration.h>
14 #include <apt-pkg/error.h>
15 #include <apt-pkg/fileutl.h>
16 #include <apt-pkg/strutl.h>
17
18 #include <ctype.h>
19 #include <signal.h>
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <sys/stat.h>
23 #include <sys/time.h>
24 #include <time.h>
25 #include <unistd.h>
26 #include <iostream>
27 #include <limits>
28 #include <map>
29 #include <string>
30 #include <vector>
31
32 #include "server.h"
33
34 #include <apti18n.h>
35 /*}}}*/
36 using namespace std;
37
38 string ServerMethod::FailFile;
39 int ServerMethod::FailFd = -1;
40 time_t ServerMethod::FailTime = 0;
41
42 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
43 // ---------------------------------------------------------------------
44 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
45 parse error occurred */
46 ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File,
47 const std::string &Uri)
48 {
49 State = Header;
50
51 Owner->Status(_("Waiting for headers"));
52
53 Major = 0;
54 Minor = 0;
55 Result = 0;
56 TotalFileSize = 0;
57 JunkSize = 0;
58 StartPos = 0;
59 Encoding = Closes;
60 HaveContent = false;
61 time(&Date);
62
63 do
64 {
65 string Data;
66 if (ReadHeaderLines(Data) == false)
67 continue;
68
69 if (Owner->Debug == true)
70 clog << "Answer for: " << Uri << endl << Data;
71
72 for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
73 {
74 string::const_iterator J = I;
75 for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
76 if (HeaderLine(string(I,J)) == false)
77 return RUN_HEADERS_PARSE_ERROR;
78 I = J;
79 }
80
81 // 100 Continue is a Nop...
82 if (Result == 100)
83 continue;
84
85 // Tidy up the connection persistence state.
86 if (Encoding == Closes && HaveContent == true)
87 Persistent = false;
88
89 return RUN_HEADERS_OK;
90 }
91 while (LoadNextResponse(false, File) == true);
92
93 return RUN_HEADERS_IO_ERROR;
94 }
95 /*}}}*/
96 // ServerState::HeaderLine - Process a header line /*{{{*/
97 // ---------------------------------------------------------------------
98 /* */
99 bool ServerState::HeaderLine(string Line)
100 {
101 if (Line.empty() == true)
102 return true;
103
104 string::size_type Pos = Line.find(' ');
105 if (Pos == string::npos || Pos+1 > Line.length())
106 {
107 // Blah, some servers use "connection:closes", evil.
108 Pos = Line.find(':');
109 if (Pos == string::npos || Pos + 2 > Line.length())
110 return _error->Error(_("Bad header line"));
111 Pos++;
112 }
113
114 // Parse off any trailing spaces between the : and the next word.
115 string::size_type Pos2 = Pos;
116 while (Pos2 < Line.length() && isspace_ascii(Line[Pos2]) != 0)
117 Pos2++;
118
119 string Tag = string(Line,0,Pos);
120 string Val = string(Line,Pos2);
121
122 if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
123 {
124 // Evil servers return no version
125 if (Line[4] == '/')
126 {
127 int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
128 if (elements == 3)
129 {
130 Code[0] = '\0';
131 if (Owner != NULL && Owner->Debug == true)
132 clog << "HTTP server doesn't give Reason-Phrase for " << std::to_string(Result) << std::endl;
133 }
134 else if (elements != 4)
135 return _error->Error(_("The HTTP server sent an invalid reply header"));
136 }
137 else
138 {
139 Major = 0;
140 Minor = 9;
141 if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
142 return _error->Error(_("The HTTP server sent an invalid reply header"));
143 }
144
145 /* Check the HTTP response header to get the default persistence
146 state. */
147 if (Major < 1)
148 Persistent = false;
149 else
150 {
151 if (Major == 1 && Minor == 0)
152 {
153 Persistent = false;
154 }
155 else
156 {
157 Persistent = true;
158 if (PipelineAllowed)
159 Pipeline = true;
160 }
161 }
162
163 return true;
164 }
165
166 if (stringcasecmp(Tag,"Content-Length:") == 0)
167 {
168 if (Encoding == Closes)
169 Encoding = Stream;
170 HaveContent = true;
171
172 unsigned long long * DownloadSizePtr = &DownloadSize;
173 if (Result == 416 || (Result >= 300 && Result < 400))
174 DownloadSizePtr = &JunkSize;
175
176 *DownloadSizePtr = strtoull(Val.c_str(), NULL, 10);
177 if (*DownloadSizePtr >= std::numeric_limits<unsigned long long>::max())
178 return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
179 else if (*DownloadSizePtr == 0)
180 HaveContent = false;
181
182 // On partial content (206) the Content-Length less than the real
183 // size, so do not set it here but leave that to the Content-Range
184 // header instead
185 if(Result != 206 && TotalFileSize == 0)
186 TotalFileSize = DownloadSize;
187
188 return true;
189 }
190
191 if (stringcasecmp(Tag,"Content-Type:") == 0)
192 {
193 HaveContent = true;
194 return true;
195 }
196
197 if (stringcasecmp(Tag,"Content-Range:") == 0)
198 {
199 HaveContent = true;
200
201 // §14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
202 if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&TotalFileSize) == 1)
203 ; // we got the expected filesize which is all we wanted
204 else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&TotalFileSize) != 2)
205 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
206 if ((unsigned long long)StartPos > TotalFileSize)
207 return _error->Error(_("This HTTP server has broken range support"));
208
209 // figure out what we will download
210 DownloadSize = TotalFileSize - StartPos;
211 return true;
212 }
213
214 if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
215 {
216 HaveContent = true;
217 if (stringcasecmp(Val,"chunked") == 0)
218 Encoding = Chunked;
219 return true;
220 }
221
222 if (stringcasecmp(Tag,"Connection:") == 0)
223 {
224 if (stringcasecmp(Val,"close") == 0)
225 Persistent = false;
226 if (stringcasecmp(Val,"keep-alive") == 0)
227 Persistent = true;
228 return true;
229 }
230
231 if (stringcasecmp(Tag,"Last-Modified:") == 0)
232 {
233 if (RFC1123StrToTime(Val.c_str(), Date) == false)
234 return _error->Error(_("Unknown date format"));
235 return true;
236 }
237
238 if (stringcasecmp(Tag,"Location:") == 0)
239 {
240 Location = Val;
241 return true;
242 }
243
244 return true;
245 }
246 /*}}}*/
247 // ServerState::ServerState - Constructor /*{{{*/
248 ServerState::ServerState(URI Srv, ServerMethod *Owner) :
249 DownloadSize(0), ServerName(Srv), TimeOut(120), Owner(Owner)
250 {
251 Reset();
252 }
253 /*}}}*/
254 bool ServerState::AddPartialFileToHashes(FileFd &File) /*{{{*/
255 {
256 File.Truncate(StartPos);
257 return GetHashes()->AddFD(File, StartPos);
258 }
259 /*}}}*/
260
261 // ServerMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
262 // ---------------------------------------------------------------------
263 /* We look at the header data we got back from the server and decide what
264 to do. Returns DealWithHeadersResult (see http.h for details).
265 */
266 ServerMethod::DealWithHeadersResult
267 ServerMethod::DealWithHeaders(FetchResult &Res)
268 {
269 // Not Modified
270 if (Server->Result == 304)
271 {
272 RemoveFile("server", Queue->DestFile);
273 Res.IMSHit = true;
274 Res.LastModified = Queue->LastModified;
275 Res.Size = 0;
276 return IMS_HIT;
277 }
278
279 /* Redirect
280 *
281 * Note that it is only OK for us to treat all redirection the same
282 * because we *always* use GET, not other HTTP methods. There are
283 * three redirection codes for which it is not appropriate that we
284 * redirect. Pass on those codes so the error handling kicks in.
285 */
286 if (AllowRedirect
287 && (Server->Result > 300 && Server->Result < 400)
288 && (Server->Result != 300 // Multiple Choices
289 && Server->Result != 304 // Not Modified
290 && Server->Result != 306)) // (Not part of HTTP/1.1, reserved)
291 {
292 if (Server->Location.empty() == true)
293 ;
294 else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
295 {
296 URI Uri = Queue->Uri;
297 if (Uri.Host.empty() == false)
298 NextURI = URI::SiteOnly(Uri);
299 else
300 NextURI.clear();
301 NextURI.append(DeQuoteString(Server->Location));
302 if (Queue->Uri == NextURI)
303 {
304 SetFailReason("RedirectionLoop");
305 _error->Error("Redirection loop encountered");
306 if (Server->HaveContent == true)
307 return ERROR_WITH_CONTENT_PAGE;
308 return ERROR_UNRECOVERABLE;
309 }
310 return TRY_AGAIN_OR_REDIRECT;
311 }
312 else
313 {
314 NextURI = DeQuoteString(Server->Location);
315 URI tmpURI = NextURI;
316 if (tmpURI.Access == "http" && Binary == "https+http")
317 {
318 tmpURI.Access = "https+http";
319 NextURI = tmpURI;
320 }
321 if (Queue->Uri == NextURI)
322 {
323 SetFailReason("RedirectionLoop");
324 _error->Error("Redirection loop encountered");
325 if (Server->HaveContent == true)
326 return ERROR_WITH_CONTENT_PAGE;
327 return ERROR_UNRECOVERABLE;
328 }
329 URI Uri = Queue->Uri;
330 // same protocol redirects are okay
331 if (tmpURI.Access == Uri.Access)
332 return TRY_AGAIN_OR_REDIRECT;
333 // as well as http to https
334 else if ((Uri.Access == "http" || Uri.Access == "https+http") && tmpURI.Access == "https")
335 return TRY_AGAIN_OR_REDIRECT;
336 else
337 _error->Error("Redirection from %s to '%s' is forbidden", Uri.Access.c_str(), NextURI.c_str());
338 }
339 /* else pass through for error message */
340 }
341 // retry after an invalid range response without partial data
342 else if (Server->Result == 416)
343 {
344 struct stat SBuf;
345 if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
346 {
347 bool partialHit = false;
348 if (Queue->ExpectedHashes.usable() == true)
349 {
350 Hashes resultHashes(Queue->ExpectedHashes);
351 FileFd file(Queue->DestFile, FileFd::ReadOnly);
352 Server->TotalFileSize = file.FileSize();
353 Server->Date = file.ModificationTime();
354 resultHashes.AddFD(file);
355 HashStringList const hashList = resultHashes.GetHashStringList();
356 partialHit = (Queue->ExpectedHashes == hashList);
357 }
358 else if ((unsigned long long)SBuf.st_size == Server->TotalFileSize)
359 partialHit = true;
360 if (partialHit == true)
361 {
362 // the file is completely downloaded, but was not moved
363 if (Server->HaveContent == true)
364 {
365 // nuke the sent error page
366 Server->RunDataToDevNull();
367 Server->HaveContent = false;
368 }
369 Server->StartPos = Server->TotalFileSize;
370 Server->Result = 200;
371 }
372 else if (RemoveFile("server", Queue->DestFile))
373 {
374 NextURI = Queue->Uri;
375 return TRY_AGAIN_OR_REDIRECT;
376 }
377 }
378 }
379
380 /* We have a reply we don't handle. This should indicate a perm server
381 failure */
382 if (Server->Result < 200 || Server->Result >= 300)
383 {
384 if (_error->PendingError() == false)
385 {
386 std::string err;
387 strprintf(err, "HttpError%u", Server->Result);
388 SetFailReason(err);
389 _error->Error("%u %s", Server->Result, Server->Code);
390 }
391 if (Server->HaveContent == true)
392 return ERROR_WITH_CONTENT_PAGE;
393 return ERROR_UNRECOVERABLE;
394 }
395
396 // This is some sort of 2xx 'data follows' reply
397 Res.LastModified = Server->Date;
398 Res.Size = Server->TotalFileSize;
399 return FILE_IS_OPEN;
400 }
401 /*}}}*/
402 // ServerMethod::SigTerm - Handle a fatal signal /*{{{*/
403 // ---------------------------------------------------------------------
404 /* This closes and timestamps the open file. This is necessary to get
405 resume behavoir on user abort */
406 void ServerMethod::SigTerm(int)
407 {
408 if (FailFd == -1)
409 _exit(100);
410
411 struct timeval times[2];
412 times[0].tv_sec = FailTime;
413 times[1].tv_sec = FailTime;
414 times[0].tv_usec = times[1].tv_usec = 0;
415 utimes(FailFile.c_str(), times);
416 close(FailFd);
417
418 _exit(100);
419 }
420 /*}}}*/
421 // ServerMethod::Fetch - Fetch an item /*{{{*/
422 // ---------------------------------------------------------------------
423 /* This adds an item to the pipeline. We keep the pipeline at a fixed
424 depth. */
425 bool ServerMethod::Fetch(FetchItem *)
426 {
427 if (Server == nullptr || QueueBack == nullptr)
428 return true;
429
430 // If pipelining is disabled, we only queue 1 request
431 auto const AllowedDepth = Server->Pipeline ? PipelineDepth : 0;
432 // how deep is our pipeline currently?
433 decltype(PipelineDepth) CurrentDepth = 0;
434 for (FetchItem const *I = Queue; I != QueueBack; I = I->Next)
435 ++CurrentDepth;
436 if (CurrentDepth > AllowedDepth)
437 return true;
438
439 do {
440 // Make sure we stick with the same server
441 if (Server->Comp(QueueBack->Uri) == false)
442 break;
443
444 bool const UsableHashes = QueueBack->ExpectedHashes.usable();
445 // if we have no hashes, do at most one such request
446 // as we can't fixup pipeling misbehaviors otherwise
447 if (CurrentDepth != 0 && UsableHashes == false)
448 break;
449
450 if (UsableHashes && FileExists(QueueBack->DestFile))
451 {
452 FileFd partial(QueueBack->DestFile, FileFd::ReadOnly);
453 Hashes wehave(QueueBack->ExpectedHashes);
454 if (QueueBack->ExpectedHashes.FileSize() == partial.FileSize())
455 {
456 if (wehave.AddFD(partial) &&
457 wehave.GetHashStringList() == QueueBack->ExpectedHashes)
458 {
459 FetchResult Res;
460 Res.Filename = QueueBack->DestFile;
461 Res.ResumePoint = QueueBack->ExpectedHashes.FileSize();
462 URIStart(Res);
463 // move item to the start of the queue as URIDone will
464 // always dequeued the first item in the queue
465 if (Queue != QueueBack)
466 {
467 FetchItem *Prev = Queue;
468 for (; Prev->Next != QueueBack; Prev = Prev->Next)
469 /* look for the previous queue item */;
470 Prev->Next = QueueBack->Next;
471 QueueBack->Next = Queue;
472 Queue = QueueBack;
473 QueueBack = Prev->Next;
474 }
475 Res.TakeHashes(wehave);
476 URIDone(Res);
477 continue;
478 }
479 else
480 RemoveFile("Fetch-Partial", QueueBack->DestFile);
481 }
482 }
483 auto const Tmp = QueueBack;
484 QueueBack = QueueBack->Next;
485 SendReq(Tmp);
486 ++CurrentDepth;
487 } while (CurrentDepth <= AllowedDepth && QueueBack != nullptr);
488
489 return true;
490 }
491 /*}}}*/
492 // ServerMethod::Loop - Main loop /*{{{*/
493 int ServerMethod::Loop()
494 {
495 signal(SIGTERM,SigTerm);
496 signal(SIGINT,SigTerm);
497
498 Server = 0;
499
500 int FailCounter = 0;
501 while (1)
502 {
503 // We have no commands, wait for some to arrive
504 if (Queue == 0)
505 {
506 if (WaitFd(STDIN_FILENO) == false)
507 return 0;
508 }
509
510 /* Run messages, we can accept 0 (no message) if we didn't
511 do a WaitFd above.. Otherwise the FD is closed. */
512 int Result = Run(true);
513 if (Result != -1 && (Result != 0 || Queue == 0))
514 {
515 if(FailReason.empty() == false ||
516 _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
517 return 100;
518 else
519 return 0;
520 }
521
522 if (Queue == 0)
523 continue;
524
525 // Connect to the server
526 if (Server == 0 || Server->Comp(Queue->Uri) == false)
527 Server = CreateServerState(Queue->Uri);
528
529 /* If the server has explicitly said this is the last connection
530 then we pre-emptively shut down the pipeline and tear down
531 the connection. This will speed up HTTP/1.0 servers a tad
532 since we don't have to wait for the close sequence to
533 complete */
534 if (Server->Persistent == false)
535 Server->Close();
536
537 // Reset the pipeline
538 if (Server->IsOpen() == false)
539 QueueBack = Queue;
540
541 // Connnect to the host
542 if (Server->Open() == false)
543 {
544 Fail(true);
545 Server = nullptr;
546 continue;
547 }
548
549 // Fill the pipeline.
550 Fetch(0);
551
552 // Fetch the next URL header data from the server.
553 switch (Server->RunHeaders(File, Queue->Uri))
554 {
555 case ServerState::RUN_HEADERS_OK:
556 break;
557
558 // The header data is bad
559 case ServerState::RUN_HEADERS_PARSE_ERROR:
560 {
561 _error->Error(_("Bad header data"));
562 Fail(true);
563 Server->Close();
564 RotateDNS();
565 continue;
566 }
567
568 // The server closed a connection during the header get..
569 default:
570 case ServerState::RUN_HEADERS_IO_ERROR:
571 {
572 FailCounter++;
573 _error->Discard();
574 Server->Close();
575 Server->Pipeline = false;
576 Server->PipelineAllowed = false;
577
578 if (FailCounter >= 2)
579 {
580 Fail(_("Connection failed"),true);
581 FailCounter = 0;
582 }
583
584 RotateDNS();
585 continue;
586 }
587 };
588
589 // Decide what to do.
590 FetchResult Res;
591 Res.Filename = Queue->DestFile;
592 switch (DealWithHeaders(Res))
593 {
594 // Ok, the file is Open
595 case FILE_IS_OPEN:
596 {
597 URIStart(Res);
598
599 // Run the data
600 bool Result = true;
601
602 // ensure we don't fetch too much
603 // we could do "Server->MaximumSize = Queue->MaximumSize" here
604 // but that would break the clever pipeline messup detection
605 // so instead we use the size of the biggest item in the queue
606 Server->MaximumSize = FindMaximumObjectSizeInQueue();
607
608 if (Server->HaveContent)
609 Result = Server->RunData(File);
610
611 /* If the server is sending back sizeless responses then fill in
612 the size now */
613 if (Res.Size == 0)
614 Res.Size = File->Size();
615
616 // Close the file, destroy the FD object and timestamp it
617 FailFd = -1;
618 delete File;
619 File = 0;
620
621 // Timestamp
622 struct timeval times[2];
623 times[0].tv_sec = times[1].tv_sec = Server->Date;
624 times[0].tv_usec = times[1].tv_usec = 0;
625 utimes(Queue->DestFile.c_str(), times);
626
627 // Send status to APT
628 if (Result == true)
629 {
630 Hashes * const resultHashes = Server->GetHashes();
631 HashStringList const hashList = resultHashes->GetHashStringList();
632 if (PipelineDepth != 0 && Queue->ExpectedHashes.usable() == true && Queue->ExpectedHashes != hashList)
633 {
634 // we did not get the expected hash… mhhh:
635 // could it be that server/proxy messed up pipelining?
636 FetchItem * BeforeI = Queue;
637 for (FetchItem *I = Queue->Next; I != 0 && I != QueueBack; I = I->Next)
638 {
639 if (I->ExpectedHashes.usable() == true && I->ExpectedHashes == hashList)
640 {
641 // yes, he did! Disable pipelining and rewrite queue
642 if (Server->Pipeline == true)
643 {
644 Warning(_("Automatically disabled %s due to incorrect response from server/proxy. (man 5 apt.conf)"), "Acquire::http::Pipeline-Depth");
645 Server->Pipeline = false;
646 Server->PipelineAllowed = false;
647 // we keep the PipelineDepth value so that the rest of the queue can be fixed up as well
648 }
649 Rename(Res.Filename, I->DestFile);
650 Res.Filename = I->DestFile;
651 BeforeI->Next = I->Next;
652 I->Next = Queue;
653 Queue = I;
654 break;
655 }
656 BeforeI = I;
657 }
658 }
659 Res.TakeHashes(*resultHashes);
660 URIDone(Res);
661 }
662 else
663 {
664 if (Server->IsOpen() == false)
665 {
666 FailCounter++;
667 _error->Discard();
668 Server->Close();
669
670 if (FailCounter >= 2)
671 {
672 Fail(_("Connection failed"),true);
673 FailCounter = 0;
674 }
675
676 QueueBack = Queue;
677 }
678 else
679 {
680 Server->Close();
681 Fail(true);
682 }
683 }
684 break;
685 }
686
687 // IMS hit
688 case IMS_HIT:
689 {
690 URIDone(Res);
691 break;
692 }
693
694 // Hard server error, not found or something
695 case ERROR_UNRECOVERABLE:
696 {
697 Fail();
698 break;
699 }
700
701 // Hard internal error, kill the connection and fail
702 case ERROR_NOT_FROM_SERVER:
703 {
704 delete File;
705 File = 0;
706
707 Fail();
708 RotateDNS();
709 Server->Close();
710 break;
711 }
712
713 // We need to flush the data, the header is like a 404 w/ error text
714 case ERROR_WITH_CONTENT_PAGE:
715 {
716 Fail();
717 Server->RunDataToDevNull();
718 break;
719 }
720
721 // Try again with a new URL
722 case TRY_AGAIN_OR_REDIRECT:
723 {
724 // Clear rest of response if there is content
725 if (Server->HaveContent)
726 Server->RunDataToDevNull();
727 Redirect(NextURI);
728 break;
729 }
730
731 default:
732 Fail(_("Internal error"));
733 break;
734 }
735
736 FailCounter = 0;
737 }
738
739 return 0;
740 }
741 /*}}}*/
742 unsigned long long ServerMethod::FindMaximumObjectSizeInQueue() const /*{{{*/
743 {
744 unsigned long long MaxSizeInQueue = 0;
745 for (FetchItem *I = Queue; I != 0 && I != QueueBack; I = I->Next)
746 MaxSizeInQueue = std::max(MaxSizeInQueue, I->MaximumSize);
747 return MaxSizeInQueue;
748 }
749 /*}}}*/
750 ServerMethod::ServerMethod(char const * const Binary, char const * const Ver,unsigned long const Flags) :/*{{{*/
751 aptMethod(Binary, Ver, Flags), Server(nullptr), File(NULL), PipelineDepth(10),
752 AllowRedirect(false), Debug(false)
753 {
754 }
755 /*}}}*/