]> git.saurik.com Git - apt.git/blob - methods/http.cc
show in the debug output if we are looping in the avoid breaks
[apt.git] / methods / http.cc
1 // -*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
4 /* ######################################################################
5
6 HTTP Acquire Method - This is the HTTP aquire method for APT.
7
8 It uses HTTP/1.1 and many of the fancy options there-in, such as
9 pipelining, range, if-range and so on.
10
11 It is based on a doubly buffered select loop. A groupe of requests are
12 fed into a single output buffer that is constantly fed out the
13 socket. This provides ideal pipelining as in many cases all of the
14 requests will fit into a single packet. The input socket is buffered
15 the same way and fed into the fd for the file (may be a pipe in future).
16
17 This double buffering provides fairly substantial transfer rates,
18 compared to wget the http method is about 4% faster. Most importantly,
19 when HTTP is compared with FTP as a protocol the speed difference is
20 huge. In tests over the internet from two sites to llug (via ATM) this
21 program got 230k/s sustained http transfer rates. FTP on the other
22 hand topped out at 170k/s. That combined with the time to setup the
23 FTP connection makes HTTP a vastly superior protocol.
24
25 ##################################################################### */
26 /*}}}*/
27 // Include Files /*{{{*/
28 #include <config.h>
29
30 #include <apt-pkg/fileutl.h>
31 #include <apt-pkg/acquire-method.h>
32 #include <apt-pkg/configuration.h>
33 #include <apt-pkg/error.h>
34 #include <apt-pkg/hashes.h>
35 #include <apt-pkg/netrc.h>
36
37 #include <sys/stat.h>
38 #include <sys/time.h>
39 #include <utime.h>
40 #include <unistd.h>
41 #include <signal.h>
42 #include <stdio.h>
43 #include <errno.h>
44 #include <string.h>
45 #include <iostream>
46 #include <map>
47
48 // Internet stuff
49 #include <netdb.h>
50
51 #include "config.h"
52 #include "connect.h"
53 #include "rfc2553emu.h"
54 #include "http.h"
55
56 #include <apti18n.h>
57 /*}}}*/
58 using namespace std;
59
60 string HttpMethod::FailFile;
61 int HttpMethod::FailFd = -1;
62 time_t HttpMethod::FailTime = 0;
63 unsigned long PipelineDepth = 10;
64 unsigned long TimeOut = 120;
65 bool AllowRedirect = false;
66 bool Debug = false;
67 URI Proxy;
68
69 unsigned long long CircleBuf::BwReadLimit=0;
70 unsigned long long CircleBuf::BwTickReadData=0;
71 struct timeval CircleBuf::BwReadTick={0,0};
72 const unsigned int CircleBuf::BW_HZ=10;
73
74 // CircleBuf::CircleBuf - Circular input buffer /*{{{*/
75 // ---------------------------------------------------------------------
76 /* */
77 CircleBuf::CircleBuf(unsigned long long Size) : Size(Size), Hash(0)
78 {
79 Buf = new unsigned char[Size];
80 Reset();
81
82 CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
83 }
84 /*}}}*/
85 // CircleBuf::Reset - Reset to the default state /*{{{*/
86 // ---------------------------------------------------------------------
87 /* */
88 void CircleBuf::Reset()
89 {
90 InP = 0;
91 OutP = 0;
92 StrPos = 0;
93 MaxGet = (unsigned long long)-1;
94 OutQueue = string();
95 if (Hash != 0)
96 {
97 delete Hash;
98 Hash = new Hashes;
99 }
100 };
101 /*}}}*/
102 // CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
103 // ---------------------------------------------------------------------
104 /* This fills up the buffer with as much data as is in the FD, assuming it
105 is non-blocking.. */
106 bool CircleBuf::Read(int Fd)
107 {
108 unsigned long long BwReadMax;
109
110 while (1)
111 {
112 // Woops, buffer is full
113 if (InP - OutP == Size)
114 return true;
115
116 // what's left to read in this tick
117 BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
118
119 if(CircleBuf::BwReadLimit) {
120 struct timeval now;
121 gettimeofday(&now,0);
122
123 unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
124 now.tv_usec-CircleBuf::BwReadTick.tv_usec;
125 if(d > 1000000/BW_HZ) {
126 CircleBuf::BwReadTick = now;
127 CircleBuf::BwTickReadData = 0;
128 }
129
130 if(CircleBuf::BwTickReadData >= BwReadMax) {
131 usleep(1000000/BW_HZ);
132 return true;
133 }
134 }
135
136 // Write the buffer segment
137 ssize_t Res;
138 if(CircleBuf::BwReadLimit) {
139 Res = read(Fd,Buf + (InP%Size),
140 BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
141 } else
142 Res = read(Fd,Buf + (InP%Size),LeftRead());
143
144 if(Res > 0 && BwReadLimit > 0)
145 CircleBuf::BwTickReadData += Res;
146
147 if (Res == 0)
148 return false;
149 if (Res < 0)
150 {
151 if (errno == EAGAIN)
152 return true;
153 return false;
154 }
155
156 if (InP == 0)
157 gettimeofday(&Start,0);
158 InP += Res;
159 }
160 }
161 /*}}}*/
162 // CircleBuf::Read - Put the string into the buffer /*{{{*/
163 // ---------------------------------------------------------------------
164 /* This will hold the string in and fill the buffer with it as it empties */
165 bool CircleBuf::Read(string Data)
166 {
167 OutQueue += Data;
168 FillOut();
169 return true;
170 }
171 /*}}}*/
172 // CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
173 // ---------------------------------------------------------------------
174 /* */
175 void CircleBuf::FillOut()
176 {
177 if (OutQueue.empty() == true)
178 return;
179 while (1)
180 {
181 // Woops, buffer is full
182 if (InP - OutP == Size)
183 return;
184
185 // Write the buffer segment
186 unsigned long long Sz = LeftRead();
187 if (OutQueue.length() - StrPos < Sz)
188 Sz = OutQueue.length() - StrPos;
189 memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
190
191 // Advance
192 StrPos += Sz;
193 InP += Sz;
194 if (OutQueue.length() == StrPos)
195 {
196 StrPos = 0;
197 OutQueue = "";
198 return;
199 }
200 }
201 }
202 /*}}}*/
203 // CircleBuf::Write - Write from the buffer into a FD /*{{{*/
204 // ---------------------------------------------------------------------
205 /* This empties the buffer into the FD. */
206 bool CircleBuf::Write(int Fd)
207 {
208 while (1)
209 {
210 FillOut();
211
212 // Woops, buffer is empty
213 if (OutP == InP)
214 return true;
215
216 if (OutP == MaxGet)
217 return true;
218
219 // Write the buffer segment
220 ssize_t Res;
221 Res = write(Fd,Buf + (OutP%Size),LeftWrite());
222
223 if (Res == 0)
224 return false;
225 if (Res < 0)
226 {
227 if (errno == EAGAIN)
228 return true;
229
230 return false;
231 }
232
233 if (Hash != 0)
234 Hash->Add(Buf + (OutP%Size),Res);
235
236 OutP += Res;
237 }
238 }
239 /*}}}*/
240 // CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
241 // ---------------------------------------------------------------------
242 /* This copies till the first empty line */
243 bool CircleBuf::WriteTillEl(string &Data,bool Single)
244 {
245 // We cheat and assume it is unneeded to have more than one buffer load
246 for (unsigned long long I = OutP; I < InP; I++)
247 {
248 if (Buf[I%Size] != '\n')
249 continue;
250 ++I;
251
252 if (Single == false)
253 {
254 if (I < InP && Buf[I%Size] == '\r')
255 ++I;
256 if (I >= InP || Buf[I%Size] != '\n')
257 continue;
258 ++I;
259 }
260
261 Data = "";
262 while (OutP < I)
263 {
264 unsigned long long Sz = LeftWrite();
265 if (Sz == 0)
266 return false;
267 if (I - OutP < Sz)
268 Sz = I - OutP;
269 Data += string((char *)(Buf + (OutP%Size)),Sz);
270 OutP += Sz;
271 }
272 return true;
273 }
274 return false;
275 }
276 /*}}}*/
277 // CircleBuf::Stats - Print out stats information /*{{{*/
278 // ---------------------------------------------------------------------
279 /* */
280 void CircleBuf::Stats()
281 {
282 if (InP == 0)
283 return;
284
285 struct timeval Stop;
286 gettimeofday(&Stop,0);
287 /* float Diff = Stop.tv_sec - Start.tv_sec +
288 (float)(Stop.tv_usec - Start.tv_usec)/1000000;
289 clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
290 }
291 /*}}}*/
292 CircleBuf::~CircleBuf()
293 {
294 delete [] Buf;
295 delete Hash;
296 }
297
298 // ServerState::ServerState - Constructor /*{{{*/
299 // ---------------------------------------------------------------------
300 /* */
301 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
302 In(64*1024), Out(4*1024),
303 ServerName(Srv)
304 {
305 Reset();
306 }
307 /*}}}*/
308 // ServerState::Open - Open a connection to the server /*{{{*/
309 // ---------------------------------------------------------------------
310 /* This opens a connection to the server. */
311 bool ServerState::Open()
312 {
313 // Use the already open connection if possible.
314 if (ServerFd != -1)
315 return true;
316
317 Close();
318 In.Reset();
319 Out.Reset();
320 Persistent = true;
321
322 // Determine the proxy setting
323 string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
324 if (!SpecificProxy.empty())
325 {
326 if (SpecificProxy == "DIRECT")
327 Proxy = "";
328 else
329 Proxy = SpecificProxy;
330 }
331 else
332 {
333 string DefProxy = _config->Find("Acquire::http::Proxy");
334 if (!DefProxy.empty())
335 {
336 Proxy = DefProxy;
337 }
338 else
339 {
340 char* result = getenv("http_proxy");
341 Proxy = result ? result : "";
342 }
343 }
344
345 // Parse no_proxy, a , separated list of domains
346 if (getenv("no_proxy") != 0)
347 {
348 if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
349 Proxy = "";
350 }
351
352 // Determine what host and port to use based on the proxy settings
353 int Port = 0;
354 string Host;
355 if (Proxy.empty() == true || Proxy.Host.empty() == true)
356 {
357 if (ServerName.Port != 0)
358 Port = ServerName.Port;
359 Host = ServerName.Host;
360 }
361 else
362 {
363 if (Proxy.Port != 0)
364 Port = Proxy.Port;
365 Host = Proxy.Host;
366 }
367
368 // Connect to the remote server
369 if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
370 return false;
371
372 return true;
373 }
374 /*}}}*/
375 // ServerState::Close - Close a connection to the server /*{{{*/
376 // ---------------------------------------------------------------------
377 /* */
378 bool ServerState::Close()
379 {
380 close(ServerFd);
381 ServerFd = -1;
382 return true;
383 }
384 /*}}}*/
385 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
386 // ---------------------------------------------------------------------
387 /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
388 parse error occurred */
389 ServerState::RunHeadersResult ServerState::RunHeaders()
390 {
391 State = Header;
392
393 Owner->Status(_("Waiting for headers"));
394
395 Major = 0;
396 Minor = 0;
397 Result = 0;
398 Size = 0;
399 StartPos = 0;
400 Encoding = Closes;
401 HaveContent = false;
402 time(&Date);
403
404 do
405 {
406 string Data;
407 if (In.WriteTillEl(Data) == false)
408 continue;
409
410 if (Debug == true)
411 clog << Data;
412
413 for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
414 {
415 string::const_iterator J = I;
416 for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
417 if (HeaderLine(string(I,J)) == false)
418 return RUN_HEADERS_PARSE_ERROR;
419 I = J;
420 }
421
422 // 100 Continue is a Nop...
423 if (Result == 100)
424 continue;
425
426 // Tidy up the connection persistance state.
427 if (Encoding == Closes && HaveContent == true)
428 Persistent = false;
429
430 return RUN_HEADERS_OK;
431 }
432 while (Owner->Go(false,this) == true);
433
434 return RUN_HEADERS_IO_ERROR;
435 }
436 /*}}}*/
437 // ServerState::RunData - Transfer the data from the socket /*{{{*/
438 // ---------------------------------------------------------------------
439 /* */
440 bool ServerState::RunData()
441 {
442 State = Data;
443
444 // Chunked transfer encoding is fun..
445 if (Encoding == Chunked)
446 {
447 while (1)
448 {
449 // Grab the block size
450 bool Last = true;
451 string Data;
452 In.Limit(-1);
453 do
454 {
455 if (In.WriteTillEl(Data,true) == true)
456 break;
457 }
458 while ((Last = Owner->Go(false,this)) == true);
459
460 if (Last == false)
461 return false;
462
463 // See if we are done
464 unsigned long long Len = strtoull(Data.c_str(),0,16);
465 if (Len == 0)
466 {
467 In.Limit(-1);
468
469 // We have to remove the entity trailer
470 Last = true;
471 do
472 {
473 if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
474 break;
475 }
476 while ((Last = Owner->Go(false,this)) == true);
477 if (Last == false)
478 return false;
479 return !_error->PendingError();
480 }
481
482 // Transfer the block
483 In.Limit(Len);
484 while (Owner->Go(true,this) == true)
485 if (In.IsLimit() == true)
486 break;
487
488 // Error
489 if (In.IsLimit() == false)
490 return false;
491
492 // The server sends an extra new line before the next block specifier..
493 In.Limit(-1);
494 Last = true;
495 do
496 {
497 if (In.WriteTillEl(Data,true) == true)
498 break;
499 }
500 while ((Last = Owner->Go(false,this)) == true);
501 if (Last == false)
502 return false;
503 }
504 }
505 else
506 {
507 /* Closes encoding is used when the server did not specify a size, the
508 loss of the connection means we are done */
509 if (Encoding == Closes)
510 In.Limit(-1);
511 else
512 In.Limit(Size - StartPos);
513
514 // Just transfer the whole block.
515 do
516 {
517 if (In.IsLimit() == false)
518 continue;
519
520 In.Limit(-1);
521 return !_error->PendingError();
522 }
523 while (Owner->Go(true,this) == true);
524 }
525
526 return Owner->Flush(this) && !_error->PendingError();
527 }
528 /*}}}*/
529 // ServerState::HeaderLine - Process a header line /*{{{*/
530 // ---------------------------------------------------------------------
531 /* */
532 bool ServerState::HeaderLine(string Line)
533 {
534 if (Line.empty() == true)
535 return true;
536
537 string::size_type Pos = Line.find(' ');
538 if (Pos == string::npos || Pos+1 > Line.length())
539 {
540 // Blah, some servers use "connection:closes", evil.
541 Pos = Line.find(':');
542 if (Pos == string::npos || Pos + 2 > Line.length())
543 return _error->Error(_("Bad header line"));
544 Pos++;
545 }
546
547 // Parse off any trailing spaces between the : and the next word.
548 string::size_type Pos2 = Pos;
549 while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
550 Pos2++;
551
552 string Tag = string(Line,0,Pos);
553 string Val = string(Line,Pos2);
554
555 if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
556 {
557 // Evil servers return no version
558 if (Line[4] == '/')
559 {
560 int const elements = sscanf(Line.c_str(),"HTTP/%u.%u %u%[^\n]",&Major,&Minor,&Result,Code);
561 if (elements == 3)
562 {
563 Code[0] = '\0';
564 if (Debug == true)
565 clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
566 }
567 else if (elements != 4)
568 return _error->Error(_("The HTTP server sent an invalid reply header"));
569 }
570 else
571 {
572 Major = 0;
573 Minor = 9;
574 if (sscanf(Line.c_str(),"HTTP %u%[^\n]",&Result,Code) != 2)
575 return _error->Error(_("The HTTP server sent an invalid reply header"));
576 }
577
578 /* Check the HTTP response header to get the default persistance
579 state. */
580 if (Major < 1)
581 Persistent = false;
582 else
583 {
584 if (Major == 1 && Minor <= 0)
585 Persistent = false;
586 else
587 Persistent = true;
588 }
589
590 return true;
591 }
592
593 if (stringcasecmp(Tag,"Content-Length:") == 0)
594 {
595 if (Encoding == Closes)
596 Encoding = Stream;
597 HaveContent = true;
598
599 // The length is already set from the Content-Range header
600 if (StartPos != 0)
601 return true;
602
603 if (sscanf(Val.c_str(),"%llu",&Size) != 1)
604 return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
605 return true;
606 }
607
608 if (stringcasecmp(Tag,"Content-Type:") == 0)
609 {
610 HaveContent = true;
611 return true;
612 }
613
614 if (stringcasecmp(Tag,"Content-Range:") == 0)
615 {
616 HaveContent = true;
617
618 if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
619 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
620 if ((unsigned long long)StartPos > Size)
621 return _error->Error(_("This HTTP server has broken range support"));
622 return true;
623 }
624
625 if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
626 {
627 HaveContent = true;
628 if (stringcasecmp(Val,"chunked") == 0)
629 Encoding = Chunked;
630 return true;
631 }
632
633 if (stringcasecmp(Tag,"Connection:") == 0)
634 {
635 if (stringcasecmp(Val,"close") == 0)
636 Persistent = false;
637 if (stringcasecmp(Val,"keep-alive") == 0)
638 Persistent = true;
639 return true;
640 }
641
642 if (stringcasecmp(Tag,"Last-Modified:") == 0)
643 {
644 if (RFC1123StrToTime(Val.c_str(), Date) == false)
645 return _error->Error(_("Unknown date format"));
646 return true;
647 }
648
649 if (stringcasecmp(Tag,"Location:") == 0)
650 {
651 Location = Val;
652 return true;
653 }
654
655 return true;
656 }
657 /*}}}*/
658
659 // HttpMethod::SendReq - Send the HTTP request /*{{{*/
660 // ---------------------------------------------------------------------
661 /* This places the http request in the outbound buffer */
662 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
663 {
664 URI Uri = Itm->Uri;
665
666 // The HTTP server expects a hostname with a trailing :port
667 char Buf[1000];
668 string ProperHost = Uri.Host;
669 if (Uri.Port != 0)
670 {
671 sprintf(Buf,":%u",Uri.Port);
672 ProperHost += Buf;
673 }
674
675 // Just in case.
676 if (Itm->Uri.length() >= sizeof(Buf))
677 abort();
678
679 /* Build the request. We include a keep-alive header only for non-proxy
680 requests. This is to tweak old http/1.0 servers that do support keep-alive
681 but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
682 will glitch HTTP/1.0 proxies because they do not filter it out and
683 pass it on, HTTP/1.1 says the connection should default to keep alive
684 and we expect the proxy to do this */
685 if (Proxy.empty() == true || Proxy.Host.empty())
686 sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
687 QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
688 else
689 {
690 /* Generate a cache control header if necessary. We place a max
691 cache age on index files, optionally set a no-cache directive
692 and a no-store directive for archives. */
693 sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
694 Itm->Uri.c_str(),ProperHost.c_str());
695 }
696 // generate a cache control header (if needed)
697 if (_config->FindB("Acquire::http::No-Cache",false) == true)
698 {
699 strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
700 }
701 else
702 {
703 if (Itm->IndexFile == true)
704 {
705 sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
706 _config->FindI("Acquire::http::Max-Age",0));
707 }
708 else
709 {
710 if (_config->FindB("Acquire::http::No-Store",false) == true)
711 strcat(Buf,"Cache-Control: no-store\r\n");
712 }
713 }
714
715 // If we ask for uncompressed files servers might respond with content-
716 // negotation which lets us end up with compressed files we do not support,
717 // see 657029, 657560 and co, so if we have no extension on the request
718 // ask for text only. As a sidenote: If there is nothing to negotate servers
719 // seem to be nice and ignore it.
720 if (_config->FindB("Acquire::http::SendAccept", true) == true)
721 {
722 size_t const filepos = Itm->Uri.find_last_of('/');
723 string const file = Itm->Uri.substr(filepos + 1);
724 if (flExtension(file) == file)
725 strcat(Buf,"Accept: text/*\r\n");
726 }
727
728 string Req = Buf;
729
730 // Check for a partial file
731 struct stat SBuf;
732 if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
733 {
734 // In this case we send an if-range query with a range header
735 sprintf(Buf,"Range: bytes=%lli-\r\nIf-Range: %s\r\n",(long long)SBuf.st_size - 1,
736 TimeRFC1123(SBuf.st_mtime).c_str());
737 Req += Buf;
738 }
739 else
740 {
741 if (Itm->LastModified != 0)
742 {
743 sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
744 Req += Buf;
745 }
746 }
747
748 if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
749 Req += string("Proxy-Authorization: Basic ") +
750 Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
751
752 maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
753 if (Uri.User.empty() == false || Uri.Password.empty() == false)
754 {
755 Req += string("Authorization: Basic ") +
756 Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
757 }
758 Req += "User-Agent: " + _config->Find("Acquire::http::User-Agent",
759 "Debian APT-HTTP/1.3 ("VERSION")") + "\r\n\r\n";
760
761 if (Debug == true)
762 cerr << Req << endl;
763
764 Out.Read(Req);
765 }
766 /*}}}*/
767 // HttpMethod::Go - Run a single loop /*{{{*/
768 // ---------------------------------------------------------------------
769 /* This runs the select loop over the server FDs, Output file FDs and
770 stdin. */
771 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
772 {
773 // Server has closed the connection
774 if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
775 ToFile == false))
776 return false;
777
778 fd_set rfds,wfds;
779 FD_ZERO(&rfds);
780 FD_ZERO(&wfds);
781
782 /* Add the server. We only send more requests if the connection will
783 be persisting */
784 if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
785 && Srv->Persistent == true)
786 FD_SET(Srv->ServerFd,&wfds);
787 if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
788 FD_SET(Srv->ServerFd,&rfds);
789
790 // Add the file
791 int FileFD = -1;
792 if (File != 0)
793 FileFD = File->Fd();
794
795 if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
796 FD_SET(FileFD,&wfds);
797
798 // Add stdin
799 if (_config->FindB("Acquire::http::DependOnSTDIN", true) == true)
800 FD_SET(STDIN_FILENO,&rfds);
801
802 // Figure out the max fd
803 int MaxFd = FileFD;
804 if (MaxFd < Srv->ServerFd)
805 MaxFd = Srv->ServerFd;
806
807 // Select
808 struct timeval tv;
809 tv.tv_sec = TimeOut;
810 tv.tv_usec = 0;
811 int Res = 0;
812 if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
813 {
814 if (errno == EINTR)
815 return true;
816 return _error->Errno("select",_("Select failed"));
817 }
818
819 if (Res == 0)
820 {
821 _error->Error(_("Connection timed out"));
822 return ServerDie(Srv);
823 }
824
825 // Handle server IO
826 if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
827 {
828 errno = 0;
829 if (Srv->In.Read(Srv->ServerFd) == false)
830 return ServerDie(Srv);
831 }
832
833 if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
834 {
835 errno = 0;
836 if (Srv->Out.Write(Srv->ServerFd) == false)
837 return ServerDie(Srv);
838 }
839
840 // Send data to the file
841 if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
842 {
843 if (Srv->In.Write(FileFD) == false)
844 return _error->Errno("write",_("Error writing to output file"));
845 }
846
847 // Handle commands from APT
848 if (FD_ISSET(STDIN_FILENO,&rfds))
849 {
850 if (Run(true) != -1)
851 exit(100);
852 }
853
854 return true;
855 }
856 /*}}}*/
857 // HttpMethod::Flush - Dump the buffer into the file /*{{{*/
858 // ---------------------------------------------------------------------
859 /* This takes the current input buffer from the Server FD and writes it
860 into the file */
861 bool HttpMethod::Flush(ServerState *Srv)
862 {
863 if (File != 0)
864 {
865 // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
866 // can't be set
867 if (File->Name() != "/dev/null")
868 SetNonBlock(File->Fd(),false);
869 if (Srv->In.WriteSpace() == false)
870 return true;
871
872 while (Srv->In.WriteSpace() == true)
873 {
874 if (Srv->In.Write(File->Fd()) == false)
875 return _error->Errno("write",_("Error writing to file"));
876 if (Srv->In.IsLimit() == true)
877 return true;
878 }
879
880 if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
881 return true;
882 }
883 return false;
884 }
885 /*}}}*/
886 // HttpMethod::ServerDie - The server has closed the connection. /*{{{*/
887 // ---------------------------------------------------------------------
888 /* */
889 bool HttpMethod::ServerDie(ServerState *Srv)
890 {
891 unsigned int LErrno = errno;
892
893 // Dump the buffer to the file
894 if (Srv->State == ServerState::Data)
895 {
896 // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
897 // can't be set
898 if (File->Name() != "/dev/null")
899 SetNonBlock(File->Fd(),false);
900 while (Srv->In.WriteSpace() == true)
901 {
902 if (Srv->In.Write(File->Fd()) == false)
903 return _error->Errno("write",_("Error writing to the file"));
904
905 // Done
906 if (Srv->In.IsLimit() == true)
907 return true;
908 }
909 }
910
911 // See if this is because the server finished the data stream
912 if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
913 Srv->Encoding != ServerState::Closes)
914 {
915 Srv->Close();
916 if (LErrno == 0)
917 return _error->Error(_("Error reading from server. Remote end closed connection"));
918 errno = LErrno;
919 return _error->Errno("read",_("Error reading from server"));
920 }
921 else
922 {
923 Srv->In.Limit(-1);
924
925 // Nothing left in the buffer
926 if (Srv->In.WriteSpace() == false)
927 return false;
928
929 // We may have got multiple responses back in one packet..
930 Srv->Close();
931 return true;
932 }
933
934 return false;
935 }
936 /*}}}*/
937 // HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
938 // ---------------------------------------------------------------------
939 /* We look at the header data we got back from the server and decide what
940 to do. Returns DealWithHeadersResult (see http.h for details).
941 */
942 HttpMethod::DealWithHeadersResult
943 HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
944 {
945 // Not Modified
946 if (Srv->Result == 304)
947 {
948 unlink(Queue->DestFile.c_str());
949 Res.IMSHit = true;
950 Res.LastModified = Queue->LastModified;
951 return IMS_HIT;
952 }
953
954 /* Redirect
955 *
956 * Note that it is only OK for us to treat all redirection the same
957 * because we *always* use GET, not other HTTP methods. There are
958 * three redirection codes for which it is not appropriate that we
959 * redirect. Pass on those codes so the error handling kicks in.
960 */
961 if (AllowRedirect
962 && (Srv->Result > 300 && Srv->Result < 400)
963 && (Srv->Result != 300 // Multiple Choices
964 && Srv->Result != 304 // Not Modified
965 && Srv->Result != 306)) // (Not part of HTTP/1.1, reserved)
966 {
967 if (Srv->Location.empty() == true);
968 else if (Srv->Location[0] == '/' && Queue->Uri.empty() == false)
969 {
970 URI Uri = Queue->Uri;
971 if (Uri.Host.empty() == false)
972 {
973 if (Uri.Port != 0)
974 strprintf(NextURI, "http://%s:%u", Uri.Host.c_str(), Uri.Port);
975 else
976 NextURI = "http://" + Uri.Host;
977 }
978 else
979 NextURI.clear();
980 NextURI.append(DeQuoteString(Srv->Location));
981 return TRY_AGAIN_OR_REDIRECT;
982 }
983 else
984 {
985 NextURI = DeQuoteString(Srv->Location);
986 return TRY_AGAIN_OR_REDIRECT;
987 }
988 /* else pass through for error message */
989 }
990
991 /* We have a reply we dont handle. This should indicate a perm server
992 failure */
993 if (Srv->Result < 200 || Srv->Result >= 300)
994 {
995 char err[255];
996 snprintf(err,sizeof(err)-1,"HttpError%i",Srv->Result);
997 SetFailReason(err);
998 _error->Error("%u %s",Srv->Result,Srv->Code);
999 if (Srv->HaveContent == true)
1000 return ERROR_WITH_CONTENT_PAGE;
1001 return ERROR_UNRECOVERABLE;
1002 }
1003
1004 // This is some sort of 2xx 'data follows' reply
1005 Res.LastModified = Srv->Date;
1006 Res.Size = Srv->Size;
1007
1008 // Open the file
1009 delete File;
1010 File = new FileFd(Queue->DestFile,FileFd::WriteAny);
1011 if (_error->PendingError() == true)
1012 return ERROR_NOT_FROM_SERVER;
1013
1014 FailFile = Queue->DestFile;
1015 FailFile.c_str(); // Make sure we dont do a malloc in the signal handler
1016 FailFd = File->Fd();
1017 FailTime = Srv->Date;
1018
1019 delete Srv->In.Hash;
1020 Srv->In.Hash = new Hashes;
1021
1022 // Set the expected size and read file for the hashes
1023 if (Srv->StartPos >= 0)
1024 {
1025 Res.ResumePoint = Srv->StartPos;
1026 File->Truncate(Srv->StartPos);
1027
1028 if (Srv->In.Hash->AddFD(*File,Srv->StartPos) == false)
1029 {
1030 _error->Errno("read",_("Problem hashing file"));
1031 return ERROR_NOT_FROM_SERVER;
1032 }
1033 }
1034
1035 SetNonBlock(File->Fd(),true);
1036 return FILE_IS_OPEN;
1037 }
1038 /*}}}*/
1039 // HttpMethod::SigTerm - Handle a fatal signal /*{{{*/
1040 // ---------------------------------------------------------------------
1041 /* This closes and timestamps the open file. This is neccessary to get
1042 resume behavoir on user abort */
1043 void HttpMethod::SigTerm(int)
1044 {
1045 if (FailFd == -1)
1046 _exit(100);
1047 close(FailFd);
1048
1049 // Timestamp
1050 struct utimbuf UBuf;
1051 UBuf.actime = FailTime;
1052 UBuf.modtime = FailTime;
1053 utime(FailFile.c_str(),&UBuf);
1054
1055 _exit(100);
1056 }
1057 /*}}}*/
1058 // HttpMethod::Fetch - Fetch an item /*{{{*/
1059 // ---------------------------------------------------------------------
1060 /* This adds an item to the pipeline. We keep the pipeline at a fixed
1061 depth. */
1062 bool HttpMethod::Fetch(FetchItem *)
1063 {
1064 if (Server == 0)
1065 return true;
1066
1067 // Queue the requests
1068 int Depth = -1;
1069 for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
1070 I = I->Next, Depth++)
1071 {
1072 // If pipelining is disabled, we only queue 1 request
1073 if (Server->Pipeline == false && Depth >= 0)
1074 break;
1075
1076 // Make sure we stick with the same server
1077 if (Server->Comp(I->Uri) == false)
1078 break;
1079 if (QueueBack == I)
1080 {
1081 QueueBack = I->Next;
1082 SendReq(I,Server->Out);
1083 continue;
1084 }
1085 }
1086
1087 return true;
1088 };
1089 /*}}}*/
1090 // HttpMethod::Configuration - Handle a configuration message /*{{{*/
1091 // ---------------------------------------------------------------------
1092 /* We stash the desired pipeline depth */
1093 bool HttpMethod::Configuration(string Message)
1094 {
1095 if (pkgAcqMethod::Configuration(Message) == false)
1096 return false;
1097
1098 AllowRedirect = _config->FindB("Acquire::http::AllowRedirect",true);
1099 TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1100 PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1101 PipelineDepth);
1102 Debug = _config->FindB("Debug::Acquire::http",false);
1103 AutoDetectProxyCmd = _config->Find("Acquire::http::ProxyAutoDetect");
1104
1105 // Get the proxy to use
1106 AutoDetectProxy();
1107
1108 return true;
1109 }
1110 /*}}}*/
1111 // HttpMethod::Loop - Main loop /*{{{*/
1112 // ---------------------------------------------------------------------
1113 /* */
1114 int HttpMethod::Loop()
1115 {
1116 typedef vector<string> StringVector;
1117 typedef vector<string>::iterator StringVectorIterator;
1118 map<string, StringVector> Redirected;
1119
1120 signal(SIGTERM,SigTerm);
1121 signal(SIGINT,SigTerm);
1122
1123 Server = 0;
1124
1125 int FailCounter = 0;
1126 while (1)
1127 {
1128 // We have no commands, wait for some to arrive
1129 if (Queue == 0)
1130 {
1131 if (WaitFd(STDIN_FILENO) == false)
1132 return 0;
1133 }
1134
1135 /* Run messages, we can accept 0 (no message) if we didn't
1136 do a WaitFd above.. Otherwise the FD is closed. */
1137 int Result = Run(true);
1138 if (Result != -1 && (Result != 0 || Queue == 0))
1139 {
1140 if(FailReason.empty() == false ||
1141 _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
1142 return 100;
1143 else
1144 return 0;
1145 }
1146
1147 if (Queue == 0)
1148 continue;
1149
1150 // Connect to the server
1151 if (Server == 0 || Server->Comp(Queue->Uri) == false)
1152 {
1153 delete Server;
1154 Server = new ServerState(Queue->Uri,this);
1155 }
1156 /* If the server has explicitly said this is the last connection
1157 then we pre-emptively shut down the pipeline and tear down
1158 the connection. This will speed up HTTP/1.0 servers a tad
1159 since we don't have to wait for the close sequence to
1160 complete */
1161 if (Server->Persistent == false)
1162 Server->Close();
1163
1164 // Reset the pipeline
1165 if (Server->ServerFd == -1)
1166 QueueBack = Queue;
1167
1168 // Connnect to the host
1169 if (Server->Open() == false)
1170 {
1171 Fail(true);
1172 delete Server;
1173 Server = 0;
1174 continue;
1175 }
1176
1177 // Fill the pipeline.
1178 Fetch(0);
1179
1180 // Fetch the next URL header data from the server.
1181 switch (Server->RunHeaders())
1182 {
1183 case ServerState::RUN_HEADERS_OK:
1184 break;
1185
1186 // The header data is bad
1187 case ServerState::RUN_HEADERS_PARSE_ERROR:
1188 {
1189 _error->Error(_("Bad header data"));
1190 Fail(true);
1191 RotateDNS();
1192 continue;
1193 }
1194
1195 // The server closed a connection during the header get..
1196 default:
1197 case ServerState::RUN_HEADERS_IO_ERROR:
1198 {
1199 FailCounter++;
1200 _error->Discard();
1201 Server->Close();
1202 Server->Pipeline = false;
1203
1204 if (FailCounter >= 2)
1205 {
1206 Fail(_("Connection failed"),true);
1207 FailCounter = 0;
1208 }
1209
1210 RotateDNS();
1211 continue;
1212 }
1213 };
1214
1215 // Decide what to do.
1216 FetchResult Res;
1217 Res.Filename = Queue->DestFile;
1218 switch (DealWithHeaders(Res,Server))
1219 {
1220 // Ok, the file is Open
1221 case FILE_IS_OPEN:
1222 {
1223 URIStart(Res);
1224
1225 // Run the data
1226 bool Result = Server->RunData();
1227
1228 /* If the server is sending back sizeless responses then fill in
1229 the size now */
1230 if (Res.Size == 0)
1231 Res.Size = File->Size();
1232
1233 // Close the file, destroy the FD object and timestamp it
1234 FailFd = -1;
1235 delete File;
1236 File = 0;
1237
1238 // Timestamp
1239 struct utimbuf UBuf;
1240 time(&UBuf.actime);
1241 UBuf.actime = Server->Date;
1242 UBuf.modtime = Server->Date;
1243 utime(Queue->DestFile.c_str(),&UBuf);
1244
1245 // Send status to APT
1246 if (Result == true)
1247 {
1248 Res.TakeHashes(*Server->In.Hash);
1249 URIDone(Res);
1250 }
1251 else
1252 {
1253 if (Server->ServerFd == -1)
1254 {
1255 FailCounter++;
1256 _error->Discard();
1257 Server->Close();
1258
1259 if (FailCounter >= 2)
1260 {
1261 Fail(_("Connection failed"),true);
1262 FailCounter = 0;
1263 }
1264
1265 QueueBack = Queue;
1266 }
1267 else
1268 Fail(true);
1269 }
1270 break;
1271 }
1272
1273 // IMS hit
1274 case IMS_HIT:
1275 {
1276 URIDone(Res);
1277 break;
1278 }
1279
1280 // Hard server error, not found or something
1281 case ERROR_UNRECOVERABLE:
1282 {
1283 Fail();
1284 break;
1285 }
1286
1287 // Hard internal error, kill the connection and fail
1288 case ERROR_NOT_FROM_SERVER:
1289 {
1290 delete File;
1291 File = 0;
1292
1293 Fail();
1294 RotateDNS();
1295 Server->Close();
1296 break;
1297 }
1298
1299 // We need to flush the data, the header is like a 404 w/ error text
1300 case ERROR_WITH_CONTENT_PAGE:
1301 {
1302 Fail();
1303
1304 // Send to content to dev/null
1305 File = new FileFd("/dev/null",FileFd::WriteExists);
1306 Server->RunData();
1307 delete File;
1308 File = 0;
1309 break;
1310 }
1311
1312 // Try again with a new URL
1313 case TRY_AGAIN_OR_REDIRECT:
1314 {
1315 // Clear rest of response if there is content
1316 if (Server->HaveContent)
1317 {
1318 File = new FileFd("/dev/null",FileFd::WriteExists);
1319 Server->RunData();
1320 delete File;
1321 File = 0;
1322 }
1323
1324 /* Detect redirect loops. No more redirects are allowed
1325 after the same URI is seen twice in a queue item. */
1326 StringVector &R = Redirected[Queue->DestFile];
1327 bool StopRedirects = false;
1328 if (R.size() == 0)
1329 R.push_back(Queue->Uri);
1330 else if (R[0] == "STOP" || R.size() > 10)
1331 StopRedirects = true;
1332 else
1333 {
1334 for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
1335 if (Queue->Uri == *I)
1336 {
1337 R[0] = "STOP";
1338 break;
1339 }
1340
1341 R.push_back(Queue->Uri);
1342 }
1343
1344 if (StopRedirects == false)
1345 Redirect(NextURI);
1346 else
1347 Fail();
1348
1349 break;
1350 }
1351
1352 default:
1353 Fail(_("Internal error"));
1354 break;
1355 }
1356
1357 FailCounter = 0;
1358 }
1359
1360 return 0;
1361 }
1362 /*}}}*/
1363 // HttpMethod::AutoDetectProxy - auto detect proxy /*{{{*/
1364 // ---------------------------------------------------------------------
1365 /* */
1366 bool HttpMethod::AutoDetectProxy()
1367 {
1368 if (AutoDetectProxyCmd.empty())
1369 return true;
1370
1371 if (Debug)
1372 clog << "Using auto proxy detect command: " << AutoDetectProxyCmd << endl;
1373
1374 int Pipes[2] = {-1,-1};
1375 if (pipe(Pipes) != 0)
1376 return _error->Errno("pipe", "Failed to create Pipe");
1377
1378 pid_t Process = ExecFork();
1379 if (Process == 0)
1380 {
1381 close(Pipes[0]);
1382 dup2(Pipes[1],STDOUT_FILENO);
1383 SetCloseExec(STDOUT_FILENO,false);
1384
1385 const char *Args[2];
1386 Args[0] = AutoDetectProxyCmd.c_str();
1387 Args[1] = 0;
1388 execv(Args[0],(char **)Args);
1389 cerr << "Failed to exec method " << Args[0] << endl;
1390 _exit(100);
1391 }
1392 char buf[512];
1393 int InFd = Pipes[0];
1394 close(Pipes[1]);
1395 int res = read(InFd, buf, sizeof(buf));
1396 ExecWait(Process, "ProxyAutoDetect", true);
1397
1398 if (res < 0)
1399 return _error->Errno("read", "Failed to read");
1400 if (res == 0)
1401 return _error->Warning("ProxyAutoDetect returned no data");
1402
1403 // add trailing \0
1404 buf[res] = 0;
1405
1406 if (Debug)
1407 clog << "auto detect command returned: '" << buf << "'" << endl;
1408
1409 if (strstr(buf, "http://") == buf)
1410 _config->Set("Acquire::http::proxy", _strstrip(buf));
1411
1412 return true;
1413 }
1414 /*}}}*/
1415
1416