]> git.saurik.com Git - apt-legacy.git/blob - methods/http.cc
Fixed If-Last-Modified support in APT to be correct.
[apt-legacy.git] / methods / http.cc
1 extern "C" {
2 #include <mach-o/nlist.h>
3 }
4
5 // -*- mode: cpp; mode: fold -*-
6 // Description /*{{{*/
7 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
8 /* ######################################################################
9
10 HTTP Aquire Method - This is the HTTP aquire method for APT.
11
12 It uses HTTP/1.1 and many of the fancy options there-in, such as
13 pipelining, range, if-range and so on.
14
15 It is based on a doubly buffered select loop. A groupe of requests are
16 fed into a single output buffer that is constantly fed out the
17 socket. This provides ideal pipelining as in many cases all of the
18 requests will fit into a single packet. The input socket is buffered
19 the same way and fed into the fd for the file (may be a pipe in future).
20
21 This double buffering provides fairly substantial transfer rates,
22 compared to wget the http method is about 4% faster. Most importantly,
23 when HTTP is compared with FTP as a protocol the speed difference is
24 huge. In tests over the internet from two sites to llug (via ATM) this
25 program got 230k/s sustained http transfer rates. FTP on the other
26 hand topped out at 170k/s. That combined with the time to setup the
27 FTP connection makes HTTP a vastly superior protocol.
28
29 ##################################################################### */
30 /*}}}*/
31 // Include Files /*{{{*/
32 #include <apt-pkg/fileutl.h>
33 #include <apt-pkg/acquire-method.h>
34 #include <apt-pkg/error.h>
35 #include <apt-pkg/hashes.h>
36
37 #include <sys/sysctl.h>
38 #include <sys/stat.h>
39 #include <sys/time.h>
40 #include <utime.h>
41 #include <unistd.h>
42 #include <signal.h>
43 #include <stdio.h>
44 #include <errno.h>
45 #include <string.h>
46 #include <iostream>
47 #include <apti18n.h>
48
49 // Internet stuff
50 #include <netdb.h>
51 #include <arpa/inet.h>
52
53 #include <CoreFoundation/CoreFoundation.h>
54 #include <CoreServices/CoreServices.h>
55 #include <SystemConfiguration/SystemConfiguration.h>
56
57 #include "connect.h"
58 #include "rfc2553emu.h"
59 #include "http.h"
60
61 /*}}}*/
62 using namespace std;
63
64 CFStringRef Firmware_;
65 const char *Machine_;
66 const char *SerialNumber_;
67
68 void CfrsError(const char *name, CFReadStreamRef rs) {
69 CFStreamError se = CFReadStreamGetError(rs);
70
71 if (se.domain == kCFStreamErrorDomainCustom) {
72 } else if (se.domain == kCFStreamErrorDomainPOSIX) {
73 _error->Error("POSIX: %s", strerror(se.error));
74 } else if (se.domain == kCFStreamErrorDomainMacOSStatus) {
75 _error->Error("MacOSStatus: %ld", se.error);
76 } else if (se.domain == kCFStreamErrorDomainNetDB) {
77 _error->Error("NetDB: %s %s", name, gai_strerror(se.error));
78 } else if (se.domain == kCFStreamErrorDomainMach) {
79 _error->Error("Mach: %ld", se.error);
80 } else if (se.domain == kCFStreamErrorDomainHTTP) {
81 switch (se.error) {
82 case kCFStreamErrorHTTPParseFailure:
83 _error->Error("Parse failure");
84 break;
85
86 case kCFStreamErrorHTTPRedirectionLoop:
87 _error->Error("Redirection loop");
88 break;
89
90 case kCFStreamErrorHTTPBadURL:
91 _error->Error("Bad URL");
92 break;
93
94 default:
95 _error->Error("Unknown HTTP error: %ld", se.error);
96 break;
97 }
98 } else if (se.domain == kCFStreamErrorDomainSOCKS) {
99 _error->Error("SOCKS: %ld", se.error);
100 } else if (se.domain == kCFStreamErrorDomainSystemConfiguration) {
101 _error->Error("SystemConfiguration: %ld", se.error);
102 } else if (se.domain == kCFStreamErrorDomainSSL) {
103 _error->Error("SSL: %ld", se.error);
104 } else {
105 _error->Error("Domain #%ld: %ld", se.domain, se.error);
106 }
107 }
108
109 string HttpMethod::FailFile;
110 int HttpMethod::FailFd = -1;
111 time_t HttpMethod::FailTime = 0;
112 unsigned long PipelineDepth = 10;
113 unsigned long TimeOut = 120;
114 bool Debug = false;
115
116 unsigned long CircleBuf::BwReadLimit=0;
117 unsigned long CircleBuf::BwTickReadData=0;
118 struct timeval CircleBuf::BwReadTick={0,0};
119 const unsigned int CircleBuf::BW_HZ=10;
120
121 // CircleBuf::CircleBuf - Circular input buffer /*{{{*/
122 // ---------------------------------------------------------------------
123 /* */
124 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
125 {
126 Buf = new unsigned char[Size];
127 Reset();
128
129 CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
130 }
131 /*}}}*/
132 // CircleBuf::Reset - Reset to the default state /*{{{*/
133 // ---------------------------------------------------------------------
134 /* */
135 void CircleBuf::Reset()
136 {
137 InP = 0;
138 OutP = 0;
139 StrPos = 0;
140 MaxGet = (unsigned int)-1;
141 OutQueue = string();
142 if (Hash != 0)
143 {
144 delete Hash;
145 Hash = new Hashes;
146 }
147 };
148 /*}}}*/
149 // CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
150 // ---------------------------------------------------------------------
151 /* This fills up the buffer with as much data as is in the FD, assuming it
152 is non-blocking.. */
153 bool CircleBuf::Read(int Fd)
154 {
155 unsigned long BwReadMax;
156
157 while (1)
158 {
159 // Woops, buffer is full
160 if (InP - OutP == Size)
161 return true;
162
163 // what's left to read in this tick
164 BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
165
166 if(CircleBuf::BwReadLimit) {
167 struct timeval now;
168 gettimeofday(&now,0);
169
170 unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
171 now.tv_usec-CircleBuf::BwReadTick.tv_usec;
172 if(d > 1000000/BW_HZ) {
173 CircleBuf::BwReadTick = now;
174 CircleBuf::BwTickReadData = 0;
175 }
176
177 if(CircleBuf::BwTickReadData >= BwReadMax) {
178 usleep(1000000/BW_HZ);
179 return true;
180 }
181 }
182
183 // Write the buffer segment
184 int Res;
185 if(CircleBuf::BwReadLimit) {
186 Res = read(Fd,Buf + (InP%Size),
187 BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
188 } else
189 Res = read(Fd,Buf + (InP%Size),LeftRead());
190
191 if(Res > 0 && BwReadLimit > 0)
192 CircleBuf::BwTickReadData += Res;
193
194 if (Res == 0)
195 return false;
196 if (Res < 0)
197 {
198 if (errno == EAGAIN)
199 return true;
200 return false;
201 }
202
203 if (InP == 0)
204 gettimeofday(&Start,0);
205 InP += Res;
206 }
207 }
208 /*}}}*/
209 // CircleBuf::Read - Put the string into the buffer /*{{{*/
210 // ---------------------------------------------------------------------
211 /* This will hold the string in and fill the buffer with it as it empties */
212 bool CircleBuf::Read(string Data)
213 {
214 OutQueue += Data;
215 FillOut();
216 return true;
217 }
218 /*}}}*/
219 // CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
220 // ---------------------------------------------------------------------
221 /* */
222 void CircleBuf::FillOut()
223 {
224 if (OutQueue.empty() == true)
225 return;
226 while (1)
227 {
228 // Woops, buffer is full
229 if (InP - OutP == Size)
230 return;
231
232 // Write the buffer segment
233 unsigned long Sz = LeftRead();
234 if (OutQueue.length() - StrPos < Sz)
235 Sz = OutQueue.length() - StrPos;
236 memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
237
238 // Advance
239 StrPos += Sz;
240 InP += Sz;
241 if (OutQueue.length() == StrPos)
242 {
243 StrPos = 0;
244 OutQueue = "";
245 return;
246 }
247 }
248 }
249 /*}}}*/
250 // CircleBuf::Write - Write from the buffer into a FD /*{{{*/
251 // ---------------------------------------------------------------------
252 /* This empties the buffer into the FD. */
253 bool CircleBuf::Write(int Fd)
254 {
255 while (1)
256 {
257 FillOut();
258
259 // Woops, buffer is empty
260 if (OutP == InP)
261 return true;
262
263 if (OutP == MaxGet)
264 return true;
265
266 // Write the buffer segment
267 int Res;
268 Res = write(Fd,Buf + (OutP%Size),LeftWrite());
269
270 if (Res == 0)
271 return false;
272 if (Res < 0)
273 {
274 if (errno == EAGAIN)
275 return true;
276
277 return false;
278 }
279
280 if (Hash != 0)
281 Hash->Add(Buf + (OutP%Size),Res);
282
283 OutP += Res;
284 }
285 }
286 /*}}}*/
287 // CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
288 // ---------------------------------------------------------------------
289 /* This copies till the first empty line */
290 bool CircleBuf::WriteTillEl(string &Data,bool Single)
291 {
292 // We cheat and assume it is unneeded to have more than one buffer load
293 for (unsigned long I = OutP; I < InP; I++)
294 {
295 if (Buf[I%Size] != '\n')
296 continue;
297 ++I;
298
299 if (Single == false)
300 {
301 if (I < InP && Buf[I%Size] == '\r')
302 ++I;
303 if (I >= InP || Buf[I%Size] != '\n')
304 continue;
305 ++I;
306 }
307
308 Data = "";
309 while (OutP < I)
310 {
311 unsigned long Sz = LeftWrite();
312 if (Sz == 0)
313 return false;
314 if (I - OutP < Sz)
315 Sz = I - OutP;
316 Data += string((char *)(Buf + (OutP%Size)),Sz);
317 OutP += Sz;
318 }
319 return true;
320 }
321 return false;
322 }
323 /*}}}*/
324 // CircleBuf::Stats - Print out stats information /*{{{*/
325 // ---------------------------------------------------------------------
326 /* */
327 void CircleBuf::Stats()
328 {
329 if (InP == 0)
330 return;
331
332 struct timeval Stop;
333 gettimeofday(&Stop,0);
334 /* float Diff = Stop.tv_sec - Start.tv_sec +
335 (float)(Stop.tv_usec - Start.tv_usec)/1000000;
336 clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
337 }
338 /*}}}*/
339
340 // ServerState::ServerState - Constructor /*{{{*/
341 // ---------------------------------------------------------------------
342 /* */
343 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
344 In(64*1024), Out(4*1024),
345 ServerName(Srv)
346 {
347 Reset();
348 }
349 /*}}}*/
350 // ServerState::Open - Open a connection to the server /*{{{*/
351 // ---------------------------------------------------------------------
352 /* This opens a connection to the server. */
353 bool ServerState::Open()
354 {
355 // Use the already open connection if possible.
356 if (ServerFd != -1)
357 return true;
358
359 Close();
360 In.Reset();
361 Out.Reset();
362 Persistent = true;
363
364 // Determine the proxy setting
365 if (getenv("http_proxy") == 0)
366 {
367 string DefProxy = _config->Find("Acquire::http::Proxy");
368 string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
369 if (SpecificProxy.empty() == false)
370 {
371 if (SpecificProxy == "DIRECT")
372 Proxy = "";
373 else
374 Proxy = SpecificProxy;
375 }
376 else
377 Proxy = DefProxy;
378 }
379 else
380 Proxy = getenv("http_proxy");
381
382 // Parse no_proxy, a , separated list of domains
383 if (getenv("no_proxy") != 0)
384 {
385 if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
386 Proxy = "";
387 }
388
389 // Determine what host and port to use based on the proxy settings
390 int Port = 0;
391 string Host;
392 if (Proxy.empty() == true || Proxy.Host.empty() == true)
393 {
394 if (ServerName.Port != 0)
395 Port = ServerName.Port;
396 Host = ServerName.Host;
397 }
398 else
399 {
400 if (Proxy.Port != 0)
401 Port = Proxy.Port;
402 Host = Proxy.Host;
403 }
404
405 // Connect to the remote server
406 if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
407 return false;
408
409 return true;
410 }
411 /*}}}*/
412 // ServerState::Close - Close a connection to the server /*{{{*/
413 // ---------------------------------------------------------------------
414 /* */
415 bool ServerState::Close()
416 {
417 close(ServerFd);
418 ServerFd = -1;
419 return true;
420 }
421 /*}}}*/
422 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
423 // ---------------------------------------------------------------------
424 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
425 parse error occured */
426 int ServerState::RunHeaders()
427 {
428 State = Header;
429
430 Owner->Status(_("Waiting for headers"));
431
432 Major = 0;
433 Minor = 0;
434 Result = 0;
435 Size = 0;
436 StartPos = 0;
437 Encoding = Closes;
438 HaveContent = false;
439 time(&Date);
440
441 do
442 {
443 string Data;
444 if (In.WriteTillEl(Data) == false)
445 continue;
446
447 if (Debug == true)
448 clog << Data;
449
450 for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
451 {
452 string::const_iterator J = I;
453 for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
454 if (HeaderLine(string(I,J)) == false)
455 return 2;
456 I = J;
457 }
458
459 // 100 Continue is a Nop...
460 if (Result == 100)
461 continue;
462
463 // Tidy up the connection persistance state.
464 if (Encoding == Closes && HaveContent == true)
465 Persistent = false;
466
467 return 0;
468 }
469 while (Owner->Go(false,this) == true);
470
471 return 1;
472 }
473 /*}}}*/
474 // ServerState::RunData - Transfer the data from the socket /*{{{*/
475 // ---------------------------------------------------------------------
476 /* */
477 bool ServerState::RunData()
478 {
479 State = Data;
480
481 // Chunked transfer encoding is fun..
482 if (Encoding == Chunked)
483 {
484 while (1)
485 {
486 // Grab the block size
487 bool Last = true;
488 string Data;
489 In.Limit(-1);
490 do
491 {
492 if (In.WriteTillEl(Data,true) == true)
493 break;
494 }
495 while ((Last = Owner->Go(false,this)) == true);
496
497 if (Last == false)
498 return false;
499
500 // See if we are done
501 unsigned long Len = strtol(Data.c_str(),0,16);
502 if (Len == 0)
503 {
504 In.Limit(-1);
505
506 // We have to remove the entity trailer
507 Last = true;
508 do
509 {
510 if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
511 break;
512 }
513 while ((Last = Owner->Go(false,this)) == true);
514 if (Last == false)
515 return false;
516 return !_error->PendingError();
517 }
518
519 // Transfer the block
520 In.Limit(Len);
521 while (Owner->Go(true,this) == true)
522 if (In.IsLimit() == true)
523 break;
524
525 // Error
526 if (In.IsLimit() == false)
527 return false;
528
529 // The server sends an extra new line before the next block specifier..
530 In.Limit(-1);
531 Last = true;
532 do
533 {
534 if (In.WriteTillEl(Data,true) == true)
535 break;
536 }
537 while ((Last = Owner->Go(false,this)) == true);
538 if (Last == false)
539 return false;
540 }
541 }
542 else
543 {
544 /* Closes encoding is used when the server did not specify a size, the
545 loss of the connection means we are done */
546 if (Encoding == Closes)
547 In.Limit(-1);
548 else
549 In.Limit(Size - StartPos);
550
551 // Just transfer the whole block.
552 do
553 {
554 if (In.IsLimit() == false)
555 continue;
556
557 In.Limit(-1);
558 return !_error->PendingError();
559 }
560 while (Owner->Go(true,this) == true);
561 }
562
563 return Owner->Flush(this) && !_error->PendingError();
564 }
565 /*}}}*/
566 // ServerState::HeaderLine - Process a header line /*{{{*/
567 // ---------------------------------------------------------------------
568 /* */
569 bool ServerState::HeaderLine(string Line)
570 {
571 if (Line.empty() == true)
572 return true;
573
574 // The http server might be trying to do something evil.
575 if (Line.length() >= MAXLEN)
576 return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
577
578 string::size_type Pos = Line.find(' ');
579 if (Pos == string::npos || Pos+1 > Line.length())
580 {
581 // Blah, some servers use "connection:closes", evil.
582 Pos = Line.find(':');
583 if (Pos == string::npos || Pos + 2 > Line.length())
584 return _error->Error(_("Bad header line"));
585 Pos++;
586 }
587
588 // Parse off any trailing spaces between the : and the next word.
589 string::size_type Pos2 = Pos;
590 while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
591 Pos2++;
592
593 string Tag = string(Line,0,Pos);
594 string Val = string(Line,Pos2);
595
596 if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
597 {
598 // Evil servers return no version
599 if (Line[4] == '/')
600 {
601 if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
602 &Result,Code) != 4)
603 return _error->Error(_("The HTTP server sent an invalid reply header"));
604 }
605 else
606 {
607 Major = 0;
608 Minor = 9;
609 if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
610 return _error->Error(_("The HTTP server sent an invalid reply header"));
611 }
612
613 /* Check the HTTP response header to get the default persistance
614 state. */
615 if (Major < 1)
616 Persistent = false;
617 else
618 {
619 if (Major == 1 && Minor <= 0)
620 Persistent = false;
621 else
622 Persistent = true;
623 }
624
625 return true;
626 }
627
628 if (stringcasecmp(Tag,"Content-Length:") == 0)
629 {
630 if (Encoding == Closes)
631 Encoding = Stream;
632 HaveContent = true;
633
634 // The length is already set from the Content-Range header
635 if (StartPos != 0)
636 return true;
637
638 if (sscanf(Val.c_str(),"%lu",&Size) != 1)
639 return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
640 return true;
641 }
642
643 if (stringcasecmp(Tag,"Content-Type:") == 0)
644 {
645 HaveContent = true;
646 return true;
647 }
648
649 if (stringcasecmp(Tag,"Content-Range:") == 0)
650 {
651 HaveContent = true;
652
653 if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
654 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
655 if ((unsigned)StartPos > Size)
656 return _error->Error(_("This HTTP server has broken range support"));
657 return true;
658 }
659
660 if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
661 {
662 HaveContent = true;
663 if (stringcasecmp(Val,"chunked") == 0)
664 Encoding = Chunked;
665 return true;
666 }
667
668 if (stringcasecmp(Tag,"Connection:") == 0)
669 {
670 if (stringcasecmp(Val,"close") == 0)
671 Persistent = false;
672 if (stringcasecmp(Val,"keep-alive") == 0)
673 Persistent = true;
674 return true;
675 }
676
677 if (stringcasecmp(Tag,"Last-Modified:") == 0)
678 {
679 if (StrToTime(Val,Date) == false)
680 return _error->Error(_("Unknown date format"));
681 return true;
682 }
683
684 return true;
685 }
686 /*}}}*/
687
688 // HttpMethod::SendReq - Send the HTTP request /*{{{*/
689 // ---------------------------------------------------------------------
690 /* This places the http request in the outbound buffer */
691 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
692 {
693 URI Uri = Itm->Uri;
694
695 // The HTTP server expects a hostname with a trailing :port
696 char Buf[1000];
697 string ProperHost = Uri.Host;
698 if (Uri.Port != 0)
699 {
700 sprintf(Buf,":%u",Uri.Port);
701 ProperHost += Buf;
702 }
703
704 // Just in case.
705 if (Itm->Uri.length() >= sizeof(Buf))
706 abort();
707
708 /* Build the request. We include a keep-alive header only for non-proxy
709 requests. This is to tweak old http/1.0 servers that do support keep-alive
710 but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
711 will glitch HTTP/1.0 proxies because they do not filter it out and
712 pass it on, HTTP/1.1 says the connection should default to keep alive
713 and we expect the proxy to do this */
714 if (Proxy.empty() == true || Proxy.Host.empty())
715 sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
716 QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
717 else
718 {
719 /* Generate a cache control header if necessary. We place a max
720 cache age on index files, optionally set a no-cache directive
721 and a no-store directive for archives. */
722 sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
723 Itm->Uri.c_str(),ProperHost.c_str());
724 // only generate a cache control header if we actually want to
725 // use a cache
726 if (_config->FindB("Acquire::http::No-Cache",false) == false)
727 {
728 if (Itm->IndexFile == true)
729 sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
730 _config->FindI("Acquire::http::Max-Age",0));
731 else
732 {
733 if (_config->FindB("Acquire::http::No-Store",false) == true)
734 strcat(Buf,"Cache-Control: no-store\r\n");
735 }
736 }
737 }
738 // generate a no-cache header if needed
739 if (_config->FindB("Acquire::http::No-Cache",false) == true)
740 strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
741
742
743 string Req = Buf;
744
745 // Check for a partial file
746 struct stat SBuf;
747 if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
748 {
749 // In this case we send an if-range query with a range header
750 sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
751 TimeRFC1123(SBuf.st_mtime).c_str());
752 Req += Buf;
753 }
754 else
755 {
756 if (Itm->LastModified != 0)
757 {
758 sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
759 Req += Buf;
760 }
761 }
762
763 if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
764 Req += string("Proxy-Authorization: Basic ") +
765 Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
766
767 if (Uri.User.empty() == false || Uri.Password.empty() == false)
768 Req += string("Authorization: Basic ") +
769 Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
770
771 Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
772
773 if (Debug == true)
774 cerr << Req << endl;
775
776 Out.Read(Req);
777 }
778 /*}}}*/
779 // HttpMethod::Go - Run a single loop /*{{{*/
780 // ---------------------------------------------------------------------
781 /* This runs the select loop over the server FDs, Output file FDs and
782 stdin. */
783 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
784 {
785 // Server has closed the connection
786 if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
787 ToFile == false))
788 return false;
789
790 fd_set rfds,wfds;
791 FD_ZERO(&rfds);
792 FD_ZERO(&wfds);
793
794 /* Add the server. We only send more requests if the connection will
795 be persisting */
796 if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
797 && Srv->Persistent == true)
798 FD_SET(Srv->ServerFd,&wfds);
799 if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
800 FD_SET(Srv->ServerFd,&rfds);
801
802 // Add the file
803 int FileFD = -1;
804 if (File != 0)
805 FileFD = File->Fd();
806
807 if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
808 FD_SET(FileFD,&wfds);
809
810 // Add stdin
811 FD_SET(STDIN_FILENO,&rfds);
812
813 // Figure out the max fd
814 int MaxFd = FileFD;
815 if (MaxFd < Srv->ServerFd)
816 MaxFd = Srv->ServerFd;
817
818 // Select
819 struct timeval tv;
820 tv.tv_sec = TimeOut;
821 tv.tv_usec = 0;
822 int Res = 0;
823 if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
824 {
825 if (errno == EINTR)
826 return true;
827 return _error->Errno("select",_("Select failed"));
828 }
829
830 if (Res == 0)
831 {
832 _error->Error(_("Connection timed out"));
833 return ServerDie(Srv);
834 }
835
836 // Handle server IO
837 if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
838 {
839 errno = 0;
840 if (Srv->In.Read(Srv->ServerFd) == false)
841 return ServerDie(Srv);
842 }
843
844 if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
845 {
846 errno = 0;
847 if (Srv->Out.Write(Srv->ServerFd) == false)
848 return ServerDie(Srv);
849 }
850
851 // Send data to the file
852 if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
853 {
854 if (Srv->In.Write(FileFD) == false)
855 return _error->Errno("write",_("Error writing to output file"));
856 }
857
858 // Handle commands from APT
859 if (FD_ISSET(STDIN_FILENO,&rfds))
860 {
861 if (Run(true) != -1)
862 exit(100);
863 }
864
865 return true;
866 }
867 /*}}}*/
868 // HttpMethod::Flush - Dump the buffer into the file /*{{{*/
869 // ---------------------------------------------------------------------
870 /* This takes the current input buffer from the Server FD and writes it
871 into the file */
872 bool HttpMethod::Flush(ServerState *Srv)
873 {
874 if (File != 0)
875 {
876 // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
877 // can't be set
878 if (File->Name() != "/dev/null")
879 SetNonBlock(File->Fd(),false);
880 if (Srv->In.WriteSpace() == false)
881 return true;
882
883 while (Srv->In.WriteSpace() == true)
884 {
885 if (Srv->In.Write(File->Fd()) == false)
886 return _error->Errno("write",_("Error writing to file"));
887 if (Srv->In.IsLimit() == true)
888 return true;
889 }
890
891 if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
892 return true;
893 }
894 return false;
895 }
896 /*}}}*/
897 // HttpMethod::ServerDie - The server has closed the connection. /*{{{*/
898 // ---------------------------------------------------------------------
899 /* */
900 bool HttpMethod::ServerDie(ServerState *Srv)
901 {
902 unsigned int LErrno = errno;
903
904 // Dump the buffer to the file
905 if (Srv->State == ServerState::Data)
906 {
907 // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
908 // can't be set
909 if (File->Name() != "/dev/null")
910 SetNonBlock(File->Fd(),false);
911 while (Srv->In.WriteSpace() == true)
912 {
913 if (Srv->In.Write(File->Fd()) == false)
914 return _error->Errno("write",_("Error writing to the file"));
915
916 // Done
917 if (Srv->In.IsLimit() == true)
918 return true;
919 }
920 }
921
922 // See if this is because the server finished the data stream
923 if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
924 Srv->Encoding != ServerState::Closes)
925 {
926 Srv->Close();
927 if (LErrno == 0)
928 return _error->Error(_("Error reading from server. Remote end closed connection"));
929 errno = LErrno;
930 return _error->Errno("read",_("Error reading from server"));
931 }
932 else
933 {
934 Srv->In.Limit(-1);
935
936 // Nothing left in the buffer
937 if (Srv->In.WriteSpace() == false)
938 return false;
939
940 // We may have got multiple responses back in one packet..
941 Srv->Close();
942 return true;
943 }
944
945 return false;
946 }
947 /*}}}*/
948 // HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
949 // ---------------------------------------------------------------------
950 /* We look at the header data we got back from the server and decide what
951 to do. Returns
952 0 - File is open,
953 1 - IMS hit
954 3 - Unrecoverable error
955 4 - Error with error content page
956 5 - Unrecoverable non-server error (close the connection) */
957 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
958 {
959 // Not Modified
960 if (Srv->Result == 304)
961 {
962 unlink(Queue->DestFile.c_str());
963 Res.IMSHit = true;
964 Res.LastModified = Queue->LastModified;
965 return 1;
966 }
967
968 /* We have a reply we dont handle. This should indicate a perm server
969 failure */
970 if (Srv->Result < 200 || Srv->Result >= 300)
971 {
972 _error->Error("%u %s",Srv->Result,Srv->Code);
973 if (Srv->HaveContent == true)
974 return 4;
975 return 3;
976 }
977
978 // This is some sort of 2xx 'data follows' reply
979 Res.LastModified = Srv->Date;
980 Res.Size = Srv->Size;
981
982 // Open the file
983 delete File;
984 File = new FileFd(Queue->DestFile,FileFd::WriteAny);
985 if (_error->PendingError() == true)
986 return 5;
987
988 FailFile = Queue->DestFile;
989 FailFile.c_str(); // Make sure we dont do a malloc in the signal handler
990 FailFd = File->Fd();
991 FailTime = Srv->Date;
992
993 // Set the expected size
994 if (Srv->StartPos >= 0)
995 {
996 Res.ResumePoint = Srv->StartPos;
997 ftruncate(File->Fd(),Srv->StartPos);
998 }
999
1000 // Set the start point
1001 lseek(File->Fd(),0,SEEK_END);
1002
1003 delete Srv->In.Hash;
1004 Srv->In.Hash = new Hashes;
1005
1006 // Fill the Hash if the file is non-empty (resume)
1007 if (Srv->StartPos > 0)
1008 {
1009 lseek(File->Fd(),0,SEEK_SET);
1010 if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
1011 {
1012 _error->Errno("read",_("Problem hashing file"));
1013 return 5;
1014 }
1015 lseek(File->Fd(),0,SEEK_END);
1016 }
1017
1018 SetNonBlock(File->Fd(),true);
1019 return 0;
1020 }
1021 /*}}}*/
1022 // HttpMethod::SigTerm - Handle a fatal signal /*{{{*/
1023 // ---------------------------------------------------------------------
1024 /* This closes and timestamps the open file. This is neccessary to get
1025 resume behavoir on user abort */
1026 void HttpMethod::SigTerm(int)
1027 {
1028 if (FailFd == -1)
1029 _exit(100);
1030 close(FailFd);
1031
1032 // Timestamp
1033 struct utimbuf UBuf;
1034 UBuf.actime = FailTime;
1035 UBuf.modtime = FailTime;
1036 utime(FailFile.c_str(),&UBuf);
1037
1038 _exit(100);
1039 }
1040 /*}}}*/
1041 // HttpMethod::Fetch - Fetch an item /*{{{*/
1042 // ---------------------------------------------------------------------
1043 /* This adds an item to the pipeline. We keep the pipeline at a fixed
1044 depth. */
1045 bool HttpMethod::Fetch(FetchItem *)
1046 {
1047 if (Server == 0)
1048 return true;
1049
1050 // Queue the requests
1051 int Depth = -1;
1052 bool Tail = false;
1053 for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
1054 I = I->Next, Depth++)
1055 {
1056 // If pipelining is disabled, we only queue 1 request
1057 if (Server->Pipeline == false && Depth >= 0)
1058 break;
1059
1060 // Make sure we stick with the same server
1061 if (Server->Comp(I->Uri) == false)
1062 break;
1063 if (QueueBack == I)
1064 Tail = true;
1065 if (Tail == true)
1066 {
1067 QueueBack = I->Next;
1068 SendReq(I,Server->Out);
1069 continue;
1070 }
1071 }
1072
1073 return true;
1074 };
1075 /*}}}*/
1076 // HttpMethod::Configuration - Handle a configuration message /*{{{*/
1077 // ---------------------------------------------------------------------
1078 /* We stash the desired pipeline depth */
1079 bool HttpMethod::Configuration(string Message)
1080 {
1081 if (pkgAcqMethod::Configuration(Message) == false)
1082 return false;
1083
1084 TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1085 PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1086 PipelineDepth);
1087 Debug = _config->FindB("Debug::Acquire::http",false);
1088
1089 return true;
1090 }
1091 /*}}}*/
1092 // HttpMethod::Loop - Main loop /*{{{*/
1093 // ---------------------------------------------------------------------
1094 /* */
1095 int HttpMethod::Loop()
1096 {
1097 signal(SIGTERM,SigTerm);
1098 signal(SIGINT,SigTerm);
1099
1100 Server = 0;
1101
1102 int FailCounter = 0;
1103 while (1)
1104 {
1105 // We have no commands, wait for some to arrive
1106 if (Queue == 0)
1107 {
1108 if (WaitFd(STDIN_FILENO) == false)
1109 return 0;
1110 }
1111
1112 /* Run messages, we can accept 0 (no message) if we didn't
1113 do a WaitFd above.. Otherwise the FD is closed. */
1114 int Result = Run(true);
1115 if (Result != -1 && (Result != 0 || Queue == 0))
1116 return 100;
1117
1118 if (Queue == 0)
1119 continue;
1120
1121 CFStringEncoding se = kCFStringEncodingUTF8;
1122
1123 char *url = strdup(Queue->Uri.c_str());
1124 url:
1125 URI uri = std::string(url);
1126 std::string hs = uri.Host;
1127
1128 std::string urs = uri;
1129
1130 CFStringRef sr = CFStringCreateWithCString(kCFAllocatorDefault, urs.c_str(), se);
1131 CFURLRef ur = CFURLCreateWithString(kCFAllocatorDefault, sr, NULL);
1132 CFRelease(sr);
1133 CFHTTPMessageRef hm = CFHTTPMessageCreateRequest(kCFAllocatorDefault, CFSTR("GET"), ur, kCFHTTPVersion1_1);
1134 CFRelease(ur);
1135
1136 struct stat SBuf;
1137 if (stat(Queue->DestFile.c_str(), &SBuf) >= 0 && SBuf.st_size > 0) {
1138 sr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("bytes=%li-"), (long) SBuf.st_size - 1);
1139 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Range"), sr);
1140 CFRelease(sr);
1141
1142 sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
1143 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Range"), sr);
1144 CFRelease(sr);
1145 } else if (Queue->LastModified != 0) {
1146 sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(Queue->LastModified).c_str(), se);
1147 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Modified-Since"), sr);
1148 CFRelease(sr);
1149 }
1150
1151 if (Firmware_ != NULL)
1152 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Firmware"), Firmware_);
1153
1154 sr = CFStringCreateWithCString(kCFAllocatorDefault, Machine_, se);
1155 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Machine"), sr);
1156 CFRelease(sr);
1157
1158 sr = CFStringCreateWithCString(kCFAllocatorDefault, SerialNumber_, se);
1159 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Serial-Number"), sr);
1160 CFRelease(sr);
1161
1162 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.98"));
1163
1164 CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm);
1165 CFRelease(hm);
1166
1167 CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL);
1168 CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr);
1169 CFRelease(dr);
1170
1171 //CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPShouldAutoredirect, kCFBooleanTrue);
1172 CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPAttemptPersistentConnection, kCFBooleanTrue);
1173
1174 FetchResult Res;
1175 CFIndex rd;
1176 UInt32 sc;
1177
1178 uint8_t data[10240];
1179 size_t offset = 0;
1180
1181 Status("Connecting to %s", hs.c_str());
1182
1183 if (!CFReadStreamOpen(rs)) {
1184 CfrsError("Open", rs);
1185 Fail(true);
1186 goto done;
1187 }
1188
1189 rd = CFReadStreamRead(rs, data, sizeof(data));
1190
1191 if (rd == -1) {
1192 CfrsError(uri.Host.c_str(), rs);
1193 Fail(true);
1194 goto done;
1195 }
1196
1197 Res.Filename = Queue->DestFile;
1198
1199 hm = (CFHTTPMessageRef) CFReadStreamCopyProperty(rs, kCFStreamPropertyHTTPResponseHeader);
1200 sc = CFHTTPMessageGetResponseStatusCode(hm);
1201
1202 if (sc == 301 || sc == 302) {
1203 sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Location"));
1204 if (sr == NULL) {
1205 Fail();
1206 goto done_;
1207 } else {
1208 size_t ln = CFStringGetLength(sr) + 1;
1209 free(url);
1210 url = static_cast<char *>(malloc(ln));
1211
1212 if (!CFStringGetCString(sr, url, ln, se)) {
1213 Fail();
1214 goto done_;
1215 }
1216
1217 CFRelease(sr);
1218 goto url;
1219 }
1220 }
1221
1222 sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Range"));
1223 if (sr != NULL) {
1224 size_t ln = CFStringGetLength(sr) + 1;
1225 char cr[ln];
1226
1227 if (!CFStringGetCString(sr, cr, ln, se)) {
1228 Fail();
1229 goto done_;
1230 }
1231
1232 CFRelease(sr);
1233
1234 if (sscanf(cr, "bytes %lu-%*u/%lu", &offset, &Res.Size) != 2) {
1235 _error->Error(_("The HTTP server sent an invalid Content-Range header"));
1236 Fail();
1237 goto done_;
1238 }
1239
1240 if (offset > Res.Size) {
1241 _error->Error(_("This HTTP server has broken range support"));
1242 Fail();
1243 goto done_;
1244 }
1245 } else {
1246 sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Length"));
1247 if (sr != NULL) {
1248 Res.Size = CFStringGetIntValue(sr);
1249 CFRelease(sr);
1250 }
1251 }
1252
1253 time(&Res.LastModified);
1254
1255 sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Last-Modified"));
1256 if (sr != NULL) {
1257 size_t ln = CFStringGetLength(sr) + 1;
1258 char cr[ln];
1259
1260 if (!CFStringGetCString(sr, cr, ln, se)) {
1261 Fail();
1262 goto done_;
1263 }
1264
1265 CFRelease(sr);
1266
1267 if (!StrToTime(cr, Res.LastModified)) {
1268 _error->Error(_("Unknown date format"));
1269 Fail();
1270 goto done_;
1271 }
1272 }
1273
1274 CFRelease(hm);
1275
1276 if (sc == 304) {
1277 unlink(Queue->DestFile.c_str());
1278 Res.IMSHit = true;
1279 Res.LastModified = Queue->LastModified;
1280 URIDone(Res);
1281 } else if (sc < 200 || sc >= 300)
1282 Fail();
1283 else {
1284 Hashes hash;
1285
1286 File = new FileFd(Queue->DestFile, FileFd::WriteAny);
1287 if (_error->PendingError() == true) {
1288 delete File;
1289 File = NULL;
1290 Fail();
1291 goto done;
1292 }
1293
1294 FailFile = Queue->DestFile;
1295 FailFile.c_str(); // Make sure we dont do a malloc in the signal handler
1296 FailFd = File->Fd();
1297 FailTime = Res.LastModified;
1298
1299 Res.ResumePoint = offset;
1300 ftruncate(File->Fd(), offset);
1301
1302 if (offset != 0) {
1303 lseek(File->Fd(), 0, SEEK_SET);
1304 if (!hash.AddFD(File->Fd(), offset)) {
1305 _error->Errno("read", _("Problem hashing file"));
1306 delete File;
1307 File = NULL;
1308 Fail();
1309 goto done;
1310 }
1311 }
1312
1313 lseek(File->Fd(), 0, SEEK_END);
1314
1315 URIStart(Res);
1316
1317 read: if (rd == -1) {
1318 CfrsError("rd", rs);
1319 Fail(true);
1320 } else if (rd == 0) {
1321 if (Res.Size == 0)
1322 Res.Size = File->Size();
1323
1324 struct utimbuf UBuf;
1325 time(&UBuf.actime);
1326 UBuf.actime = Res.LastModified;
1327 UBuf.modtime = Res.LastModified;
1328 utime(Queue->DestFile.c_str(), &UBuf);
1329
1330 Res.TakeHashes(hash);
1331 URIDone(Res);
1332 } else {
1333 hash.Add(data, rd);
1334
1335 uint8_t *dt = data;
1336 while (rd != 0) {
1337 int sz = write(File->Fd(), dt, rd);
1338
1339 if (sz == -1) {
1340 delete File;
1341 File = NULL;
1342 Fail();
1343 goto done;
1344 }
1345
1346 dt += sz;
1347 rd -= sz;
1348 }
1349
1350 rd = CFReadStreamRead(rs, data, sizeof(data));
1351 goto read;
1352 }
1353 }
1354
1355 goto done;
1356 done_:
1357 CFRelease(hm);
1358 done:
1359 CFReadStreamClose(rs);
1360 CFRelease(rs);
1361 free(url);
1362
1363 FailCounter = 0;
1364 }
1365
1366 return 0;
1367 }
1368 /*}}}*/
1369
1370 int main()
1371 {
1372 #if !defined(__ENVIRONMENT_ASPEN_VERSION_MIN_REQUIRED__) || __ENVIRONMENT_ASPEN_VERSION_MIN_REQUIRED__ < 10200
1373 struct nlist nl[2];
1374 memset(nl, 0, sizeof(nl));
1375 nl[0].n_un.n_name = (char *) "_useMDNSResponder";
1376 nlist("/usr/lib/libc.dylib", nl);
1377 if (nl[0].n_type != N_UNDF)
1378 *(int *) nl[0].n_value = 0;
1379 #endif
1380
1381 setlocale(LC_ALL, "");
1382
1383 HttpMethod Mth;
1384
1385 size_t size;
1386 sysctlbyname("hw.machine", NULL, &size, NULL, 0);
1387 char *machine = new char[size];
1388 sysctlbyname("hw.machine", machine, &size, NULL, 0);
1389 Machine_ = machine;
1390
1391 const char *path = "/System/Library/CoreServices/SystemVersion.plist";
1392 CFURLRef url = CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault, (uint8_t *) path, strlen(path), false);
1393
1394 CFPropertyListRef plist; {
1395 CFReadStreamRef stream = CFReadStreamCreateWithFile(kCFAllocatorDefault, url);
1396 CFReadStreamOpen(stream);
1397 plist = CFPropertyListCreateFromStream(kCFAllocatorDefault, stream, 0, kCFPropertyListImmutable, NULL, NULL);
1398 CFReadStreamClose(stream);
1399 }
1400
1401 CFRelease(url);
1402
1403 if (plist != NULL) {
1404 Firmware_ = (CFStringRef) CFRetain(CFDictionaryGetValue((CFDictionaryRef) plist, CFSTR("ProductVersion")));
1405 CFRelease(plist);
1406 }
1407
1408 if (CFMutableDictionaryRef dict = IOServiceMatching("IOPlatformExpertDevice"))
1409 if (io_service_t service = IOServiceGetMatchingService(kIOMasterPortDefault, dict)) {
1410 if (CFTypeRef serial = IORegistryEntryCreateCFProperty(service, CFSTR(kIOPlatformSerialNumberKey), kCFAllocatorDefault, 0)) {
1411 SerialNumber_ = strdup(CFStringGetCStringPtr((CFStringRef) serial, CFStringGetSystemEncoding()));
1412 CFRelease(serial);
1413 }
1414
1415 IOObjectRelease(service);
1416 }
1417
1418 return Mth.Loop();
1419 }
1420
1421