1 // -*- mode: cpp; mode: fold -*-
3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
4 /* ######################################################################
6 HTTP Aquire Method - This is the HTTP aquire method for APT.
8 It uses HTTP/1.1 and many of the fancy options there-in, such as
9 pipelining, range, if-range and so on.
11 It is based on a doubly buffered select loop. A groupe of requests are
12 fed into a single output buffer that is constantly fed out the
13 socket. This provides ideal pipelining as in many cases all of the
14 requests will fit into a single packet. The input socket is buffered
15 the same way and fed into the fd for the file (may be a pipe in future).
17 This double buffering provides fairly substantial transfer rates,
18 compared to wget the http method is about 4% faster. Most importantly,
19 when HTTP is compared with FTP as a protocol the speed difference is
20 huge. In tests over the internet from two sites to llug (via ATM) this
21 program got 230k/s sustained http transfer rates. FTP on the other
22 hand topped out at 170k/s. That combined with the time to setup the
23 FTP connection makes HTTP a vastly superior protocol.
25 ##################################################################### */
27 // Include Files /*{{{*/
28 #include <apt-pkg/fileutl.h>
29 #include <apt-pkg/acquire-method.h>
30 #include <apt-pkg/error.h>
31 #include <apt-pkg/hashes.h>
33 #include <sys/sysctl.h>
47 #include <arpa/inet.h>
49 #include <CoreFoundation/CoreFoundation.h>
50 #include <CoreServices/CoreServices.h>
51 #include <SystemConfiguration/SystemConfiguration.h>
54 #include "rfc2553emu.h"
61 const char *SerialNumber_;
63 void CfrsError(const char *name, CFReadStreamRef rs) {
64 CFStreamError se = CFReadStreamGetError(rs);
66 if (se.domain == kCFStreamErrorDomainCustom) {
67 } else if (se.domain == kCFStreamErrorDomainPOSIX) {
68 _error->Error("POSIX: %s", strerror(se.error));
69 } else if (se.domain == kCFStreamErrorDomainMacOSStatus) {
70 _error->Error("MacOSStatus: %ld", se.error);
71 } else if (se.domain == kCFStreamErrorDomainNetDB) {
72 _error->Error("NetDB: %s %s", name, gai_strerror(se.error));
73 } else if (se.domain == kCFStreamErrorDomainMach) {
74 _error->Error("Mach: %ld", se.error);
75 } else if (se.domain == kCFStreamErrorDomainHTTP) {
77 case kCFStreamErrorHTTPParseFailure:
78 _error->Error("Parse failure");
81 case kCFStreamErrorHTTPRedirectionLoop:
82 _error->Error("Redirection loop");
85 case kCFStreamErrorHTTPBadURL:
86 _error->Error("Bad URL");
90 _error->Error("Unknown HTTP error: %ld", se.error);
93 } else if (se.domain == kCFStreamErrorDomainSOCKS) {
94 _error->Error("SOCKS: %ld", se.error);
95 } else if (se.domain == kCFStreamErrorDomainSystemConfiguration) {
96 _error->Error("SystemConfiguration: %ld", se.error);
97 } else if (se.domain == kCFStreamErrorDomainSSL) {
98 _error->Error("SSL: %ld", se.error);
100 _error->Error("Domain #%ld: %ld", se.domain, se.error);
104 string HttpMethod::FailFile;
105 int HttpMethod::FailFd = -1;
106 time_t HttpMethod::FailTime = 0;
107 unsigned long PipelineDepth = 10;
108 unsigned long TimeOut = 120;
111 unsigned long CircleBuf::BwReadLimit=0;
112 unsigned long CircleBuf::BwTickReadData=0;
113 struct timeval CircleBuf::BwReadTick={0,0};
114 const unsigned int CircleBuf::BW_HZ=10;
116 // CircleBuf::CircleBuf - Circular input buffer /*{{{*/
117 // ---------------------------------------------------------------------
119 CircleBuf::CircleBuf(unsigned long Size) : Size(Size), Hash(0)
121 Buf = new unsigned char[Size];
124 CircleBuf::BwReadLimit = _config->FindI("Acquire::http::Dl-Limit",0)*1024;
127 // CircleBuf::Reset - Reset to the default state /*{{{*/
128 // ---------------------------------------------------------------------
130 void CircleBuf::Reset()
135 MaxGet = (unsigned int)-1;
144 // CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
145 // ---------------------------------------------------------------------
146 /* This fills up the buffer with as much data as is in the FD, assuming it
148 bool CircleBuf::Read(int Fd)
150 unsigned long BwReadMax;
154 // Woops, buffer is full
155 if (InP - OutP == Size)
158 // what's left to read in this tick
159 BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
161 if(CircleBuf::BwReadLimit) {
163 gettimeofday(&now,0);
165 unsigned long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
166 now.tv_usec-CircleBuf::BwReadTick.tv_usec;
167 if(d > 1000000/BW_HZ) {
168 CircleBuf::BwReadTick = now;
169 CircleBuf::BwTickReadData = 0;
172 if(CircleBuf::BwTickReadData >= BwReadMax) {
173 usleep(1000000/BW_HZ);
178 // Write the buffer segment
180 if(CircleBuf::BwReadLimit) {
181 Res = read(Fd,Buf + (InP%Size),
182 BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
184 Res = read(Fd,Buf + (InP%Size),LeftRead());
186 if(Res > 0 && BwReadLimit > 0)
187 CircleBuf::BwTickReadData += Res;
199 gettimeofday(&Start,0);
204 // CircleBuf::Read - Put the string into the buffer /*{{{*/
205 // ---------------------------------------------------------------------
206 /* This will hold the string in and fill the buffer with it as it empties */
207 bool CircleBuf::Read(string Data)
214 // CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
215 // ---------------------------------------------------------------------
217 void CircleBuf::FillOut()
219 if (OutQueue.empty() == true)
223 // Woops, buffer is full
224 if (InP - OutP == Size)
227 // Write the buffer segment
228 unsigned long Sz = LeftRead();
229 if (OutQueue.length() - StrPos < Sz)
230 Sz = OutQueue.length() - StrPos;
231 memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
236 if (OutQueue.length() == StrPos)
245 // CircleBuf::Write - Write from the buffer into a FD /*{{{*/
246 // ---------------------------------------------------------------------
247 /* This empties the buffer into the FD. */
248 bool CircleBuf::Write(int Fd)
254 // Woops, buffer is empty
261 // Write the buffer segment
263 Res = write(Fd,Buf + (OutP%Size),LeftWrite());
276 Hash->Add(Buf + (OutP%Size),Res);
282 // CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
283 // ---------------------------------------------------------------------
284 /* This copies till the first empty line */
285 bool CircleBuf::WriteTillEl(string &Data,bool Single)
287 // We cheat and assume it is unneeded to have more than one buffer load
288 for (unsigned long I = OutP; I < InP; I++)
290 if (Buf[I%Size] != '\n')
296 if (I < InP && Buf[I%Size] == '\r')
298 if (I >= InP || Buf[I%Size] != '\n')
306 unsigned long Sz = LeftWrite();
311 Data += string((char *)(Buf + (OutP%Size)),Sz);
319 // CircleBuf::Stats - Print out stats information /*{{{*/
320 // ---------------------------------------------------------------------
322 void CircleBuf::Stats()
328 gettimeofday(&Stop,0);
329 /* float Diff = Stop.tv_sec - Start.tv_sec +
330 (float)(Stop.tv_usec - Start.tv_usec)/1000000;
331 clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
335 // ServerState::ServerState - Constructor /*{{{*/
336 // ---------------------------------------------------------------------
338 ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
339 In(64*1024), Out(4*1024),
345 // ServerState::Open - Open a connection to the server /*{{{*/
346 // ---------------------------------------------------------------------
347 /* This opens a connection to the server. */
348 bool ServerState::Open()
350 // Use the already open connection if possible.
359 // Determine the proxy setting
360 if (getenv("http_proxy") == 0)
362 string DefProxy = _config->Find("Acquire::http::Proxy");
363 string SpecificProxy = _config->Find("Acquire::http::Proxy::" + ServerName.Host);
364 if (SpecificProxy.empty() == false)
366 if (SpecificProxy == "DIRECT")
369 Proxy = SpecificProxy;
375 Proxy = getenv("http_proxy");
377 // Parse no_proxy, a , separated list of domains
378 if (getenv("no_proxy") != 0)
380 if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
384 // Determine what host and port to use based on the proxy settings
387 if (Proxy.empty() == true || Proxy.Host.empty() == true)
389 if (ServerName.Port != 0)
390 Port = ServerName.Port;
391 Host = ServerName.Host;
400 // Connect to the remote server
401 if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
407 // ServerState::Close - Close a connection to the server /*{{{*/
408 // ---------------------------------------------------------------------
410 bool ServerState::Close()
417 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
418 // ---------------------------------------------------------------------
419 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
420 parse error occured */
421 int ServerState::RunHeaders()
425 Owner->Status(_("Waiting for headers"));
439 if (In.WriteTillEl(Data) == false)
445 for (string::const_iterator I = Data.begin(); I < Data.end(); I++)
447 string::const_iterator J = I;
448 for (; J != Data.end() && *J != '\n' && *J != '\r';J++);
449 if (HeaderLine(string(I,J)) == false)
454 // 100 Continue is a Nop...
458 // Tidy up the connection persistance state.
459 if (Encoding == Closes && HaveContent == true)
464 while (Owner->Go(false,this) == true);
469 // ServerState::RunData - Transfer the data from the socket /*{{{*/
470 // ---------------------------------------------------------------------
472 bool ServerState::RunData()
476 // Chunked transfer encoding is fun..
477 if (Encoding == Chunked)
481 // Grab the block size
487 if (In.WriteTillEl(Data,true) == true)
490 while ((Last = Owner->Go(false,this)) == true);
495 // See if we are done
496 unsigned long Len = strtol(Data.c_str(),0,16);
501 // We have to remove the entity trailer
505 if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
508 while ((Last = Owner->Go(false,this)) == true);
511 return !_error->PendingError();
514 // Transfer the block
516 while (Owner->Go(true,this) == true)
517 if (In.IsLimit() == true)
521 if (In.IsLimit() == false)
524 // The server sends an extra new line before the next block specifier..
529 if (In.WriteTillEl(Data,true) == true)
532 while ((Last = Owner->Go(false,this)) == true);
539 /* Closes encoding is used when the server did not specify a size, the
540 loss of the connection means we are done */
541 if (Encoding == Closes)
544 In.Limit(Size - StartPos);
546 // Just transfer the whole block.
549 if (In.IsLimit() == false)
553 return !_error->PendingError();
555 while (Owner->Go(true,this) == true);
558 return Owner->Flush(this) && !_error->PendingError();
561 // ServerState::HeaderLine - Process a header line /*{{{*/
562 // ---------------------------------------------------------------------
564 bool ServerState::HeaderLine(string Line)
566 if (Line.empty() == true)
569 // The http server might be trying to do something evil.
570 if (Line.length() >= MAXLEN)
571 return _error->Error(_("Got a single header line over %u chars"),MAXLEN);
573 string::size_type Pos = Line.find(' ');
574 if (Pos == string::npos || Pos+1 > Line.length())
576 // Blah, some servers use "connection:closes", evil.
577 Pos = Line.find(':');
578 if (Pos == string::npos || Pos + 2 > Line.length())
579 return _error->Error(_("Bad header line"));
583 // Parse off any trailing spaces between the : and the next word.
584 string::size_type Pos2 = Pos;
585 while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
588 string Tag = string(Line,0,Pos);
589 string Val = string(Line,Pos2);
591 if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
593 // Evil servers return no version
596 if (sscanf(Line.c_str(),"HTTP/%u.%u %u %[^\n]",&Major,&Minor,
598 return _error->Error(_("The HTTP server sent an invalid reply header"));
604 if (sscanf(Line.c_str(),"HTTP %u %[^\n]",&Result,Code) != 2)
605 return _error->Error(_("The HTTP server sent an invalid reply header"));
608 /* Check the HTTP response header to get the default persistance
614 if (Major == 1 && Minor <= 0)
623 if (stringcasecmp(Tag,"Content-Length:") == 0)
625 if (Encoding == Closes)
629 // The length is already set from the Content-Range header
633 if (sscanf(Val.c_str(),"%lu",&Size) != 1)
634 return _error->Error(_("The HTTP server sent an invalid Content-Length header"));
638 if (stringcasecmp(Tag,"Content-Type:") == 0)
644 if (stringcasecmp(Tag,"Content-Range:") == 0)
648 if (sscanf(Val.c_str(),"bytes %lu-%*u/%lu",&StartPos,&Size) != 2)
649 return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
650 if ((unsigned)StartPos > Size)
651 return _error->Error(_("This HTTP server has broken range support"));
655 if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
658 if (stringcasecmp(Val,"chunked") == 0)
663 if (stringcasecmp(Tag,"Connection:") == 0)
665 if (stringcasecmp(Val,"close") == 0)
667 if (stringcasecmp(Val,"keep-alive") == 0)
672 if (stringcasecmp(Tag,"Last-Modified:") == 0)
674 if (StrToTime(Val,Date) == false)
675 return _error->Error(_("Unknown date format"));
683 // HttpMethod::SendReq - Send the HTTP request /*{{{*/
684 // ---------------------------------------------------------------------
685 /* This places the http request in the outbound buffer */
686 void HttpMethod::SendReq(FetchItem *Itm,CircleBuf &Out)
690 // The HTTP server expects a hostname with a trailing :port
692 string ProperHost = Uri.Host;
695 sprintf(Buf,":%u",Uri.Port);
700 if (Itm->Uri.length() >= sizeof(Buf))
703 /* Build the request. We include a keep-alive header only for non-proxy
704 requests. This is to tweak old http/1.0 servers that do support keep-alive
705 but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
706 will glitch HTTP/1.0 proxies because they do not filter it out and
707 pass it on, HTTP/1.1 says the connection should default to keep alive
708 and we expect the proxy to do this */
709 if (Proxy.empty() == true || Proxy.Host.empty())
710 sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
711 QuoteString(Uri.Path,"~").c_str(),ProperHost.c_str());
714 /* Generate a cache control header if necessary. We place a max
715 cache age on index files, optionally set a no-cache directive
716 and a no-store directive for archives. */
717 sprintf(Buf,"GET %s HTTP/1.1\r\nHost: %s\r\n",
718 Itm->Uri.c_str(),ProperHost.c_str());
719 // only generate a cache control header if we actually want to
721 if (_config->FindB("Acquire::http::No-Cache",false) == false)
723 if (Itm->IndexFile == true)
724 sprintf(Buf+strlen(Buf),"Cache-Control: max-age=%u\r\n",
725 _config->FindI("Acquire::http::Max-Age",0));
728 if (_config->FindB("Acquire::http::No-Store",false) == true)
729 strcat(Buf,"Cache-Control: no-store\r\n");
733 // generate a no-cache header if needed
734 if (_config->FindB("Acquire::http::No-Cache",false) == true)
735 strcat(Buf,"Cache-Control: no-cache\r\nPragma: no-cache\r\n");
740 // Check for a partial file
742 if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
744 // In this case we send an if-range query with a range header
745 sprintf(Buf,"Range: bytes=%li-\r\nIf-Range: %s\r\n",(long)SBuf.st_size - 1,
746 TimeRFC1123(SBuf.st_mtime).c_str());
751 if (Itm->LastModified != 0)
753 sprintf(Buf,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm->LastModified).c_str());
758 if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
759 Req += string("Proxy-Authorization: Basic ") +
760 Base64Encode(Proxy.User + ":" + Proxy.Password) + "\r\n";
762 if (Uri.User.empty() == false || Uri.Password.empty() == false)
763 Req += string("Authorization: Basic ") +
764 Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
766 Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
774 // HttpMethod::Go - Run a single loop /*{{{*/
775 // ---------------------------------------------------------------------
776 /* This runs the select loop over the server FDs, Output file FDs and
778 bool HttpMethod::Go(bool ToFile,ServerState *Srv)
780 // Server has closed the connection
781 if (Srv->ServerFd == -1 && (Srv->In.WriteSpace() == false ||
789 /* Add the server. We only send more requests if the connection will
791 if (Srv->Out.WriteSpace() == true && Srv->ServerFd != -1
792 && Srv->Persistent == true)
793 FD_SET(Srv->ServerFd,&wfds);
794 if (Srv->In.ReadSpace() == true && Srv->ServerFd != -1)
795 FD_SET(Srv->ServerFd,&rfds);
802 if (Srv->In.WriteSpace() == true && ToFile == true && FileFD != -1)
803 FD_SET(FileFD,&wfds);
806 FD_SET(STDIN_FILENO,&rfds);
808 // Figure out the max fd
810 if (MaxFd < Srv->ServerFd)
811 MaxFd = Srv->ServerFd;
818 if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
822 return _error->Errno("select",_("Select failed"));
827 _error->Error(_("Connection timed out"));
828 return ServerDie(Srv);
832 if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&rfds))
835 if (Srv->In.Read(Srv->ServerFd) == false)
836 return ServerDie(Srv);
839 if (Srv->ServerFd != -1 && FD_ISSET(Srv->ServerFd,&wfds))
842 if (Srv->Out.Write(Srv->ServerFd) == false)
843 return ServerDie(Srv);
846 // Send data to the file
847 if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
849 if (Srv->In.Write(FileFD) == false)
850 return _error->Errno("write",_("Error writing to output file"));
853 // Handle commands from APT
854 if (FD_ISSET(STDIN_FILENO,&rfds))
863 // HttpMethod::Flush - Dump the buffer into the file /*{{{*/
864 // ---------------------------------------------------------------------
865 /* This takes the current input buffer from the Server FD and writes it
867 bool HttpMethod::Flush(ServerState *Srv)
871 // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
873 if (File->Name() != "/dev/null")
874 SetNonBlock(File->Fd(),false);
875 if (Srv->In.WriteSpace() == false)
878 while (Srv->In.WriteSpace() == true)
880 if (Srv->In.Write(File->Fd()) == false)
881 return _error->Errno("write",_("Error writing to file"));
882 if (Srv->In.IsLimit() == true)
886 if (Srv->In.IsLimit() == true || Srv->Encoding == ServerState::Closes)
892 // HttpMethod::ServerDie - The server has closed the connection. /*{{{*/
893 // ---------------------------------------------------------------------
895 bool HttpMethod::ServerDie(ServerState *Srv)
897 unsigned int LErrno = errno;
899 // Dump the buffer to the file
900 if (Srv->State == ServerState::Data)
902 // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
904 if (File->Name() != "/dev/null")
905 SetNonBlock(File->Fd(),false);
906 while (Srv->In.WriteSpace() == true)
908 if (Srv->In.Write(File->Fd()) == false)
909 return _error->Errno("write",_("Error writing to the file"));
912 if (Srv->In.IsLimit() == true)
917 // See if this is because the server finished the data stream
918 if (Srv->In.IsLimit() == false && Srv->State != ServerState::Header &&
919 Srv->Encoding != ServerState::Closes)
923 return _error->Error(_("Error reading from server. Remote end closed connection"));
925 return _error->Errno("read",_("Error reading from server"));
931 // Nothing left in the buffer
932 if (Srv->In.WriteSpace() == false)
935 // We may have got multiple responses back in one packet..
943 // HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
944 // ---------------------------------------------------------------------
945 /* We look at the header data we got back from the server and decide what
949 3 - Unrecoverable error
950 4 - Error with error content page
951 5 - Unrecoverable non-server error (close the connection) */
952 int HttpMethod::DealWithHeaders(FetchResult &Res,ServerState *Srv)
955 if (Srv->Result == 304)
957 unlink(Queue->DestFile.c_str());
959 Res.LastModified = Queue->LastModified;
963 /* We have a reply we dont handle. This should indicate a perm server
965 if (Srv->Result < 200 || Srv->Result >= 300)
967 _error->Error("%u %s",Srv->Result,Srv->Code);
968 if (Srv->HaveContent == true)
973 // This is some sort of 2xx 'data follows' reply
974 Res.LastModified = Srv->Date;
975 Res.Size = Srv->Size;
979 File = new FileFd(Queue->DestFile,FileFd::WriteAny);
980 if (_error->PendingError() == true)
983 FailFile = Queue->DestFile;
984 FailFile.c_str(); // Make sure we dont do a malloc in the signal handler
986 FailTime = Srv->Date;
988 // Set the expected size
989 if (Srv->StartPos >= 0)
991 Res.ResumePoint = Srv->StartPos;
992 ftruncate(File->Fd(),Srv->StartPos);
995 // Set the start point
996 lseek(File->Fd(),0,SEEK_END);
999 Srv->In.Hash = new Hashes;
1001 // Fill the Hash if the file is non-empty (resume)
1002 if (Srv->StartPos > 0)
1004 lseek(File->Fd(),0,SEEK_SET);
1005 if (Srv->In.Hash->AddFD(File->Fd(),Srv->StartPos) == false)
1007 _error->Errno("read",_("Problem hashing file"));
1010 lseek(File->Fd(),0,SEEK_END);
1013 SetNonBlock(File->Fd(),true);
1017 // HttpMethod::SigTerm - Handle a fatal signal /*{{{*/
1018 // ---------------------------------------------------------------------
1019 /* This closes and timestamps the open file. This is neccessary to get
1020 resume behavoir on user abort */
1021 void HttpMethod::SigTerm(int)
1028 struct utimbuf UBuf;
1029 UBuf.actime = FailTime;
1030 UBuf.modtime = FailTime;
1031 utime(FailFile.c_str(),&UBuf);
1036 // HttpMethod::Fetch - Fetch an item /*{{{*/
1037 // ---------------------------------------------------------------------
1038 /* This adds an item to the pipeline. We keep the pipeline at a fixed
1040 bool HttpMethod::Fetch(FetchItem *)
1045 // Queue the requests
1048 for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
1049 I = I->Next, Depth++)
1051 // If pipelining is disabled, we only queue 1 request
1052 if (Server->Pipeline == false && Depth >= 0)
1055 // Make sure we stick with the same server
1056 if (Server->Comp(I->Uri) == false)
1062 QueueBack = I->Next;
1063 SendReq(I,Server->Out);
1071 // HttpMethod::Configuration - Handle a configuration message /*{{{*/
1072 // ---------------------------------------------------------------------
1073 /* We stash the desired pipeline depth */
1074 bool HttpMethod::Configuration(string Message)
1076 if (pkgAcqMethod::Configuration(Message) == false)
1079 TimeOut = _config->FindI("Acquire::http::Timeout",TimeOut);
1080 PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
1082 Debug = _config->FindB("Debug::Acquire::http",false);
1087 // HttpMethod::Loop - Main loop /*{{{*/
1088 // ---------------------------------------------------------------------
1090 int HttpMethod::Loop()
1092 signal(SIGTERM,SigTerm);
1093 signal(SIGINT,SigTerm);
1097 int FailCounter = 0;
1100 // We have no commands, wait for some to arrive
1103 if (WaitFd(STDIN_FILENO) == false)
1107 /* Run messages, we can accept 0 (no message) if we didn't
1108 do a WaitFd above.. Otherwise the FD is closed. */
1109 int Result = Run(true);
1110 if (Result != -1 && (Result != 0 || Queue == 0))
1116 CFStringEncoding se = kCFStringEncodingUTF8;
1118 char *url = strdup(Queue->Uri.c_str());
1120 URI uri = std::string(url);
1121 std::string hs = uri.Host;
1123 struct hostent *he = gethostbyname(hs.c_str());
1124 if (he == NULL || he->h_addr_list[0] == NULL) {
1125 _error->Error(hstrerror(h_errno));
1130 uri.Host = inet_ntoa(* (struct in_addr *) he->h_addr_list[0]);
1132 std::string urs = uri;
1134 CFStringRef sr = CFStringCreateWithCString(kCFAllocatorDefault, urs.c_str(), se);
1135 CFURLRef ur = CFURLCreateWithString(kCFAllocatorDefault, sr, NULL);
1137 CFHTTPMessageRef hm = CFHTTPMessageCreateRequest(kCFAllocatorDefault, CFSTR("GET"), ur, kCFHTTPVersion1_1);
1141 if (stat(Queue->DestFile.c_str(), &SBuf) >= 0 && SBuf.st_size > 0) {
1142 sr = CFStringCreateWithFormat(kCFAllocatorDefault, NULL, CFSTR("bytes=%li-"), (long) SBuf.st_size - 1);
1143 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Range"), sr);
1146 sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
1147 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Range"), sr);
1149 } else if (Queue->LastModified != 0) {
1150 sr = CFStringCreateWithCString(kCFAllocatorDefault, TimeRFC1123(SBuf.st_mtime).c_str(), se);
1151 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("If-Modified-Since"), sr);
1155 sr = CFStringCreateWithCString(kCFAllocatorDefault, Machine_, se);
1156 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Machine"), sr);
1159 sr = CFStringCreateWithCString(kCFAllocatorDefault, SerialNumber_, se);
1160 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("X-Serial-Number"), sr);
1163 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("User-Agent"), CFSTR("Telesphoreo APT-HTTP/1.0.98"));
1165 sr = CFStringCreateWithCString(kCFAllocatorDefault, hs.c_str(), se);
1166 CFHTTPMessageSetHeaderFieldValue(hm, CFSTR("Host"), sr);
1169 CFReadStreamRef rs = CFReadStreamCreateForHTTPRequest(kCFAllocatorDefault, hm);
1172 CFDictionaryRef dr = SCDynamicStoreCopyProxies(NULL);
1173 CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPProxy, dr);
1176 //CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPShouldAutoredirect, kCFBooleanTrue);
1177 CFReadStreamSetProperty(rs, kCFStreamPropertyHTTPAttemptPersistentConnection, kCFBooleanTrue);
1183 uint8_t data[10240];
1186 Status("Connecting to %s", hs.c_str());
1188 if (!CFReadStreamOpen(rs)) {
1189 CfrsError("Open", rs);
1194 rd = CFReadStreamRead(rs, data, sizeof(data));
1197 CfrsError(uri.Host.c_str(), rs);
1202 Res.Filename = Queue->DestFile;
1204 hm = (CFHTTPMessageRef) CFReadStreamCopyProperty(rs, kCFStreamPropertyHTTPResponseHeader);
1205 sc = CFHTTPMessageGetResponseStatusCode(hm);
1207 if (sc == 301 || sc == 302) {
1208 sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Location"));
1213 size_t ln = CFStringGetLength(sr) + 1;
1215 url = static_cast<char *>(malloc(ln));
1217 if (!CFStringGetCString(sr, url, ln, se)) {
1227 sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Range"));
1229 size_t ln = CFStringGetLength(sr) + 1;
1232 if (!CFStringGetCString(sr, cr, ln, se)) {
1239 if (sscanf(cr, "bytes %lu-%*u/%lu", &offset, &Res.Size) != 2) {
1240 _error->Error(_("The HTTP server sent an invalid Content-Range header"));
1245 if (offset > Res.Size) {
1246 _error->Error(_("This HTTP server has broken range support"));
1251 sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Content-Length"));
1253 Res.Size = CFStringGetIntValue(sr);
1258 time(&Res.LastModified);
1260 sr = CFHTTPMessageCopyHeaderFieldValue(hm, CFSTR("Last-Modified"));
1262 size_t ln = CFStringGetLength(sr) + 1;
1265 if (!CFStringGetCString(sr, cr, ln, se)) {
1272 if (!StrToTime(cr, Res.LastModified)) {
1273 _error->Error(_("Unknown date format"));
1282 unlink(Queue->DestFile.c_str());
1284 Res.LastModified = Queue->LastModified;
1286 } else if (sc < 200 || sc >= 300)
1291 File = new FileFd(Queue->DestFile, FileFd::WriteAny);
1292 if (_error->PendingError() == true) {
1299 FailFile = Queue->DestFile;
1300 FailFile.c_str(); // Make sure we dont do a malloc in the signal handler
1301 FailFd = File->Fd();
1302 FailTime = Res.LastModified;
1304 Res.ResumePoint = offset;
1305 ftruncate(File->Fd(), offset);
1308 lseek(File->Fd(), 0, SEEK_SET);
1309 if (!hash.AddFD(File->Fd(), offset)) {
1310 _error->Errno("read", _("Problem hashing file"));
1318 lseek(File->Fd(), 0, SEEK_END);
1322 read: if (rd == -1) {
1323 CfrsError("rd", rs);
1325 } else if (rd == 0) {
1327 Res.Size = File->Size();
1329 struct utimbuf UBuf;
1331 UBuf.actime = Res.LastModified;
1332 UBuf.modtime = Res.LastModified;
1333 utime(Queue->DestFile.c_str(), &UBuf);
1335 Res.TakeHashes(hash);
1342 int sz = write(File->Fd(), dt, rd);
1355 rd = CFReadStreamRead(rs, data, sizeof(data));
1364 CFReadStreamClose(rs);
1377 setlocale(LC_ALL, "");
1382 sysctlbyname("hw.machine", NULL, &size, NULL, 0);
1383 char *machine = new char[size];
1384 sysctlbyname("hw.machine", machine, &size, NULL, 0);
1387 if (CFMutableDictionaryRef dict = IOServiceMatching("IOPlatformExpertDevice"))
1388 if (io_service_t service = IOServiceGetMatchingService(kIOMasterPortDefault, dict)) {
1389 if (CFTypeRef serial = IORegistryEntryCreateCFProperty(service, CFSTR(kIOPlatformSerialNumberKey), kCFAllocatorDefault, 0)) {
1390 SerialNumber_ = strdup(CFStringGetCStringPtr((CFStringRef) serial, CFStringGetSystemEncoding()));
1394 IOObjectRelease(service);