]>
git.saurik.com Git - apt.git/blob - methods/http.cc
1 // -*- mode: cpp; mode: fold -*-
3 // $Id: http.cc,v 1.6 1998/11/21 06:09:09 jgg Exp $
4 /* ######################################################################
6 HTTP Aquire Method - This is the HTTP aquire method for APT.
8 It uses HTTP/1.1 and many of the fancy options there-in, such as
9 pipelining, range, if-range and so on. It accepts on the command line
10 a list of url destination pairs and writes to stdout the status of the
11 operation as defined in the APT method spec.
13 It is based on a doubly buffered select loop. All the requests are
14 fed into a single output buffer that is constantly fed out the
15 socket. This provides ideal pipelining as in many cases all of the
16 requests will fit into a single packet. The input socket is buffered
17 the same way and fed into the fd for the file.
19 This double buffering provides fairly substantial transfer rates,
20 compared to wget the http method is about 4% faster. Most importantly,
21 when HTTP is compared with FTP as a protocol the speed difference is
22 huge. In tests over the internet from two sites to llug (via ATM) this
23 program got 230k/s sustained http transfer rates. FTP on the other
24 hand topped out at 170k/s. That combined with the time to setup the
25 FTP connection makes HTTP a vastly superior protocol.
27 ##################################################################### */
29 // Include Files /*{{{*/
30 #include <apt-pkg/fileutl.h>
31 #include <apt-pkg/acquire-method.h>
32 #include <apt-pkg/error.h>
33 #include <apt-pkg/md5.h>
43 #include <netinet/in.h>
44 #include <sys/socket.h>
45 #include <arpa/inet.h>
51 string
HttpMethod::FailFile
;
52 int HttpMethod::FailFd
= -1;
53 time_t HttpMethod::FailTime
= 0;
55 // CircleBuf::CircleBuf - Circular input buffer /*{{{*/
56 // ---------------------------------------------------------------------
58 CircleBuf::CircleBuf(unsigned long Size
) : Size(Size
), MD5(0)
60 Buf
= new unsigned char[Size
];
64 // CircleBuf::Reset - Reset to the default state /*{{{*/
65 // ---------------------------------------------------------------------
67 void CircleBuf::Reset()
72 MaxGet
= (unsigned int)-1;
77 MD5
= new MD5Summation
;
81 // CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
82 // ---------------------------------------------------------------------
83 /* This fills up the buffer with as much data as is in the FD, assuming it
85 bool CircleBuf::Read(int Fd
)
89 // Woops, buffer is full
90 if (InP
- OutP
== Size
)
93 // Write the buffer segment
95 Res
= read(Fd
,Buf
+ (InP%Size
),LeftRead());
107 gettimeofday(&Start
,0);
112 // CircleBuf::Read - Put the string into the buffer /*{{{*/
113 // ---------------------------------------------------------------------
114 /* This will hold the string in and fill the buffer with it as it empties */
115 bool CircleBuf::Read(string Data
)
122 // CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
123 // ---------------------------------------------------------------------
125 void CircleBuf::FillOut()
127 if (OutQueue
.empty() == true)
131 // Woops, buffer is full
132 if (InP
- OutP
== Size
)
135 // Write the buffer segment
136 unsigned long Sz
= LeftRead();
137 if (OutQueue
.length() - StrPos
< Sz
)
138 Sz
= OutQueue
.length() - StrPos
;
139 memcpy(Buf
+ (InP%Size
),OutQueue
.begin() + StrPos
,Sz
);
144 if (OutQueue
.length() == StrPos
)
153 // CircleBuf::Write - Write from the buffer into a FD /*{{{*/
154 // ---------------------------------------------------------------------
155 /* This empties the buffer into the FD. */
156 bool CircleBuf::Write(int Fd
)
162 // Woops, buffer is empty
169 // Write the buffer segment
171 Res
= write(Fd
,Buf
+ (OutP%Size
),LeftWrite());
184 MD5
->Add(Buf
+ (OutP%Size
),Res
);
190 // CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
191 // ---------------------------------------------------------------------
192 /* This copies till the first empty line */
193 bool CircleBuf::WriteTillEl(string
&Data
,bool Single
)
195 // We cheat and assume it is unneeded to have more than one buffer load
196 for (unsigned long I
= OutP
; I
< InP
; I
++)
198 if (Buf
[I%Size
] != '\n')
200 for (I
++; I
< InP
&& Buf
[I%Size
] == '\r'; I
++);
204 if (Buf
[I%Size
] != '\n')
206 for (I
++; I
< InP
&& Buf
[I%Size
] == '\r'; I
++);
215 unsigned long Sz
= LeftWrite();
218 if (I
- OutP
< LeftWrite())
220 Data
+= string((char *)(Buf
+ (OutP%Size
)),Sz
);
228 // CircleBuf::Stats - Print out stats information /*{{{*/
229 // ---------------------------------------------------------------------
231 void CircleBuf::Stats()
237 gettimeofday(&Stop
,0);
238 /* float Diff = Stop.tv_sec - Start.tv_sec +
239 (float)(Stop.tv_usec - Start.tv_usec)/1000000;
240 clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
244 // ServerState::ServerState - Constructor /*{{{*/
245 // ---------------------------------------------------------------------
247 ServerState::ServerState(URI Srv
,HttpMethod
*Owner
) : Owner(Owner
),
248 In(64*1024), Out(1*1024),
254 // ServerState::Open - Open a connection to the server /*{{{*/
255 // ---------------------------------------------------------------------
256 /* This opens a connection to the server. */
259 bool ServerState::Open()
261 // Use the already open connection if possible.
269 // Determine the proxy setting
270 string DefProxy
= _config
->Find("Acquire::http::Proxy",getenv("http_proxy"));
271 string SpecificProxy
= _config
->Find("Acquire::http::Proxy::" + ServerName
.Host
);
272 if (SpecificProxy
.empty() == false)
274 if (SpecificProxy
== "DIRECT")
277 Proxy
= SpecificProxy
;
282 // Determine what host and port to use based on the proxy settings
285 if (Proxy
.empty() == true)
287 if (ServerName
.Port
!= 0)
288 Port
= ServerName
.Port
;
289 Host
= ServerName
.Host
;
298 /* We used a cached address record.. Yes this is against the spec but
299 the way we have setup our rotating dns suggests that this is more
301 if (LastHost
!= Host
)
303 Owner
->Status("Connecting to %s",Host
.c_str());
306 hostent
*Addr
= gethostbyname(Host
.c_str());
308 return _error
->Error("Could not resolve '%s'",Host
.c_str());
310 LastHostA
= *(in_addr
*)(Addr
->h_addr_list
[0]);
313 Owner
->Status("Connecting to %s (%s)",Host
.c_str(),inet_ntoa(LastHostA
));
316 if ((ServerFd
= socket(AF_INET
,SOCK_STREAM
,0)) < 0)
317 return _error
->Errno("socket","Could not create a socket");
319 // Connect to the server
320 struct sockaddr_in server
;
321 server
.sin_family
= AF_INET
;
322 server
.sin_port
= htons(Port
);
323 server
.sin_addr
= LastHostA
;
324 if (connect(ServerFd
,(sockaddr
*)&server
,sizeof(server
)) < 0)
325 return _error
->Errno("socket","Could not create a socket");
327 SetNonBlock(ServerFd
,true);
331 // ServerState::Close - Close a connection to the server /*{{{*/
332 // ---------------------------------------------------------------------
334 bool ServerState::Close()
341 // ServerState::RunHeaders - Get the headers before the data /*{{{*/
342 // ---------------------------------------------------------------------
343 /* Returns 0 if things are OK, 1 if an IO error occursed and 2 if a header
344 parse error occured */
345 int ServerState::RunHeaders()
349 Owner
->Status("Waiting for file");
363 if (In
.WriteTillEl(Data
) == false)
366 for (string::const_iterator I
= Data
.begin(); I
< Data
.end(); I
++)
368 string::const_iterator J
= I
;
369 for (; J
!= Data
.end() && *J
!= '\n' && *J
!= '\r';J
++);
370 if (HeaderLine(string(I
,J
-I
)) == false)
376 while (Owner
->Go(false,this) == true);
381 // ServerState::RunData - Transfer the data from the socket /*{{{*/
382 // ---------------------------------------------------------------------
384 bool ServerState::RunData()
388 // Chunked transfer encoding is fun..
389 if (Encoding
== Chunked
)
393 // Grab the block size
399 if (In
.WriteTillEl(Data
,true) == true)
402 while ((Last
= Owner
->Go(false,this)) == true);
407 // See if we are done
408 unsigned long Len
= strtol(Data
.c_str(),0,16);
413 // We have to remove the entity trailer
417 if (In
.WriteTillEl(Data
,true) == true && Data
.length() <= 2)
420 while ((Last
= Owner
->Go(false,this)) == true);
426 // Transfer the block
428 while (Owner
->Go(true,this) == true)
429 if (In
.IsLimit() == true)
433 if (In
.IsLimit() == false)
436 // The server sends an extra new line before the next block specifier..
441 if (In
.WriteTillEl(Data
,true) == true)
444 while ((Last
= Owner
->Go(false,this)) == true);
451 /* Closes encoding is used when the server did not specify a size, the
452 loss of the connection means we are done */
453 if (Encoding
== Closes
)
456 In
.Limit(Size
- StartPos
);
458 // Just transfer the whole block.
461 if (In
.IsLimit() == false)
467 while (Owner
->Go(true,this) == true);
470 return Owner
->Flush(this);
473 // ServerState::HeaderLine - Process a header line /*{{{*/
474 // ---------------------------------------------------------------------
476 bool ServerState::HeaderLine(string Line
)
478 if (Line
.empty() == true)
481 // The http server might be trying to do something evil.
482 if (Line
.length() >= MAXLEN
)
483 return _error
->Error("Got a single header line over %u chars",MAXLEN
);
485 string::size_type Pos
= Line
.find(' ');
486 if (Pos
== string::npos
|| Pos
+1 > Line
.length())
487 return _error
->Error("Bad header line");
489 string Tag
= string(Line
,0,Pos
);
490 string Val
= string(Line
,Pos
+1);
492 if (stringcasecmp(Tag
.begin(),Tag
.begin()+4,"HTTP") == 0)
494 // Evil servers return no version
497 if (sscanf(Line
.c_str(),"HTTP/%u.%u %u %[^\n]",&Major
,&Minor
,
499 return _error
->Error("The http server sent an invalid reply header");
505 if (sscanf(Line
.c_str(),"HTTP %u %[^\n]",&Result
,Code
) != 2)
506 return _error
->Error("The http server sent an invalid reply header");
512 if (stringcasecmp(Tag
,"Content-Length:") == 0)
514 if (Encoding
== Closes
)
518 // The length is already set from the Content-Range header
522 if (sscanf(Val
.c_str(),"%lu",&Size
) != 1)
523 return _error
->Error("The http server sent an invalid Content-Length header");
527 if (stringcasecmp(Tag
,"Content-Type:") == 0)
533 if (stringcasecmp(Tag
,"Content-Range:") == 0)
537 if (sscanf(Val
.c_str(),"bytes %lu-%*u/%lu",&StartPos
,&Size
) != 2)
538 return _error
->Error("The http server sent an invalid Content-Range header");
539 if ((unsigned)StartPos
> Size
)
540 return _error
->Error("This http server has broken range support");
544 if (stringcasecmp(Tag
,"Transfer-Encoding:") == 0)
547 if (stringcasecmp(Val
,"chunked") == 0)
553 if (stringcasecmp(Tag
,"Last-Modified:") == 0)
555 if (StrToTime(Val
,Date
) == false)
556 return _error
->Error("Unknown date format");
564 // HttpMethod::SendReq - Send the HTTP request /*{{{*/
565 // ---------------------------------------------------------------------
566 /* This places the http request in the outbound buffer */
567 void HttpMethod::SendReq(FetchItem
*Itm
,CircleBuf
&Out
)
571 // The HTTP server expects a hostname with a trailing :port
573 string ProperHost
= Uri
.Host
;
576 sprintf(Buf
,":%u",Uri
.Port
);
580 /* Build the request. We include a keep-alive header only for non-proxy
581 requests. This is to tweak old http/1.0 servers that do support keep-alive
582 but not HTTP/1.1 automatic keep-alive. Doing this with a proxy server
583 will glitch HTTP/1.0 proxies because they do not filter it out and
584 pass it on, HTTP/1.1 says the connection should default to keep alive
585 and we expect the proxy to do this */
586 if (Proxy
.empty() == true)
587 sprintf(Buf
,"GET %s HTTP/1.1\r\nHost: %s\r\nConnection: keep-alive\r\n",
588 Uri
.Path
.c_str(),ProperHost
.c_str());
590 sprintf(Buf
,"GET %s HTTP/1.1\r\nHost: %s\r\n",
591 Itm
->Uri
.c_str(),ProperHost
.c_str());
594 // Check for a partial file
596 if (stat(Itm
->DestFile
.c_str(),&SBuf
) >= 0 && SBuf
.st_size
> 0)
598 // In this case we send an if-range query with a range header
599 sprintf(Buf
,"Range: bytes=%li-\r\nIf-Range: %s\r\n",SBuf
.st_size
- 1,
600 TimeRFC1123(SBuf
.st_mtime
).c_str());
605 if (Itm
->LastModified
!= 0)
607 sprintf(Buf
,"If-Modified-Since: %s\r\n",TimeRFC1123(Itm
->LastModified
).c_str());
612 /* if (ProxyAuth.empty() == false)
613 Req += string("Proxy-Authorization: Basic ") + Base64Encode(ProxyAuth) + "\r\n";*/
615 Req
+= "User-Agent: Debian APT-HTTP/1.2\r\n\r\n";
616 // cerr << Req << endl;
621 // HttpMethod::Go - Run a single loop /*{{{*/
622 // ---------------------------------------------------------------------
623 /* This runs the select loop over the server FDs, Output file FDs and
625 bool HttpMethod::Go(bool ToFile
,ServerState
*Srv
)
627 // Server has closed the connection
628 if (Srv
->ServerFd
== -1 && Srv
->In
.WriteSpace() == false)
631 fd_set rfds
,wfds
,efds
;
637 if (Srv
->Out
.WriteSpace() == true && Srv
->ServerFd
!= -1)
638 FD_SET(Srv
->ServerFd
,&wfds
);
639 if (Srv
->In
.ReadSpace() == true && Srv
->ServerFd
!= -1)
640 FD_SET(Srv
->ServerFd
,&rfds
);
647 if (Srv
->In
.WriteSpace() == true && ToFile
== true && FileFD
!= -1)
648 FD_SET(FileFD
,&wfds
);
651 FD_SET(STDIN_FILENO
,&rfds
);
655 FD_SET(FileFD
,&efds
);
656 if (Srv
->ServerFd
!= -1)
657 FD_SET(Srv
->ServerFd
,&efds
);
659 // Figure out the max fd
661 if (MaxFd
< Srv
->ServerFd
)
662 MaxFd
= Srv
->ServerFd
;
669 if ((Res
= select(MaxFd
+1,&rfds
,&wfds
,&efds
,&tv
)) < 0)
670 return _error
->Errno("select","Select failed");
674 _error
->Error("Connection timed out");
675 return ServerDie(Srv
);
678 // Some kind of exception (error) on the sockets, die
679 if ((FileFD
!= -1 && FD_ISSET(FileFD
,&efds
)) ||
680 (Srv
->ServerFd
!= -1 && FD_ISSET(Srv
->ServerFd
,&efds
)))
681 return _error
->Error("Socket Exception");
684 if (Srv
->ServerFd
!= -1 && FD_ISSET(Srv
->ServerFd
,&rfds
))
687 if (Srv
->In
.Read(Srv
->ServerFd
) == false)
688 return ServerDie(Srv
);
691 if (Srv
->ServerFd
!= -1 && FD_ISSET(Srv
->ServerFd
,&wfds
))
694 if (Srv
->Out
.Write(Srv
->ServerFd
) == false)
695 return ServerDie(Srv
);
698 // Send data to the file
699 if (FileFD
!= -1 && FD_ISSET(FileFD
,&wfds
))
701 if (Srv
->In
.Write(FileFD
) == false)
702 return _error
->Errno("write","Error writing to output file");
705 // Handle commands from APT
706 if (FD_ISSET(STDIN_FILENO
,&rfds
))
715 // HttpMethod::Flush - Dump the buffer into the file /*{{{*/
716 // ---------------------------------------------------------------------
717 /* This takes the current input buffer from the Server FD and writes it
719 bool HttpMethod::Flush(ServerState
*Srv
)
723 SetNonBlock(File
->Fd(),false);
724 if (Srv
->In
.WriteSpace() == false)
727 while (Srv
->In
.WriteSpace() == true)
729 if (Srv
->In
.Write(File
->Fd()) == false)
730 return _error
->Errno("write","Error writing to file");
731 if (Srv
->In
.IsLimit() == true)
735 if (Srv
->In
.IsLimit() == true || Srv
->Encoding
== ServerState::Closes
)
741 // HttpMethod::ServerDie - The server has closed the connection. /*{{{*/
742 // ---------------------------------------------------------------------
744 bool HttpMethod::ServerDie(ServerState
*Srv
)
746 // Dump the buffer to the file
747 if (Srv
->State
== ServerState::Data
)
749 SetNonBlock(File
->Fd(),false);
750 while (Srv
->In
.WriteSpace() == true)
752 if (Srv
->In
.Write(File
->Fd()) == false)
753 return _error
->Errno("write","Error writing to the file");
756 if (Srv
->In
.IsLimit() == true)
761 // See if this is because the server finished the data stream
762 if (Srv
->In
.IsLimit() == false && Srv
->State
!= ServerState::Header
&&
763 Srv
->Encoding
!= ServerState::Closes
)
766 return _error
->Error("Error reading from server Remote end closed connection");
767 return _error
->Errno("read","Error reading from server");
773 // Nothing left in the buffer
774 if (Srv
->In
.WriteSpace() == false)
777 // We may have got multiple responses back in one packet..
785 // HttpMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
786 // ---------------------------------------------------------------------
787 /* We look at the header data we got back from the server and decide what
791 3 - Unrecoverable error
792 4 - Error with error content page
793 5 - Unrecoverable non-server error (close the connection) */
794 int HttpMethod::DealWithHeaders(FetchResult
&Res
,ServerState
*Srv
)
797 if (Srv
->Result
== 304)
799 unlink(Queue
->DestFile
.c_str());
801 Res
.LastModified
= Queue
->LastModified
;
805 /* We have a reply we dont handle. This should indicate a perm server
807 if (Srv
->Result
< 200 || Srv
->Result
>= 300)
809 _error
->Error("%u %s",Srv
->Result
,Srv
->Code
);
810 if (Srv
->HaveContent
== true)
815 // This is some sort of 2xx 'data follows' reply
816 Res
.LastModified
= Srv
->Date
;
817 Res
.Size
= Srv
->Size
;
821 File
= new FileFd(Queue
->DestFile
,FileFd::WriteAny
);
822 if (_error
->PendingError() == true)
825 FailFile
= Queue
->DestFile
;
827 FailTime
= Srv
->Date
;
829 // Set the expected size
830 if (Srv
->StartPos
>= 0)
832 Res
.ResumePoint
= Srv
->StartPos
;
833 ftruncate(File
->Fd(),Srv
->StartPos
);
836 // Set the start point
837 lseek(File
->Fd(),0,SEEK_END
);
840 Srv
->In
.MD5
= new MD5Summation
;
842 // Fill the MD5 Hash if the file is non-empty (resume)
843 if (Srv
->StartPos
> 0)
845 lseek(File
->Fd(),0,SEEK_SET
);
846 if (Srv
->In
.MD5
->AddFD(File
->Fd(),Srv
->StartPos
) == false)
848 _error
->Errno("read","Problem hashing file");
851 lseek(File
->Fd(),0,SEEK_END
);
854 SetNonBlock(File
->Fd(),true);
858 // HttpMethod::SigTerm - Handle a fatal signal /*{{{*/
859 // ---------------------------------------------------------------------
860 /* This closes and timestamps the open file. This is neccessary to get
861 resume behavoir on user abort */
862 void HttpMethod::SigTerm(int)
871 UBuf
.actime
= FailTime
;
872 UBuf
.modtime
= FailTime
;
873 utime(FailFile
.c_str(),&UBuf
);
878 // HttpMethod::Loop - Main loop /*{{{*/
879 // ---------------------------------------------------------------------
881 int HttpMethod::Loop()
883 signal(SIGTERM
,SigTerm
);
884 signal(SIGINT
,SigTerm
);
886 ServerState
*Server
= 0;
891 if (FailCounter
>= 2)
893 Fail("Massive Server Brain Damage");
897 // We have no commands, wait for some to arrive
900 if (WaitFd(STDIN_FILENO
) == false)
911 // Connect to the server
912 if (Server
== 0 || Server
->Comp(Queue
->Uri
) == false)
915 Server
= new ServerState(Queue
->Uri
,this);
918 // Connnect to the host
919 if (Server
->Open() == false)
926 SendReq(Queue
,Server
->Out
);
928 // Fetch the next URL header data from the server.
929 switch (Server
->RunHeaders())
934 // The header data is bad
937 _error
->Error("Bad header Data");
942 // The server closed a connection during the header get..
947 _error
->DumpErrors();
953 // Decide what to do.
955 Res
.Filename
= Queue
->DestFile
;
956 switch (DealWithHeaders(Res
,Server
))
958 // Ok, the file is Open
964 bool Result
= Server
->RunData();
966 // Close the file, destroy the FD object and timestamp it
974 UBuf
.actime
= Server
->Date
;
975 UBuf
.modtime
= Server
->Date
;
976 utime(Queue
->DestFile
.c_str(),&UBuf
);
978 // Send status to APT
981 Res
.MD5Sum
= Server
->In
.MD5
->Result();
997 // Hard server error, not found or something
1004 // Hard internal error, kill the connection and fail
1012 // We need to flush the data, the header is like a 404 w/ error text
1017 // Send to content to dev/null
1018 File
= new FileFd("/dev/null",FileFd::WriteExists
);
1026 Fail("Internal error");