]> git.saurik.com Git - apt.git/blame - methods/http.cc
implement generic config fallback for methods
[apt.git] / methods / http.cc
CommitLineData
be4401bf
AL
1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
2cbcabd8 3// $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
be4401bf
AL
4/* ######################################################################
5
1e3f4083 6 HTTP Acquire Method - This is the HTTP acquire method for APT.
be4401bf
AL
7
8 It uses HTTP/1.1 and many of the fancy options there-in, such as
e836f356
AL
9 pipelining, range, if-range and so on.
10
11 It is based on a doubly buffered select loop. A groupe of requests are
be4401bf
AL
12 fed into a single output buffer that is constantly fed out the
13 socket. This provides ideal pipelining as in many cases all of the
14 requests will fit into a single packet. The input socket is buffered
e836f356 15 the same way and fed into the fd for the file (may be a pipe in future).
be4401bf
AL
16
17 This double buffering provides fairly substantial transfer rates,
18 compared to wget the http method is about 4% faster. Most importantly,
19 when HTTP is compared with FTP as a protocol the speed difference is
20 huge. In tests over the internet from two sites to llug (via ATM) this
21 program got 230k/s sustained http transfer rates. FTP on the other
22 hand topped out at 170k/s. That combined with the time to setup the
23 FTP connection makes HTTP a vastly superior protocol.
24
25 ##################################################################### */
26 /*}}}*/
27// Include Files /*{{{*/
ea542140
DK
28#include <config.h>
29
be4401bf 30#include <apt-pkg/fileutl.h>
472ff00e 31#include <apt-pkg/configuration.h>
be4401bf 32#include <apt-pkg/error.h>
63b1700f 33#include <apt-pkg/hashes.h>
592b7800 34#include <apt-pkg/netrc.h>
453b82a3 35#include <apt-pkg/strutl.h>
c6ee61ea 36#include <apt-pkg/proxy.h>
be4401bf 37
453b82a3
DK
38#include <stddef.h>
39#include <stdlib.h>
40#include <sys/select.h>
41#include <cstring>
be4401bf
AL
42#include <sys/stat.h>
43#include <sys/time.h>
be4401bf
AL
44#include <unistd.h>
45#include <stdio.h>
65a1e968 46#include <errno.h>
42195eb2 47#include <iostream>
b123b0ba 48#include <sstream>
be4401bf 49
59b46c41 50#include "config.h"
0837bd25 51#include "connect.h"
be4401bf 52#include "http.h"
ea542140
DK
53
54#include <apti18n.h>
be4401bf 55 /*}}}*/
42195eb2 56using namespace std;
be4401bf 57
650faab0
DK
58unsigned long long CircleBuf::BwReadLimit=0;
59unsigned long long CircleBuf::BwTickReadData=0;
7c6e2dc7
MV
60struct timeval CircleBuf::BwReadTick={0,0};
61const unsigned int CircleBuf::BW_HZ=10;
d3e8fbb3 62
be4401bf
AL
63// CircleBuf::CircleBuf - Circular input buffer /*{{{*/
64// ---------------------------------------------------------------------
65/* */
30060442 66CircleBuf::CircleBuf(HttpMethod const * const Owner, unsigned long long Size)
9224ce3d 67 : Size(Size), Hash(NULL), TotalWriten(0)
be4401bf
AL
68{
69 Buf = new unsigned char[Size];
70 Reset();
7c6e2dc7 71
30060442 72 CircleBuf::BwReadLimit = Owner->ConfigFindI("Dl-Limit", 0) * 1024;
be4401bf
AL
73}
74 /*}}}*/
75// CircleBuf::Reset - Reset to the default state /*{{{*/
76// ---------------------------------------------------------------------
77/* */
78void CircleBuf::Reset()
79{
80 InP = 0;
81 OutP = 0;
82 StrPos = 0;
dcd5856b 83 TotalWriten = 0;
650faab0 84 MaxGet = (unsigned long long)-1;
be4401bf 85 OutQueue = string();
9224ce3d 86 if (Hash != NULL)
be4401bf 87 {
63b1700f 88 delete Hash;
9224ce3d 89 Hash = NULL;
d3e8fbb3
DK
90 }
91}
be4401bf
AL
92 /*}}}*/
93// CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
94// ---------------------------------------------------------------------
95/* This fills up the buffer with as much data as is in the FD, assuming it
96 is non-blocking.. */
97bool CircleBuf::Read(int Fd)
98{
99 while (1)
100 {
101 // Woops, buffer is full
102 if (InP - OutP == Size)
103 return true;
7c6e2dc7
MV
104
105 // what's left to read in this tick
9ce3cfc9 106 unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
7c6e2dc7
MV
107
108 if(CircleBuf::BwReadLimit) {
109 struct timeval now;
110 gettimeofday(&now,0);
111
650faab0 112 unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
7c6e2dc7
MV
113 now.tv_usec-CircleBuf::BwReadTick.tv_usec;
114 if(d > 1000000/BW_HZ) {
115 CircleBuf::BwReadTick = now;
116 CircleBuf::BwTickReadData = 0;
117 }
118
119 if(CircleBuf::BwTickReadData >= BwReadMax) {
120 usleep(1000000/BW_HZ);
121 return true;
122 }
123 }
124
be4401bf 125 // Write the buffer segment
650faab0 126 ssize_t Res;
7c6e2dc7
MV
127 if(CircleBuf::BwReadLimit) {
128 Res = read(Fd,Buf + (InP%Size),
129 BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
130 } else
131 Res = read(Fd,Buf + (InP%Size),LeftRead());
be4401bf 132
7c6e2dc7
MV
133 if(Res > 0 && BwReadLimit > 0)
134 CircleBuf::BwTickReadData += Res;
135
be4401bf
AL
136 if (Res == 0)
137 return false;
138 if (Res < 0)
139 {
140 if (errno == EAGAIN)
141 return true;
142 return false;
143 }
144
145 if (InP == 0)
146 gettimeofday(&Start,0);
147 InP += Res;
148 }
149}
150 /*}}}*/
151// CircleBuf::Read - Put the string into the buffer /*{{{*/
152// ---------------------------------------------------------------------
153/* This will hold the string in and fill the buffer with it as it empties */
154bool CircleBuf::Read(string Data)
155{
156 OutQueue += Data;
157 FillOut();
158 return true;
159}
160 /*}}}*/
161// CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
162// ---------------------------------------------------------------------
163/* */
164void CircleBuf::FillOut()
165{
166 if (OutQueue.empty() == true)
167 return;
168 while (1)
169 {
170 // Woops, buffer is full
171 if (InP - OutP == Size)
172 return;
173
174 // Write the buffer segment
650faab0 175 unsigned long long Sz = LeftRead();
be4401bf
AL
176 if (OutQueue.length() - StrPos < Sz)
177 Sz = OutQueue.length() - StrPos;
42195eb2 178 memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
be4401bf
AL
179
180 // Advance
181 StrPos += Sz;
182 InP += Sz;
183 if (OutQueue.length() == StrPos)
184 {
185 StrPos = 0;
186 OutQueue = "";
187 return;
188 }
189 }
190}
191 /*}}}*/
192// CircleBuf::Write - Write from the buffer into a FD /*{{{*/
193// ---------------------------------------------------------------------
194/* This empties the buffer into the FD. */
195bool CircleBuf::Write(int Fd)
196{
197 while (1)
198 {
199 FillOut();
200
201 // Woops, buffer is empty
202 if (OutP == InP)
203 return true;
204
205 if (OutP == MaxGet)
206 return true;
207
208 // Write the buffer segment
650faab0 209 ssize_t Res;
be4401bf
AL
210 Res = write(Fd,Buf + (OutP%Size),LeftWrite());
211
212 if (Res == 0)
213 return false;
214 if (Res < 0)
215 {
216 if (errno == EAGAIN)
217 return true;
218
219 return false;
220 }
dcd5856b
MV
221
222 TotalWriten += Res;
be4401bf 223
9224ce3d 224 if (Hash != NULL)
63b1700f 225 Hash->Add(Buf + (OutP%Size),Res);
be4401bf
AL
226
227 OutP += Res;
228 }
229}
230 /*}}}*/
231// CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
232// ---------------------------------------------------------------------
233/* This copies till the first empty line */
234bool CircleBuf::WriteTillEl(string &Data,bool Single)
235{
236 // We cheat and assume it is unneeded to have more than one buffer load
650faab0 237 for (unsigned long long I = OutP; I < InP; I++)
be4401bf
AL
238 {
239 if (Buf[I%Size] != '\n')
240 continue;
2cbcabd8 241 ++I;
be4401bf
AL
242
243 if (Single == false)
244 {
2cbcabd8
AL
245 if (I < InP && Buf[I%Size] == '\r')
246 ++I;
927c393f
MV
247 if (I >= InP || Buf[I%Size] != '\n')
248 continue;
249 ++I;
be4401bf
AL
250 }
251
be4401bf
AL
252 Data = "";
253 while (OutP < I)
254 {
650faab0 255 unsigned long long Sz = LeftWrite();
be4401bf
AL
256 if (Sz == 0)
257 return false;
927c393f 258 if (I - OutP < Sz)
be4401bf
AL
259 Sz = I - OutP;
260 Data += string((char *)(Buf + (OutP%Size)),Sz);
261 OutP += Sz;
262 }
263 return true;
264 }
265 return false;
266}
267 /*}}}*/
268// CircleBuf::Stats - Print out stats information /*{{{*/
269// ---------------------------------------------------------------------
270/* */
271void CircleBuf::Stats()
272{
273 if (InP == 0)
274 return;
275
276 struct timeval Stop;
277 gettimeofday(&Stop,0);
278/* float Diff = Stop.tv_sec - Start.tv_sec +
279 (float)(Stop.tv_usec - Start.tv_usec)/1000000;
280 clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
281}
282 /*}}}*/
472ff00e
DK
283CircleBuf::~CircleBuf()
284{
285 delete [] Buf;
286 delete Hash;
287}
be4401bf 288
7330f4df 289// HttpServerState::HttpServerState - Constructor /*{{{*/
30060442 290HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, 64*1024), Out(Owner, 4*1024)
be4401bf 291{
30060442 292 TimeOut = Owner->ConfigFindI("Timeout", TimeOut);
be4401bf
AL
293 Reset();
294}
295 /*}}}*/
7330f4df 296// HttpServerState::Open - Open a connection to the server /*{{{*/
be4401bf
AL
297// ---------------------------------------------------------------------
298/* This opens a connection to the server. */
7330f4df 299bool HttpServerState::Open()
be4401bf 300{
92e889c8
AL
301 // Use the already open connection if possible.
302 if (ServerFd != -1)
303 return true;
304
be4401bf 305 Close();
492f957a
AL
306 In.Reset();
307 Out.Reset();
e836f356
AL
308 Persistent = true;
309
492f957a 310 // Determine the proxy setting
c6ee61ea 311 AutoDetectProxy(ServerName);
30060442 312 string SpecificProxy = Owner->ConfigFind("Proxy::" + ServerName.Host, "");
788a8f42 313 if (!SpecificProxy.empty())
492f957a 314 {
788a8f42
EL
315 if (SpecificProxy == "DIRECT")
316 Proxy = "";
317 else
318 Proxy = SpecificProxy;
352c2768 319 }
492f957a 320 else
788a8f42 321 {
30060442 322 string DefProxy = Owner->ConfigFind("Proxy", "");
788a8f42
EL
323 if (!DefProxy.empty())
324 {
325 Proxy = DefProxy;
326 }
327 else
328 {
329 char* result = getenv("http_proxy");
330 Proxy = result ? result : "";
331 }
332 }
352c2768 333
f8081133 334 // Parse no_proxy, a , separated list of domains
9e2a06ff
AL
335 if (getenv("no_proxy") != 0)
336 {
f8081133
AL
337 if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
338 Proxy = "";
339 }
340
492f957a 341 // Determine what host and port to use based on the proxy settings
934b6582 342 int Port = 0;
492f957a 343 string Host;
dd1fd92b 344 if (Proxy.empty() == true || Proxy.Host.empty() == true)
be4401bf 345 {
92e889c8
AL
346 if (ServerName.Port != 0)
347 Port = ServerName.Port;
be4401bf
AL
348 Host = ServerName.Host;
349 }
ece81b75
DK
350 else if (Proxy.Access != "http")
351 return _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str());
be4401bf
AL
352 else
353 {
92e889c8
AL
354 if (Proxy.Port != 0)
355 Port = Proxy.Port;
be4401bf
AL
356 Host = Proxy.Host;
357 }
358
0837bd25 359 // Connect to the remote server
9505213b 360 if (Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner) == false)
0837bd25 361 return false;
3000ccea 362
be4401bf
AL
363 return true;
364}
365 /*}}}*/
7330f4df 366// HttpServerState::Close - Close a connection to the server /*{{{*/
be4401bf
AL
367// ---------------------------------------------------------------------
368/* */
7330f4df 369bool HttpServerState::Close()
be4401bf
AL
370{
371 close(ServerFd);
372 ServerFd = -1;
be4401bf
AL
373 return true;
374}
375 /*}}}*/
7330f4df
DK
376// HttpServerState::RunData - Transfer the data from the socket /*{{{*/
377bool HttpServerState::RunData(FileFd * const File)
be4401bf
AL
378{
379 State = Data;
380
381 // Chunked transfer encoding is fun..
382 if (Encoding == Chunked)
383 {
384 while (1)
385 {
386 // Grab the block size
387 bool Last = true;
388 string Data;
389 In.Limit(-1);
390 do
391 {
392 if (In.WriteTillEl(Data,true) == true)
393 break;
394 }
7330f4df 395 while ((Last = Go(false, File)) == true);
be4401bf
AL
396
397 if (Last == false)
398 return false;
399
400 // See if we are done
650faab0 401 unsigned long long Len = strtoull(Data.c_str(),0,16);
be4401bf
AL
402 if (Len == 0)
403 {
404 In.Limit(-1);
405
406 // We have to remove the entity trailer
407 Last = true;
408 do
409 {
410 if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
411 break;
412 }
7330f4df 413 while ((Last = Go(false, File)) == true);
be4401bf
AL
414 if (Last == false)
415 return false;
e1b96638 416 return !_error->PendingError();
be4401bf
AL
417 }
418
419 // Transfer the block
420 In.Limit(Len);
7330f4df 421 while (Go(true, File) == true)
be4401bf
AL
422 if (In.IsLimit() == true)
423 break;
424
425 // Error
426 if (In.IsLimit() == false)
427 return false;
428
429 // The server sends an extra new line before the next block specifier..
430 In.Limit(-1);
431 Last = true;
432 do
433 {
434 if (In.WriteTillEl(Data,true) == true)
435 break;
436 }
7330f4df 437 while ((Last = Go(false, File)) == true);
be4401bf
AL
438 if (Last == false)
439 return false;
92e889c8 440 }
be4401bf
AL
441 }
442 else
443 {
444 /* Closes encoding is used when the server did not specify a size, the
445 loss of the connection means we are done */
99968cf7 446 if (JunkSize != 0)
ed793a19 447 In.Limit(JunkSize);
99968cf7 448 else if (DownloadSize != 0)
ceafe8a6 449 In.Limit(DownloadSize);
99968cf7
DK
450 else if (Persistent == false)
451 In.Limit(-1);
be4401bf
AL
452
453 // Just transfer the whole block.
454 do
455 {
456 if (In.IsLimit() == false)
457 continue;
458
459 In.Limit(-1);
e1b96638 460 return !_error->PendingError();
be4401bf 461 }
7330f4df 462 while (Go(true, File) == true);
be4401bf
AL
463 }
464
7330f4df 465 return Owner->Flush() && !_error->PendingError();
be4401bf
AL
466}
467 /*}}}*/
57401c48
DK
468bool HttpServerState::RunDataToDevNull() /*{{{*/
469{
470 FileFd DevNull("/dev/null", FileFd::WriteOnly);
471 return RunData(&DevNull);
472}
473 /*}}}*/
7330f4df 474bool HttpServerState::ReadHeaderLines(std::string &Data) /*{{{*/
be4401bf 475{
7330f4df
DK
476 return In.WriteTillEl(Data);
477}
478 /*}}}*/
479bool HttpServerState::LoadNextResponse(bool const ToFile, FileFd * const File)/*{{{*/
480{
481 return Go(ToFile, File);
482}
483 /*}}}*/
484bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/
485{
486 return Out.Read(Data);
487}
488 /*}}}*/
a02db58f 489APT_PURE bool HttpServerState::IsOpen() /*{{{*/
7330f4df
DK
490{
491 return (ServerFd != -1);
492}
493 /*}}}*/
34faa8f7 494bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/
7330f4df
DK
495{
496 delete In.Hash;
9224ce3d 497 In.Hash = new Hashes(ExpectedHashes);
34faa8f7 498 return true;
7330f4df
DK
499}
500 /*}}}*/
34faa8f7 501
a02db58f 502APT_PURE Hashes * HttpServerState::GetHashes() /*{{{*/
7330f4df
DK
503{
504 return In.Hash;
505}
506 /*}}}*/
507// HttpServerState::Die - The server has closed the connection. /*{{{*/
44605518 508bool HttpServerState::Die(FileFd * const File)
7330f4df
DK
509{
510 unsigned int LErrno = errno;
be4401bf 511
7330f4df
DK
512 // Dump the buffer to the file
513 if (State == ServerState::Data)
be4401bf 514 {
44605518
DK
515 if (File == nullptr)
516 return true;
7330f4df
DK
517 // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
518 // can't be set
44605518
DK
519 if (File->Name() != "/dev/null")
520 SetNonBlock(File->Fd(),false);
7330f4df 521 while (In.WriteSpace() == true)
be4401bf 522 {
44605518 523 if (In.Write(File->Fd()) == false)
7330f4df 524 return _error->Errno("write",_("Error writing to the file"));
e836f356 525
7330f4df
DK
526 // Done
527 if (In.IsLimit() == true)
528 return true;
e836f356 529 }
7330f4df 530 }
b2e465d6 531
7330f4df
DK
532 // See if this is because the server finished the data stream
533 if (In.IsLimit() == false && State != HttpServerState::Header &&
117038ba 534 Persistent == true)
be4401bf 535 {
7330f4df
DK
536 Close();
537 if (LErrno == 0)
538 return _error->Error(_("Error reading from server. Remote end closed connection"));
539 errno = LErrno;
540 return _error->Errno("read",_("Error reading from server"));
be4401bf 541 }
7330f4df 542 else
92e889c8 543 {
7330f4df
DK
544 In.Limit(-1);
545
546 // Nothing left in the buffer
547 if (In.WriteSpace() == false)
548 return false;
549
550 // We may have got multiple responses back in one packet..
551 Close();
92e889c8
AL
552 return true;
553 }
331e8396 554
7330f4df
DK
555 return false;
556}
557 /*}}}*/
558// HttpServerState::Flush - Dump the buffer into the file /*{{{*/
559// ---------------------------------------------------------------------
560/* This takes the current input buffer from the Server FD and writes it
561 into the file */
562bool HttpServerState::Flush(FileFd * const File)
563{
564 if (File != NULL)
565 {
566 // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
567 // can't be set
568 if (File->Name() != "/dev/null")
569 SetNonBlock(File->Fd(),false);
570 if (In.WriteSpace() == false)
571 return true;
572
573 while (In.WriteSpace() == true)
331e8396 574 {
7330f4df
DK
575 if (In.Write(File->Fd()) == false)
576 return _error->Errno("write",_("Error writing to file"));
577 if (In.IsLimit() == true)
578 return true;
331e8396 579 }
7330f4df 580
117038ba 581 if (In.IsLimit() == true || Persistent == false)
7330f4df 582 return true;
be4401bf 583 }
7330f4df
DK
584 return false;
585}
586 /*}}}*/
587// HttpServerState::Go - Run a single loop /*{{{*/
588// ---------------------------------------------------------------------
589/* This runs the select loop over the server FDs, Output file FDs and
590 stdin. */
591bool HttpServerState::Go(bool ToFile, FileFd * const File)
592{
593 // Server has closed the connection
594 if (ServerFd == -1 && (In.WriteSpace() == false ||
595 ToFile == false))
596 return false;
597
598 fd_set rfds,wfds;
599 FD_ZERO(&rfds);
600 FD_ZERO(&wfds);
601
602 /* Add the server. We only send more requests if the connection will
603 be persisting */
604 if (Out.WriteSpace() == true && ServerFd != -1
605 && Persistent == true)
606 FD_SET(ServerFd,&wfds);
607 if (In.ReadSpace() == true && ServerFd != -1)
608 FD_SET(ServerFd,&rfds);
be4401bf 609
7330f4df
DK
610 // Add the file
611 int FileFD = -1;
612 if (File != NULL)
613 FileFD = File->Fd();
614
615 if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
616 FD_SET(FileFD,&wfds);
617
618 // Add stdin
30060442 619 if (Owner->ConfigFindB("DependOnSTDIN", true) == true)
7330f4df
DK
620 FD_SET(STDIN_FILENO,&rfds);
621
622 // Figure out the max fd
623 int MaxFd = FileFD;
624 if (MaxFd < ServerFd)
625 MaxFd = ServerFd;
626
627 // Select
628 struct timeval tv;
629 tv.tv_sec = TimeOut;
630 tv.tv_usec = 0;
631 int Res = 0;
632 if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
be4401bf 633 {
7330f4df
DK
634 if (errno == EINTR)
635 return true;
636 return _error->Errno("select",_("Select failed"));
be4401bf 637 }
7330f4df
DK
638
639 if (Res == 0)
e836f356 640 {
7330f4df 641 _error->Error(_("Connection timed out"));
44605518 642 return Die(File);
e836f356
AL
643 }
644
7330f4df
DK
645 // Handle server IO
646 if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
be4401bf 647 {
7330f4df
DK
648 errno = 0;
649 if (In.Read(ServerFd) == false)
44605518 650 return Die(File);
7330f4df
DK
651 }
652
653 if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
654 {
655 errno = 0;
656 if (Out.Write(ServerFd) == false)
44605518 657 return Die(File);
be4401bf
AL
658 }
659
7330f4df
DK
660 // Send data to the file
661 if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
15d7e515 662 {
7330f4df
DK
663 if (In.Write(FileFD) == false)
664 return _error->Errno("write",_("Error writing to output file"));
15d7e515
MV
665 }
666
c48eea97 667 if (MaximumSize > 0 && File && File->Tell() > MaximumSize)
a2d40703 668 {
ee279506 669 Owner->SetFailReason("MaximumSizeExceeded");
dcd5856b 670 return _error->Error("Writing more data than expected (%llu > %llu)",
c48eea97 671 File->Tell(), MaximumSize);
a2d40703 672 }
dcd5856b 673
7330f4df
DK
674 // Handle commands from APT
675 if (FD_ISSET(STDIN_FILENO,&rfds))
676 {
677 if (Owner->Run(true) != -1)
678 exit(100);
679 }
680
be4401bf
AL
681 return true;
682}
683 /*}}}*/
684
685// HttpMethod::SendReq - Send the HTTP request /*{{{*/
686// ---------------------------------------------------------------------
687/* This places the http request in the outbound buffer */
7330f4df 688void HttpMethod::SendReq(FetchItem *Itm)
be4401bf
AL
689{
690 URI Uri = Itm->Uri;
30060442
DK
691 {
692 auto const plus = Binary.find('+');
693 if (plus != std::string::npos)
694 Uri.Access = Binary.substr(plus + 1);
695 }
c1a22377 696
be4401bf 697 // The HTTP server expects a hostname with a trailing :port
b123b0ba 698 std::stringstream Req;
5b63d2a9
MV
699 string ProperHost;
700
701 if (Uri.Host.find(':') != string::npos)
702 ProperHost = '[' + Uri.Host + ']';
703 else
704 ProperHost = Uri.Host;
f2380a78
DK
705
706 /* RFC 2616 ยง5.1.2 requires absolute URIs for requests to proxies,
707 but while its a must for all servers to accept absolute URIs,
708 it is assumed clients will sent an absolute path for non-proxies */
709 std::string requesturi;
7330f4df 710 if (Server->Proxy.empty() == true || Server->Proxy.Host.empty())
f2380a78
DK
711 requesturi = Uri.Path;
712 else
30060442 713 requesturi = Uri;
f2380a78
DK
714
715 // The "+" is encoded as a workaround for a amazon S3 bug
716 // see LP bugs #1003633 and #1086997.
717 requesturi = QuoteString(requesturi, "+~ ");
718
2b9c9b7f
RG
719 /* Build the request. No keep-alive is included as it is the default
720 in 1.1, can cause problems with proxies, and we are an HTTP/1.1
721 client anyway.
722 C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
b123b0ba
DK
723 Req << "GET " << requesturi << " HTTP/1.1\r\n";
724 if (Uri.Port != 0)
b58e2c7c 725 Req << "Host: " << ProperHost << ":" << std::to_string(Uri.Port) << "\r\n";
b123b0ba
DK
726 else
727 Req << "Host: " << ProperHost << "\r\n";
f2380a78 728
c9cd3b70 729 // generate a cache control header (if needed)
30060442 730 if (ConfigFindB("No-Cache",false) == true)
b123b0ba
DK
731 Req << "Cache-Control: no-cache\r\n"
732 << "Pragma: no-cache\r\n";
733 else if (Itm->IndexFile == true)
30060442
DK
734 Req << "Cache-Control: max-age=" << std::to_string(ConfigFindI("Max-Age", 0)) << "\r\n";
735 else if (ConfigFindB("No-Store", false) == true)
b123b0ba 736 Req << "Cache-Control: no-store\r\n";
106e6740 737
6f4501f9 738 // If we ask for uncompressed files servers might respond with content-
1e3f4083 739 // negotiation which lets us end up with compressed files we do not support,
6f4501f9
DK
740 // see 657029, 657560 and co, so if we have no extension on the request
741 // ask for text only. As a sidenote: If there is nothing to negotate servers
742 // seem to be nice and ignore it.
30060442 743 if (ConfigFindB("SendAccept", true) == true)
6f4501f9
DK
744 {
745 size_t const filepos = Itm->Uri.find_last_of('/');
746 string const file = Itm->Uri.substr(filepos + 1);
747 if (flExtension(file) == file)
b123b0ba 748 Req << "Accept: text/*\r\n";
6f4501f9
DK
749 }
750
b123b0ba 751 // Check for a partial file and send if-queries accordingly
be4401bf
AL
752 struct stat SBuf;
753 if (stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
7303e11f 754 Req << "Range: bytes=" << std::to_string(SBuf.st_size) << "-\r\n"
0b45b6e5 755 << "If-Range: " << TimeRFC1123(SBuf.st_mtime, false) << "\r\n";
b123b0ba 756 else if (Itm->LastModified != 0)
0b45b6e5 757 Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified, false).c_str() << "\r\n";
be4401bf 758
7330f4df 759 if (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false)
b123b0ba
DK
760 Req << "Proxy-Authorization: Basic "
761 << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n";
be4401bf 762
1de1f703 763 maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
b2e465d6 764 if (Uri.User.empty() == false || Uri.Password.empty() == false)
b123b0ba
DK
765 Req << "Authorization: Basic "
766 << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n";
767
30060442 768 Req << "User-Agent: " << ConfigFind("User-Agent",
b123b0ba
DK
769 "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n";
770
771 Req << "\r\n";
772
c98b1307 773 if (Debug == true)
7b734b09 774 cerr << Req.str() << endl;
c1a22377 775
b123b0ba 776 Server->WriteResponse(Req.str());
be4401bf
AL
777}
778 /*}}}*/
830a1b8c 779std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/
7330f4df 780{
830a1b8c 781 return std::unique_ptr<ServerState>(new HttpServerState(uri, this));
7330f4df
DK
782}
783 /*}}}*/
fd46d305
DK
784void HttpMethod::RotateDNS() /*{{{*/
785{
786 ::RotateDNS();
787}
788 /*}}}*/
4bba5a88
DK
789ServerMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res)/*{{{*/
790{
791 auto ret = ServerMethod::DealWithHeaders(Res);
792 if (ret != ServerMethod::FILE_IS_OPEN)
793 return ret;
794
795 // Open the file
796 delete File;
797 File = new FileFd(Queue->DestFile,FileFd::WriteAny);
798 if (_error->PendingError() == true)
799 return ERROR_NOT_FROM_SERVER;
800
801 FailFile = Queue->DestFile;
802 FailFile.c_str(); // Make sure we don't do a malloc in the signal handler
803 FailFd = File->Fd();
804 FailTime = Server->Date;
805
806 if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
807 {
808 _error->Errno("read",_("Problem hashing file"));
809 return ERROR_NOT_FROM_SERVER;
810 }
811 if (Server->StartPos > 0)
812 Res.ResumePoint = Server->StartPos;
813
814 SetNonBlock(File->Fd(),true);
815 return FILE_IS_OPEN;
816}
817 /*}}}*/
30060442
DK
818HttpMethod::HttpMethod(std::string &&pProg) : ServerMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/
819{
820 auto addName = std::inserter(methodNames, methodNames.begin());
821 if (Binary != "http")
822 addName = "http";
823 auto const plus = Binary.find('+');
824 if (plus != std::string::npos)
825 addName = Binary.substr(0, plus);
826 File = 0;
827 Server = 0;
828}
829 /*}}}*/