1 //-*- mode: cpp; mode: fold -*-
3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
4 /* ######################################################################
6 HTTPS Acquire Method - This is the HTTPS acquire method for APT.
10 ##################################################################### */
12 // Include Files /*{{{*/
15 #include <apt-pkg/fileutl.h>
16 #include <apt-pkg/error.h>
17 #include <apt-pkg/hashes.h>
18 #include <apt-pkg/netrc.h>
19 #include <apt-pkg/configuration.h>
20 #include <apt-pkg/macros.h>
21 #include <apt-pkg/strutl.h>
22 #include <apt-pkg/proxy.h>
42 struct APT_HIDDEN CURLUserPointer
{
43 HttpsMethod
* const https
;
44 HttpsMethod::FetchResult
* const Res
;
45 HttpsMethod::FetchItem
const * const Itm
;
46 RequestState
* const Req
;
47 CURLUserPointer(HttpsMethod
* const https
, HttpsMethod::FetchResult
* const Res
,
48 HttpsMethod::FetchItem
const * const Itm
, RequestState
* const Req
) : https(https
), Res(Res
), Itm(Itm
), Req(Req
) {}
52 HttpsMethod::parse_header(void *buffer
, size_t size
, size_t nmemb
, void *userp
)
54 size_t len
= size
* nmemb
;
55 CURLUserPointer
*me
= static_cast<CURLUserPointer
*>(userp
);
56 std::string
line((char*) buffer
, len
);
57 for (--len
; len
> 0; --len
)
58 if (isspace_ascii(line
[len
]) == 0)
65 if (line
.empty() == true)
67 if (me
->Req
->File
.Open(me
->Itm
->DestFile
, FileFd::WriteAny
) == false)
68 return ERROR_NOT_FROM_SERVER
;
70 me
->Req
->JunkSize
= 0;
71 if (me
->Req
->Result
!= 416 && me
->Req
->StartPos
!= 0)
73 else if (me
->Req
->Result
== 416)
75 bool partialHit
= false;
76 if (me
->Itm
->ExpectedHashes
.usable() == true)
78 Hashes
resultHashes(me
->Itm
->ExpectedHashes
);
79 FileFd
file(me
->Itm
->DestFile
, FileFd::ReadOnly
);
80 me
->Req
->TotalFileSize
= file
.FileSize();
81 me
->Req
->Date
= file
.ModificationTime();
82 resultHashes
.AddFD(file
);
83 HashStringList
const hashList
= resultHashes
.GetHashStringList();
84 partialHit
= (me
->Itm
->ExpectedHashes
== hashList
);
86 else if (me
->Req
->Result
== 416 && me
->Req
->TotalFileSize
== me
->Req
->File
.FileSize())
89 if (partialHit
== true)
91 me
->Req
->Result
= 200;
92 me
->Req
->StartPos
= me
->Req
->TotalFileSize
;
93 // the actual size is not important for https as curl will deal with it
94 // by itself and e.g. doesn't bother us with transport-encoding…
95 me
->Req
->JunkSize
= std::numeric_limits
<unsigned long long>::max();
98 me
->Req
->StartPos
= 0;
101 me
->Req
->StartPos
= 0;
103 me
->Res
->LastModified
= me
->Req
->Date
;
104 me
->Res
->Size
= me
->Req
->TotalFileSize
;
105 me
->Res
->ResumePoint
= me
->Req
->StartPos
;
107 // we expect valid data, so tell our caller we get the file now
108 if (me
->Req
->Result
>= 200 && me
->Req
->Result
< 300)
110 if (me
->Res
->Size
!= 0 && me
->Res
->Size
> me
->Res
->ResumePoint
)
111 me
->https
->URIStart(*me
->Res
);
112 if (me
->Req
->AddPartialFileToHashes(me
->Req
->File
) == false)
116 me
->Req
->JunkSize
= std::numeric_limits
<decltype(me
->Req
->JunkSize
)>::max();
118 else if (me
->Req
->HeaderLine(line
) == false)
125 HttpsMethod::write_data(void *buffer
, size_t size
, size_t nmemb
, void *userp
)
127 CURLUserPointer
*me
= static_cast<CURLUserPointer
*>(userp
);
128 size_t buffer_size
= size
* nmemb
;
129 // we don't need to count the junk here, just drop anything we get as
130 // we don't always know how long it would be, e.g. in chunked encoding.
131 if (me
->Req
->JunkSize
!= 0)
134 if(me
->Req
->File
.Write(buffer
, buffer_size
) != true)
137 if(me
->https
->Queue
->MaximumSize
> 0)
139 unsigned long long const TotalWritten
= me
->Req
->File
.Tell();
140 if (TotalWritten
> me
->https
->Queue
->MaximumSize
)
142 me
->https
->SetFailReason("MaximumSizeExceeded");
143 _error
->Error("Writing more data than expected (%llu > %llu)",
144 TotalWritten
, me
->https
->Queue
->MaximumSize
);
149 if (me
->https
->Server
->GetHashes()->Add((unsigned char const * const)buffer
, buffer_size
) == false)
155 // HttpsServerState::HttpsServerState - Constructor /*{{{*/
156 HttpsServerState::HttpsServerState(URI Srv
,HttpsMethod
* Owner
) : ServerState(Srv
, Owner
), Hash(NULL
)
158 TimeOut
= Owner
->ConfigFindI("Timeout", TimeOut
);
162 bool HttpsServerState::InitHashes(HashStringList
const &ExpectedHashes
) /*{{{*/
165 Hash
= new Hashes(ExpectedHashes
);
169 APT_PURE Hashes
* HttpsServerState::GetHashes() /*{{{*/
175 bool HttpsMethod::SetupProxy() /*{{{*/
177 URI ServerName
= Queue
->Uri
;
179 // Determine the proxy setting
180 AutoDetectProxy(ServerName
);
182 // Curl should never read proxy settings from the environment, as
183 // we determine which proxy to use. Do this for consistency among
184 // methods and prevent an environment variable overriding a
185 // no-proxy ("DIRECT") setting in apt.conf.
186 curl_easy_setopt(curl
, CURLOPT_PROXY
, "");
188 // Determine the proxy setting - try https first, fallback to http and use env at last
189 string UseProxy
= ConfigFind("Proxy::" + ServerName
.Host
, "");
190 if (UseProxy
.empty() == true)
191 UseProxy
= ConfigFind("Proxy", "");
192 // User wants to use NO proxy, so nothing to setup
193 if (UseProxy
== "DIRECT")
196 // Parse no_proxy, a comma (,) separated list of domains we don't want to use
197 // a proxy for so we stop right here if it is in the list
198 if (getenv("no_proxy") != 0 && CheckDomainList(ServerName
.Host
,getenv("no_proxy")) == true)
201 if (UseProxy
.empty() == true)
203 const char* result
= nullptr;
204 if (std::find(methodNames
.begin(), methodNames
.end(), "https") != methodNames
.end())
205 result
= getenv("https_proxy");
206 // FIXME: Fall back to http_proxy is to remain compatible with
207 // existing setups and behaviour of apt.conf. This should be
208 // deprecated in the future (including apt.conf). Most other
209 // programs do not fall back to http proxy settings and neither
211 if (result
== nullptr && std::find(methodNames
.begin(), methodNames
.end(), "http") != methodNames
.end())
212 result
= getenv("http_proxy");
213 UseProxy
= result
== nullptr ? "" : result
;
216 // Determine what host and port to use based on the proxy settings
217 if (UseProxy
.empty() == false)
220 AddProxyAuth(Proxy
, ServerName
);
222 if (Proxy
.Access
== "socks5h")
223 curl_easy_setopt(curl
, CURLOPT_PROXYTYPE
, CURLPROXY_SOCKS5_HOSTNAME
);
224 else if (Proxy
.Access
== "socks5")
225 curl_easy_setopt(curl
, CURLOPT_PROXYTYPE
, CURLPROXY_SOCKS5
);
226 else if (Proxy
.Access
== "socks4a")
227 curl_easy_setopt(curl
, CURLOPT_PROXYTYPE
, CURLPROXY_SOCKS4A
);
228 else if (Proxy
.Access
== "socks")
229 curl_easy_setopt(curl
, CURLOPT_PROXYTYPE
, CURLPROXY_SOCKS4
);
230 else if (Proxy
.Access
== "http" || Proxy
.Access
== "https")
231 curl_easy_setopt(curl
, CURLOPT_PROXYTYPE
, CURLPROXY_HTTP
);
236 curl_easy_setopt(curl
, CURLOPT_PROXYPORT
, Proxy
.Port
);
237 curl_easy_setopt(curl
, CURLOPT_PROXY
, Proxy
.Host
.c_str());
238 if (Proxy
.User
.empty() == false || Proxy
.Password
.empty() == false)
240 curl_easy_setopt(curl
, CURLOPT_PROXYUSERNAME
, Proxy
.User
.c_str());
241 curl_easy_setopt(curl
, CURLOPT_PROXYPASSWORD
, Proxy
.Password
.c_str());
246 // HttpsMethod::Fetch - Fetch an item /*{{{*/
247 // ---------------------------------------------------------------------
248 /* This adds an item to the pipeline. We keep the pipeline at a fixed
250 bool HttpsMethod::Fetch(FetchItem
*Itm
)
253 struct curl_slist
*headers
=NULL
;
254 char curl_errorstr
[CURL_ERROR_SIZE
];
256 setPostfixForMethodNames(Uri
.Host
.c_str());
257 AllowRedirect
= ConfigFindB("AllowRedirect", true);
258 Debug
= DebugEnabled();
261 // - http::Pipeline-Depth
262 // - error checking/reporting
263 // - more debug options? (CURLOPT_DEBUGFUNCTION?)
265 auto const plus
= Binary
.find('+');
266 if (plus
!= std::string::npos
)
267 Uri
.Access
= Binary
.substr(plus
+ 1);
270 curl_easy_reset(curl
);
271 if (SetupProxy() == false)
272 return _error
->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy
).c_str());
274 maybe_add_auth (Uri
, _config
->FindFile("Dir::Etc::netrc"));
275 if (Server
== nullptr || Server
->Comp(Itm
->Uri
) == false)
276 Server
= CreateServerState(Itm
->Uri
);
279 RequestState
Req(this, Server
.get());
280 CURLUserPointer
userp(this, &Res
, Itm
, &Req
);
282 curl_easy_setopt(curl
, CURLOPT_URL
, static_cast<string
>(Uri
).c_str());
283 curl_easy_setopt(curl
, CURLOPT_HEADERFUNCTION
, parse_header
);
284 curl_easy_setopt(curl
, CURLOPT_WRITEHEADER
, &userp
);
285 curl_easy_setopt(curl
, CURLOPT_WRITEFUNCTION
, write_data
);
286 curl_easy_setopt(curl
, CURLOPT_WRITEDATA
, &userp
);
288 curl_easy_setopt(curl
, CURLOPT_NOPROGRESS
, true);
289 curl_easy_setopt(curl
, CURLOPT_FILETIME
, true);
290 curl_easy_setopt(curl
, CURLOPT_FOLLOWLOCATION
, 0);
292 if (std::find(methodNames
.begin(), methodNames
.end(), "https") != methodNames
.end())
294 curl_easy_setopt(curl
, CURLOPT_PROTOCOLS
, CURLPROTO_HTTPS
);
295 curl_easy_setopt(curl
, CURLOPT_REDIR_PROTOCOLS
, CURLPROTO_HTTPS
);
297 // File containing the list of trusted CA.
298 std::string
const cainfo
= ConfigFind("CaInfo", "");
299 if(cainfo
.empty() == false)
300 curl_easy_setopt(curl
, CURLOPT_CAINFO
, cainfo
.c_str());
301 // Check server certificate against previous CA list ...
302 curl_easy_setopt(curl
, CURLOPT_SSL_VERIFYPEER
, ConfigFindB("Verify-Peer", true) ? 1 : 0);
303 // ... and hostname against cert CN or subjectAltName
304 curl_easy_setopt(curl
, CURLOPT_SSL_VERIFYHOST
, ConfigFindB("Verify-Host", true) ? 2 : 0);
305 // Also enforce issuer of server certificate using its cert
306 std::string
const issuercert
= ConfigFind("IssuerCert", "");
307 if(issuercert
.empty() == false)
308 curl_easy_setopt(curl
, CURLOPT_ISSUERCERT
, issuercert
.c_str());
309 // For client authentication, certificate file ...
310 std::string
const pem
= ConfigFind("SslCert", "");
311 if(pem
.empty() == false)
312 curl_easy_setopt(curl
, CURLOPT_SSLCERT
, pem
.c_str());
313 // ... and associated key.
314 std::string
const key
= ConfigFind("SslKey", "");
315 if(key
.empty() == false)
316 curl_easy_setopt(curl
, CURLOPT_SSLKEY
, key
.c_str());
317 // Allow forcing SSL version to SSLv3 or TLSv1
318 long final_version
= CURL_SSLVERSION_DEFAULT
;
319 std::string
const sslversion
= ConfigFind("SslForceVersion", "");
320 if(sslversion
== "TLSv1")
321 final_version
= CURL_SSLVERSION_TLSv1
;
322 else if(sslversion
== "TLSv1.0")
323 final_version
= CURL_SSLVERSION_TLSv1_0
;
324 else if(sslversion
== "TLSv1.1")
325 final_version
= CURL_SSLVERSION_TLSv1_1
;
326 else if(sslversion
== "TLSv1.2")
327 final_version
= CURL_SSLVERSION_TLSv1_2
;
328 else if(sslversion
== "SSLv3")
329 final_version
= CURL_SSLVERSION_SSLv3
;
330 curl_easy_setopt(curl
, CURLOPT_SSLVERSION
, final_version
);
332 std::string
const crlfile
= ConfigFind("CrlFile", "");
333 if(crlfile
.empty() == false)
334 curl_easy_setopt(curl
, CURLOPT_CRLFILE
, crlfile
.c_str());
338 curl_easy_setopt(curl
, CURLOPT_PROTOCOLS
, CURLPROTO_HTTP
);
339 curl_easy_setopt(curl
, CURLOPT_REDIR_PROTOCOLS
, CURLPROTO_HTTP
);
342 if(ConfigFindB("No-Cache", false) == false)
345 if (ConfigFindB("No-Store", false) == true)
346 headers
= curl_slist_append(headers
,"Cache-Control: no-store");
348 strprintf(ss
, "Cache-Control: max-age=%u", ConfigFindI("Max-Age", 0));
349 headers
= curl_slist_append(headers
, ss
.c_str());
351 // cache disabled by user
352 headers
= curl_slist_append(headers
, "Cache-Control: no-cache");
353 headers
= curl_slist_append(headers
, "Pragma: no-cache");
355 curl_easy_setopt(curl
, CURLOPT_HTTPHEADER
, headers
);
357 int const dlLimit
= ConfigFindI("Dl-Limit", 0) * 1024;
359 curl_easy_setopt(curl
, CURLOPT_MAX_RECV_SPEED_LARGE
, dlLimit
);
362 curl_easy_setopt(curl
, CURLOPT_USERAGENT
, ConfigFind("User-Agent", "Debian APT-CURL/1.0 (" PACKAGE_VERSION
")").c_str());
365 int const timeout
= ConfigFindI("Timeout", 120);
366 curl_easy_setopt(curl
, CURLOPT_CONNECTTIMEOUT
, timeout
);
367 //set really low lowspeed timeout (see #497983)
368 curl_easy_setopt(curl
, CURLOPT_LOW_SPEED_LIMIT
, DL_MIN_SPEED
);
369 curl_easy_setopt(curl
, CURLOPT_LOW_SPEED_TIME
, timeout
);
371 if(_config
->FindB("Acquire::ForceIPv4", false) == true)
372 curl_easy_setopt(curl
, CURLOPT_IPRESOLVE
, CURL_IPRESOLVE_V4
);
373 else if(_config
->FindB("Acquire::ForceIPv6", false) == true)
374 curl_easy_setopt(curl
, CURLOPT_IPRESOLVE
, CURL_IPRESOLVE_V6
);
378 curl_easy_setopt(curl
, CURLOPT_VERBOSE
, true);
381 curl_errorstr
[0] = '\0';
382 curl_easy_setopt(curl
, CURLOPT_ERRORBUFFER
, curl_errorstr
);
384 // If we ask for uncompressed files servers might respond with content-
385 // negotiation which lets us end up with compressed files we do not support,
386 // see 657029, 657560 and co, so if we have no extension on the request
387 // ask for text only. As a sidenote: If there is nothing to negotate servers
388 // seem to be nice and ignore it.
389 if (ConfigFindB("SendAccept", true))
391 size_t const filepos
= Itm
->Uri
.find_last_of('/');
392 string
const file
= Itm
->Uri
.substr(filepos
+ 1);
393 if (flExtension(file
) == file
)
394 headers
= curl_slist_append(headers
, "Accept: text/*");
397 // if we have the file send an if-range query with a range header
398 if (Server
->RangesAllowed
&& stat(Itm
->DestFile
.c_str(),&SBuf
) >= 0 && SBuf
.st_size
> 0)
401 strprintf(Buf
, "Range: bytes=%lli-", (long long) SBuf
.st_size
);
402 headers
= curl_slist_append(headers
, Buf
.c_str());
403 strprintf(Buf
, "If-Range: %s", TimeRFC1123(SBuf
.st_mtime
, false).c_str());
404 headers
= curl_slist_append(headers
, Buf
.c_str());
406 else if(Itm
->LastModified
> 0)
408 curl_easy_setopt(curl
, CURLOPT_TIMECONDITION
, CURL_TIMECOND_IFMODSINCE
);
409 curl_easy_setopt(curl
, CURLOPT_TIMEVALUE
, Itm
->LastModified
);
412 if (Server
->InitHashes(Itm
->ExpectedHashes
) == false)
416 Res
.Filename
= Itm
->DestFile
;
419 CURLcode success
= curl_easy_perform(curl
);
421 // If the server returns 200 OK but the If-Modified-Since condition is not
422 // met, CURLINFO_CONDITION_UNMET will be set to 1
423 long curl_condition_unmet
= 0;
424 curl_easy_getinfo(curl
, CURLINFO_CONDITION_UNMET
, &curl_condition_unmet
);
425 if (curl_condition_unmet
== 1)
429 curl_slist_free_all(headers
);
432 if (success
!= CURLE_OK
)
434 #pragma GCC diagnostic push
435 #pragma GCC diagnostic ignored "-Wswitch"
438 case CURLE_COULDNT_RESOLVE_PROXY
:
439 case CURLE_COULDNT_RESOLVE_HOST
:
440 SetFailReason("ResolveFailure");
442 case CURLE_COULDNT_CONNECT
:
443 SetFailReason("ConnectionRefused");
445 case CURLE_OPERATION_TIMEDOUT
:
446 SetFailReason("Timeout");
449 #pragma GCC diagnostic pop
450 // only take curls technical errors if we haven't our own
451 // (e.g. for the maximum size limit we have and curls can be confusing)
452 if (_error
->PendingError() == false)
453 _error
->Error("%s", curl_errorstr
);
455 _error
->Warning("curl: %s", curl_errorstr
);
459 switch (DealWithHeaders(Res
, Req
))
461 case BaseHttpMethod::IMS_HIT
:
465 case BaseHttpMethod::ERROR_WITH_CONTENT_PAGE
:
466 // unlink, no need keep 401/404 page content in partial/
467 RemoveFile(Binary
.c_str(), Req
.File
.Name());
468 case BaseHttpMethod::ERROR_UNRECOVERABLE
:
469 case BaseHttpMethod::ERROR_NOT_FROM_SERVER
:
472 case BaseHttpMethod::TRY_AGAIN_OR_REDIRECT
:
476 case BaseHttpMethod::FILE_IS_OPEN
:
477 struct stat resultStat
;
478 if (unlikely(stat(Req
.File
.Name().c_str(), &resultStat
) != 0))
480 _error
->Errno("stat", "Unable to access file %s", Req
.File
.Name().c_str());
483 Res
.Size
= resultStat
.st_size
;
486 curl_easy_getinfo(curl
, CURLINFO_FILETIME
, &Res
.LastModified
);
487 if (Res
.LastModified
!= -1)
489 struct timeval times
[2];
490 times
[0].tv_sec
= Res
.LastModified
;
491 times
[1].tv_sec
= Res
.LastModified
;
492 times
[0].tv_usec
= times
[1].tv_usec
= 0;
493 utimes(Req
.File
.Name().c_str(), times
);
496 Res
.LastModified
= resultStat
.st_mtime
;
499 Res
.TakeHashes(*(Server
->GetHashes()));
508 std::unique_ptr
<ServerState
> HttpsMethod::CreateServerState(URI
const &uri
)/*{{{*/
510 return std::unique_ptr
<ServerState
>(new HttpsServerState(uri
, this));
513 HttpsMethod::HttpsMethod(std::string
&&pProg
) : BaseHttpMethod(std::move(pProg
),"1.2",Pipeline
| SendConfig
)/*{{{*/
515 auto addName
= std::inserter(methodNames
, methodNames
.begin());
517 auto const plus
= Binary
.find('+');
518 if (plus
!= std::string::npos
)
520 addName
= Binary
.substr(plus
+ 1);
521 auto base
= Binary
.substr(0, plus
);
525 if (std::find(methodNames
.begin(), methodNames
.end(), "https") != methodNames
.end())
526 curl_global_init(CURL_GLOBAL_SSL
);
528 curl_global_init(CURL_GLOBAL_NOTHING
);
529 curl
= curl_easy_init();
532 HttpsMethod::~HttpsMethod() /*{{{*/
534 curl_easy_cleanup(curl
);
537 int main(int, const char *argv
[]) /*{{{*/
539 std::string Binary
= flNotDir(argv
[0]);
540 if (Binary
.find('+') == std::string::npos
&& Binary
!= "https")
541 Binary
.append("+https");
542 return HttpsMethod(std::move(Binary
)).Run();