1 //-*- mode: cpp; mode: fold -*-
3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
4 /* ######################################################################
6 HTTPS Acquire Method - This is the HTTPS acquire method for APT.
10 ##################################################################### */
12 // Include Files /*{{{*/
15 #include <apt-pkg/fileutl.h>
16 #include <apt-pkg/error.h>
17 #include <apt-pkg/hashes.h>
18 #include <apt-pkg/netrc.h>
19 #include <apt-pkg/configuration.h>
20 #include <apt-pkg/macros.h>
21 #include <apt-pkg/strutl.h>
22 #include <apt-pkg/proxy.h>
39 struct APT_HIDDEN CURLUserPointer
{
40 HttpsMethod
* const https
;
41 HttpsMethod::FetchResult
* const Res
;
42 HttpsMethod::FetchItem
const * const Itm
;
43 CURLUserPointer(HttpsMethod
* const https
, HttpsMethod::FetchResult
* const Res
,
44 HttpsMethod::FetchItem
const * const Itm
) : https(https
), Res(Res
), Itm(Itm
) {}
48 HttpsMethod::parse_header(void *buffer
, size_t size
, size_t nmemb
, void *userp
)
50 size_t len
= size
* nmemb
;
51 CURLUserPointer
*me
= static_cast<CURLUserPointer
*>(userp
);
52 std::string
line((char*) buffer
, len
);
53 for (--len
; len
> 0; --len
)
54 if (isspace_ascii(line
[len
]) == 0)
61 if (line
.empty() == true)
63 if (me
->https
->Server
->Result
!= 416 && me
->https
->Server
->StartPos
!= 0)
65 else if (me
->https
->Server
->Result
== 416)
67 bool partialHit
= false;
68 if (me
->Itm
->ExpectedHashes
.usable() == true)
70 Hashes
resultHashes(me
->Itm
->ExpectedHashes
);
71 FileFd
file(me
->Itm
->DestFile
, FileFd::ReadOnly
);
72 me
->https
->Server
->TotalFileSize
= file
.FileSize();
73 me
->https
->Server
->Date
= file
.ModificationTime();
74 resultHashes
.AddFD(file
);
75 HashStringList
const hashList
= resultHashes
.GetHashStringList();
76 partialHit
= (me
->Itm
->ExpectedHashes
== hashList
);
78 else if (me
->https
->Server
->Result
== 416 && me
->https
->Server
->TotalFileSize
== me
->https
->File
->FileSize())
81 if (partialHit
== true)
83 me
->https
->Server
->Result
= 200;
84 me
->https
->Server
->StartPos
= me
->https
->Server
->TotalFileSize
;
85 // the actual size is not important for https as curl will deal with it
86 // by itself and e.g. doesn't bother us with transport-encoding…
87 me
->https
->Server
->JunkSize
= std::numeric_limits
<unsigned long long>::max();
90 me
->https
->Server
->StartPos
= 0;
93 me
->https
->Server
->StartPos
= 0;
95 me
->Res
->LastModified
= me
->https
->Server
->Date
;
96 me
->Res
->Size
= me
->https
->Server
->TotalFileSize
;
97 me
->Res
->ResumePoint
= me
->https
->Server
->StartPos
;
99 // we expect valid data, so tell our caller we get the file now
100 if (me
->https
->Server
->Result
>= 200 && me
->https
->Server
->Result
< 300)
102 if (me
->https
->Server
->JunkSize
== 0 && me
->Res
->Size
!= 0 && me
->Res
->Size
> me
->Res
->ResumePoint
)
103 me
->https
->URIStart(*me
->Res
);
104 if (me
->https
->Server
->AddPartialFileToHashes(*(me
->https
->File
)) == false)
108 else if (me
->https
->Server
->HeaderLine(line
) == false)
115 HttpsMethod::write_data(void *buffer
, size_t size
, size_t nmemb
, void *userp
)
117 HttpsMethod
*me
= static_cast<HttpsMethod
*>(userp
);
118 size_t buffer_size
= size
* nmemb
;
119 // we don't need to count the junk here, just drop anything we get as
120 // we don't always know how long it would be, e.g. in chunked encoding.
121 if (me
->Server
->JunkSize
!= 0)
124 if(me
->File
->Write(buffer
, buffer_size
) != true)
127 if(me
->Queue
->MaximumSize
> 0)
129 unsigned long long const TotalWritten
= me
->File
->Tell();
130 if (TotalWritten
> me
->Queue
->MaximumSize
)
132 me
->SetFailReason("MaximumSizeExceeded");
133 _error
->Error("Writing more data than expected (%llu > %llu)",
134 TotalWritten
, me
->Queue
->MaximumSize
);
139 if (me
->Server
->GetHashes()->Add((unsigned char const * const)buffer
, buffer_size
) == false)
145 // HttpsServerState::HttpsServerState - Constructor /*{{{*/
146 HttpsServerState::HttpsServerState(URI Srv
,HttpsMethod
* Owner
) : ServerState(Srv
, Owner
), Hash(NULL
)
148 TimeOut
= _config
->FindI("Acquire::https::Timeout",TimeOut
);
152 bool HttpsServerState::InitHashes(HashStringList
const &ExpectedHashes
) /*{{{*/
155 Hash
= new Hashes(ExpectedHashes
);
159 APT_PURE Hashes
* HttpsServerState::GetHashes() /*{{{*/
165 bool HttpsMethod::SetupProxy() /*{{{*/
167 URI ServerName
= Queue
->Uri
;
169 // Determine the proxy setting
170 AutoDetectProxy(ServerName
);
172 // Curl should never read proxy settings from the environment, as
173 // we determine which proxy to use. Do this for consistency among
174 // methods and prevent an environment variable overriding a
175 // no-proxy ("DIRECT") setting in apt.conf.
176 curl_easy_setopt(curl
, CURLOPT_PROXY
, "");
178 // Determine the proxy setting - try https first, fallback to http and use env at last
179 string UseProxy
= _config
->Find("Acquire::https::Proxy::" + ServerName
.Host
,
180 _config
->Find("Acquire::http::Proxy::" + ServerName
.Host
).c_str());
182 if (UseProxy
.empty() == true)
183 UseProxy
= _config
->Find("Acquire::https::Proxy", _config
->Find("Acquire::http::Proxy").c_str());
185 // User want to use NO proxy, so nothing to setup
186 if (UseProxy
== "DIRECT")
189 // Parse no_proxy, a comma (,) separated list of domains we don't want to use
190 // a proxy for so we stop right here if it is in the list
191 if (getenv("no_proxy") != 0 && CheckDomainList(ServerName
.Host
,getenv("no_proxy")) == true)
194 if (UseProxy
.empty() == true)
196 const char* result
= getenv("https_proxy");
197 // FIXME: Fall back to http_proxy is to remain compatible with
198 // existing setups and behaviour of apt.conf. This should be
199 // deprecated in the future (including apt.conf). Most other
200 // programs do not fall back to http proxy settings and neither
203 result
= getenv("http_proxy");
204 UseProxy
= result
== NULL
? "" : result
;
207 // Determine what host and port to use based on the proxy settings
208 if (UseProxy
.empty() == false)
211 if (Proxy
.Access
== "socks5h")
212 curl_easy_setopt(curl
, CURLOPT_PROXYTYPE
, CURLPROXY_SOCKS5_HOSTNAME
);
213 else if (Proxy
.Access
== "socks5")
214 curl_easy_setopt(curl
, CURLOPT_PROXYTYPE
, CURLPROXY_SOCKS5
);
215 else if (Proxy
.Access
== "socks4a")
216 curl_easy_setopt(curl
, CURLOPT_PROXYTYPE
, CURLPROXY_SOCKS4A
);
217 else if (Proxy
.Access
== "socks")
218 curl_easy_setopt(curl
, CURLOPT_PROXYTYPE
, CURLPROXY_SOCKS4
);
219 else if (Proxy
.Access
== "http" || Proxy
.Access
== "https")
220 curl_easy_setopt(curl
, CURLOPT_PROXYTYPE
, CURLPROXY_HTTP
);
225 curl_easy_setopt(curl
, CURLOPT_PROXYPORT
, Proxy
.Port
);
226 curl_easy_setopt(curl
, CURLOPT_PROXY
, Proxy
.Host
.c_str());
227 if (Proxy
.User
.empty() == false || Proxy
.Password
.empty() == false)
229 curl_easy_setopt(curl
, CURLOPT_PROXYUSERNAME
, Proxy
.User
.c_str());
230 curl_easy_setopt(curl
, CURLOPT_PROXYPASSWORD
, Proxy
.Password
.c_str());
235 // HttpsMethod::Fetch - Fetch an item /*{{{*/
236 // ---------------------------------------------------------------------
237 /* This adds an item to the pipeline. We keep the pipeline at a fixed
239 bool HttpsMethod::Fetch(FetchItem
*Itm
)
242 struct curl_slist
*headers
=NULL
;
243 char curl_errorstr
[CURL_ERROR_SIZE
];
245 string remotehost
= Uri
.Host
;
248 // - http::Pipeline-Depth
249 // - error checking/reporting
250 // - more debug options? (CURLOPT_DEBUGFUNCTION?)
252 curl_easy_reset(curl
);
253 if (SetupProxy() == false)
254 return _error
->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy
).c_str());
256 maybe_add_auth (Uri
, _config
->FindFile("Dir::Etc::netrc"));
259 CURLUserPointer
userp(this, &Res
, Itm
);
261 curl_easy_setopt(curl
, CURLOPT_URL
, static_cast<string
>(Uri
).c_str());
262 curl_easy_setopt(curl
, CURLOPT_HEADERFUNCTION
, parse_header
);
263 curl_easy_setopt(curl
, CURLOPT_WRITEHEADER
, &userp
);
264 curl_easy_setopt(curl
, CURLOPT_WRITEFUNCTION
, write_data
);
265 curl_easy_setopt(curl
, CURLOPT_WRITEDATA
, this);
267 curl_easy_setopt(curl
, CURLOPT_NOPROGRESS
, true);
268 curl_easy_setopt(curl
, CURLOPT_FILETIME
, true);
269 // only allow curl to handle https, not the other stuff it supports
270 curl_easy_setopt(curl
, CURLOPT_PROTOCOLS
, CURLPROTO_HTTPS
);
271 curl_easy_setopt(curl
, CURLOPT_REDIR_PROTOCOLS
, CURLPROTO_HTTPS
);
273 // SSL parameters are set by default to the common (non mirror-specific) value
274 // if available (or a default one) and gets overload by mirror-specific ones.
276 // File containing the list of trusted CA.
277 string cainfo
= _config
->Find("Acquire::https::CaInfo","");
278 string knob
= "Acquire::https::"+remotehost
+"::CaInfo";
279 cainfo
= _config
->Find(knob
.c_str(),cainfo
.c_str());
280 if(cainfo
.empty() == false)
281 curl_easy_setopt(curl
, CURLOPT_CAINFO
,cainfo
.c_str());
283 // Check server certificate against previous CA list ...
284 bool peer_verify
= _config
->FindB("Acquire::https::Verify-Peer",true);
285 knob
= "Acquire::https::" + remotehost
+ "::Verify-Peer";
286 peer_verify
= _config
->FindB(knob
.c_str(), peer_verify
);
287 curl_easy_setopt(curl
, CURLOPT_SSL_VERIFYPEER
, peer_verify
);
289 // ... and hostname against cert CN or subjectAltName
290 bool verify
= _config
->FindB("Acquire::https::Verify-Host",true);
291 knob
= "Acquire::https::"+remotehost
+"::Verify-Host";
292 verify
= _config
->FindB(knob
.c_str(),verify
);
293 int const default_verify
= (verify
== true) ? 2 : 0;
294 curl_easy_setopt(curl
, CURLOPT_SSL_VERIFYHOST
, default_verify
);
296 // Also enforce issuer of server certificate using its cert
297 string issuercert
= _config
->Find("Acquire::https::IssuerCert","");
298 knob
= "Acquire::https::"+remotehost
+"::IssuerCert";
299 issuercert
= _config
->Find(knob
.c_str(),issuercert
.c_str());
300 if(issuercert
.empty() == false)
301 curl_easy_setopt(curl
, CURLOPT_ISSUERCERT
,issuercert
.c_str());
303 // For client authentication, certificate file ...
304 string pem
= _config
->Find("Acquire::https::SslCert","");
305 knob
= "Acquire::https::"+remotehost
+"::SslCert";
306 pem
= _config
->Find(knob
.c_str(),pem
.c_str());
307 if(pem
.empty() == false)
308 curl_easy_setopt(curl
, CURLOPT_SSLCERT
, pem
.c_str());
310 // ... and associated key.
311 string key
= _config
->Find("Acquire::https::SslKey","");
312 knob
= "Acquire::https::"+remotehost
+"::SslKey";
313 key
= _config
->Find(knob
.c_str(),key
.c_str());
314 if(key
.empty() == false)
315 curl_easy_setopt(curl
, CURLOPT_SSLKEY
, key
.c_str());
317 // Allow forcing SSL version to SSLv3 or TLSv1 (SSLv2 is not
318 // supported by GnuTLS).
319 long final_version
= CURL_SSLVERSION_DEFAULT
;
320 string sslversion
= _config
->Find("Acquire::https::SslForceVersion","");
321 knob
= "Acquire::https::"+remotehost
+"::SslForceVersion";
322 sslversion
= _config
->Find(knob
.c_str(),sslversion
.c_str());
323 if(sslversion
== "TLSv1")
324 final_version
= CURL_SSLVERSION_TLSv1
;
325 else if(sslversion
== "SSLv3")
326 final_version
= CURL_SSLVERSION_SSLv3
;
327 curl_easy_setopt(curl
, CURLOPT_SSLVERSION
, final_version
);
330 string crlfile
= _config
->Find("Acquire::https::CrlFile","");
331 knob
= "Acquire::https::"+remotehost
+"::CrlFile";
332 crlfile
= _config
->Find(knob
.c_str(),crlfile
.c_str());
333 if(crlfile
.empty() == false)
334 curl_easy_setopt(curl
, CURLOPT_CRLFILE
, crlfile
.c_str());
337 if(_config
->FindB("Acquire::https::No-Cache",
338 _config
->FindB("Acquire::http::No-Cache",false)) == false)
341 if (_config
->FindB("Acquire::https::No-Store",
342 _config
->FindB("Acquire::http::No-Store",false)) == true)
343 headers
= curl_slist_append(headers
,"Cache-Control: no-store");
345 ioprintf(ss
, "Cache-Control: max-age=%u", _config
->FindI("Acquire::https::Max-Age",
346 _config
->FindI("Acquire::http::Max-Age",0)));
347 headers
= curl_slist_append(headers
, ss
.str().c_str());
349 // cache disabled by user
350 headers
= curl_slist_append(headers
, "Cache-Control: no-cache");
351 headers
= curl_slist_append(headers
, "Pragma: no-cache");
353 curl_easy_setopt(curl
, CURLOPT_HTTPHEADER
, headers
);
356 int const dlLimit
= _config
->FindI("Acquire::https::Dl-Limit",
357 _config
->FindI("Acquire::http::Dl-Limit",0))*1024;
359 curl_easy_setopt(curl
, CURLOPT_MAX_RECV_SPEED_LARGE
, dlLimit
);
362 curl_easy_setopt(curl
, CURLOPT_USERAGENT
,
363 _config
->Find("Acquire::https::User-Agent",
364 _config
->Find("Acquire::http::User-Agent",
365 "Debian APT-CURL/1.0 (" PACKAGE_VERSION
")").c_str()).c_str());
368 int const timeout
= _config
->FindI("Acquire::https::Timeout",
369 _config
->FindI("Acquire::http::Timeout",120));
370 curl_easy_setopt(curl
, CURLOPT_CONNECTTIMEOUT
, timeout
);
371 //set really low lowspeed timeout (see #497983)
372 curl_easy_setopt(curl
, CURLOPT_LOW_SPEED_LIMIT
, DL_MIN_SPEED
);
373 curl_easy_setopt(curl
, CURLOPT_LOW_SPEED_TIME
, timeout
);
375 // set redirect options and default to 10 redirects
376 curl_easy_setopt(curl
, CURLOPT_FOLLOWLOCATION
, AllowRedirect
);
377 curl_easy_setopt(curl
, CURLOPT_MAXREDIRS
, 10);
381 curl_easy_setopt(curl
, CURLOPT_VERBOSE
, true);
384 curl_errorstr
[0] = '\0';
385 curl_easy_setopt(curl
, CURLOPT_ERRORBUFFER
, curl_errorstr
);
387 // If we ask for uncompressed files servers might respond with content-
388 // negotiation which lets us end up with compressed files we do not support,
389 // see 657029, 657560 and co, so if we have no extension on the request
390 // ask for text only. As a sidenote: If there is nothing to negotate servers
391 // seem to be nice and ignore it.
392 if (_config
->FindB("Acquire::https::SendAccept", _config
->FindB("Acquire::http::SendAccept", true)) == true)
394 size_t const filepos
= Itm
->Uri
.find_last_of('/');
395 string
const file
= Itm
->Uri
.substr(filepos
+ 1);
396 if (flExtension(file
) == file
)
397 headers
= curl_slist_append(headers
, "Accept: text/*");
400 // if we have the file send an if-range query with a range header
401 if (stat(Itm
->DestFile
.c_str(),&SBuf
) >= 0 && SBuf
.st_size
> 0)
404 strprintf(Buf
, "Range: bytes=%lli-", (long long) SBuf
.st_size
);
405 headers
= curl_slist_append(headers
, Buf
.c_str());
406 strprintf(Buf
, "If-Range: %s", TimeRFC1123(SBuf
.st_mtime
, false).c_str());
407 headers
= curl_slist_append(headers
, Buf
.c_str());
409 else if(Itm
->LastModified
> 0)
411 curl_easy_setopt(curl
, CURLOPT_TIMECONDITION
, CURL_TIMECOND_IFMODSINCE
);
412 curl_easy_setopt(curl
, CURLOPT_TIMEVALUE
, Itm
->LastModified
);
415 // go for it - if the file exists, append on it
416 File
= new FileFd(Itm
->DestFile
, FileFd::WriteAny
);
417 Server
= CreateServerState(Itm
->Uri
);
418 if (Server
->InitHashes(Itm
->ExpectedHashes
) == false)
422 Res
.Filename
= Itm
->DestFile
;
425 CURLcode success
= curl_easy_perform(curl
);
427 // If the server returns 200 OK but the If-Modified-Since condition is not
428 // met, CURLINFO_CONDITION_UNMET will be set to 1
429 long curl_condition_unmet
= 0;
430 curl_easy_getinfo(curl
, CURLINFO_CONDITION_UNMET
, &curl_condition_unmet
);
433 curl_slist_free_all(headers
);
436 if (success
!= CURLE_OK
)
438 #pragma GCC diagnostic push
439 #pragma GCC diagnostic ignored "-Wswitch"
442 case CURLE_COULDNT_RESOLVE_PROXY
:
443 case CURLE_COULDNT_RESOLVE_HOST
:
444 SetFailReason("ResolveFailure");
446 case CURLE_COULDNT_CONNECT
:
447 SetFailReason("ConnectionRefused");
449 case CURLE_OPERATION_TIMEDOUT
:
450 SetFailReason("Timeout");
453 #pragma GCC diagnostic pop
454 // only take curls technical errors if we haven't our own
455 // (e.g. for the maximum size limit we have and curls can be confusing)
456 if (_error
->PendingError() == false)
457 _error
->Error("%s", curl_errorstr
);
459 _error
->Warning("curl: %s", curl_errorstr
);
463 // server says file not modified
464 if (Server
->Result
== 304 || curl_condition_unmet
== 1)
466 RemoveFile("https", File
->Name());
468 Res
.LastModified
= Itm
->LastModified
;
475 if (Server
->Result
!= 200 && // OK
476 Server
->Result
!= 206 && // Partial
477 Server
->Result
!= 416) // invalid Range
480 snprintf(err
, sizeof(err
) - 1, "HttpError%i", Server
->Result
);
482 _error
->Error("%i %s", Server
->Result
, Server
->Code
);
483 // unlink, no need keep 401/404 page content in partial/
484 RemoveFile("https", File
->Name());
488 // invalid range-request
489 if (Server
->Result
== 416)
491 RemoveFile("https", File
->Name());
497 struct stat resultStat
;
498 if (unlikely(stat(File
->Name().c_str(), &resultStat
) != 0))
500 _error
->Errno("stat", "Unable to access file %s", File
->Name().c_str());
503 Res
.Size
= resultStat
.st_size
;
506 curl_easy_getinfo(curl
, CURLINFO_FILETIME
, &Res
.LastModified
);
507 if (Res
.LastModified
!= -1)
509 struct timeval times
[2];
510 times
[0].tv_sec
= Res
.LastModified
;
511 times
[1].tv_sec
= Res
.LastModified
;
512 times
[0].tv_usec
= times
[1].tv_usec
= 0;
513 utimes(File
->Name().c_str(), times
);
516 Res
.LastModified
= resultStat
.st_mtime
;
519 Res
.TakeHashes(*(Server
->GetHashes()));
530 // HttpsMethod::Configuration - Handle a configuration message /*{{{*/
531 bool HttpsMethod::Configuration(string Message
)
533 if (ServerMethod::Configuration(Message
) == false)
536 AllowRedirect
= _config
->FindB("Acquire::https::AllowRedirect",
537 _config
->FindB("Acquire::http::AllowRedirect", true));
538 Debug
= _config
->FindB("Debug::Acquire::https",false);
543 std::unique_ptr
<ServerState
> HttpsMethod::CreateServerState(URI
const &uri
)/*{{{*/
545 return std::unique_ptr
<ServerState
>(new HttpsServerState(uri
, this));
551 return HttpsMethod().Run();