]> git.saurik.com Git - apt.git/blob - methods/https.cc
d60bc6fbc65c659eeb727aa604234ab44a900b75
[apt.git] / methods / https.cc
1 //-*- mode: cpp; mode: fold -*-
2 // Description /*{{{*/
3 // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
4 /* ######################################################################
5
6 HTTPS Acquire Method - This is the HTTPS acquire method for APT.
7
8 It uses libcurl
9
10 ##################################################################### */
11 /*}}}*/
12 // Include Files /*{{{*/
13 #include <config.h>
14
15 #include <apt-pkg/fileutl.h>
16 #include <apt-pkg/error.h>
17 #include <apt-pkg/hashes.h>
18 #include <apt-pkg/netrc.h>
19 #include <apt-pkg/configuration.h>
20 #include <apt-pkg/macros.h>
21 #include <apt-pkg/strutl.h>
22 #include <apt-pkg/proxy.h>
23
24 #include <sys/stat.h>
25 #include <sys/time.h>
26 #include <unistd.h>
27 #include <stdio.h>
28 #include <ctype.h>
29 #include <stdlib.h>
30
31 #include <array>
32 #include <iostream>
33 #include <sstream>
34
35
36 #include "https.h"
37
38 #include <apti18n.h>
39 /*}}}*/
40 using namespace std;
41
42 struct APT_HIDDEN CURLUserPointer {
43 HttpsMethod * const https;
44 HttpsMethod::FetchResult * const Res;
45 HttpsMethod::FetchItem const * const Itm;
46 RequestState * const Req;
47 CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res,
48 HttpsMethod::FetchItem const * const Itm, RequestState * const Req) : https(https), Res(Res), Itm(Itm), Req(Req) {}
49 };
50
51 size_t
52 HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
53 {
54 size_t len = size * nmemb;
55 CURLUserPointer *me = static_cast<CURLUserPointer *>(userp);
56 std::string line((char*) buffer, len);
57 for (--len; len > 0; --len)
58 if (isspace_ascii(line[len]) == 0)
59 {
60 ++len;
61 break;
62 }
63 line.erase(len);
64
65 if (line.empty() == true)
66 {
67 if (me->Req->File.Open(me->Itm->DestFile, FileFd::WriteAny) == false)
68 return ERROR_NOT_FROM_SERVER;
69
70 me->Req->JunkSize = 0;
71 if (me->Req->Result != 416 && me->Req->StartPos != 0)
72 ;
73 else if (me->Req->Result == 416)
74 {
75 bool partialHit = false;
76 if (me->Itm->ExpectedHashes.usable() == true)
77 {
78 Hashes resultHashes(me->Itm->ExpectedHashes);
79 FileFd file(me->Itm->DestFile, FileFd::ReadOnly);
80 me->Req->TotalFileSize = file.FileSize();
81 me->Req->Date = file.ModificationTime();
82 resultHashes.AddFD(file);
83 HashStringList const hashList = resultHashes.GetHashStringList();
84 partialHit = (me->Itm->ExpectedHashes == hashList);
85 }
86 else if (me->Req->Result == 416 && me->Req->TotalFileSize == me->Req->File.FileSize())
87 partialHit = true;
88
89 if (partialHit == true)
90 {
91 me->Req->Result = 200;
92 me->Req->StartPos = me->Req->TotalFileSize;
93 // the actual size is not important for https as curl will deal with it
94 // by itself and e.g. doesn't bother us with transport-encoding…
95 me->Req->JunkSize = std::numeric_limits<unsigned long long>::max();
96 }
97 else
98 me->Req->StartPos = 0;
99 }
100 else
101 me->Req->StartPos = 0;
102
103 me->Res->LastModified = me->Req->Date;
104 me->Res->Size = me->Req->TotalFileSize;
105 me->Res->ResumePoint = me->Req->StartPos;
106
107 // we expect valid data, so tell our caller we get the file now
108 if (me->Req->Result >= 200 && me->Req->Result < 300)
109 {
110 if (me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint)
111 me->https->URIStart(*me->Res);
112 if (me->Req->AddPartialFileToHashes(me->Req->File) == false)
113 return 0;
114 }
115 else
116 me->Req->JunkSize = std::numeric_limits<decltype(me->Req->JunkSize)>::max();
117 }
118 else if (me->Req->HeaderLine(line) == false)
119 return 0;
120
121 return size*nmemb;
122 }
123
124 size_t
125 HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
126 {
127 CURLUserPointer *me = static_cast<CURLUserPointer *>(userp);
128 size_t buffer_size = size * nmemb;
129 // we don't need to count the junk here, just drop anything we get as
130 // we don't always know how long it would be, e.g. in chunked encoding.
131 if (me->Req->JunkSize != 0)
132 return buffer_size;
133
134 if(me->Req->File.Write(buffer, buffer_size) != true)
135 return 0;
136
137 if(me->https->Queue->MaximumSize > 0)
138 {
139 unsigned long long const TotalWritten = me->Req->File.Tell();
140 if (TotalWritten > me->https->Queue->MaximumSize)
141 {
142 me->https->SetFailReason("MaximumSizeExceeded");
143 _error->Error("Writing more data than expected (%llu > %llu)",
144 TotalWritten, me->https->Queue->MaximumSize);
145 return 0;
146 }
147 }
148
149 if (me->https->Server->GetHashes()->Add((unsigned char const * const)buffer, buffer_size) == false)
150 return 0;
151
152 return buffer_size;
153 }
154
155 // HttpsServerState::HttpsServerState - Constructor /*{{{*/
156 HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner), Hash(NULL)
157 {
158 TimeOut = Owner->ConfigFindI("Timeout", TimeOut);
159 Reset();
160 }
161 /*}}}*/
162 bool HttpsServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/
163 {
164 delete Hash;
165 Hash = new Hashes(ExpectedHashes);
166 return true;
167 }
168 /*}}}*/
169 APT_PURE Hashes * HttpsServerState::GetHashes() /*{{{*/
170 {
171 return Hash;
172 }
173 /*}}}*/
174
175 bool HttpsMethod::SetupProxy() /*{{{*/
176 {
177 URI ServerName = Queue->Uri;
178
179 // Determine the proxy setting
180 AutoDetectProxy(ServerName);
181
182 // Curl should never read proxy settings from the environment, as
183 // we determine which proxy to use. Do this for consistency among
184 // methods and prevent an environment variable overriding a
185 // no-proxy ("DIRECT") setting in apt.conf.
186 curl_easy_setopt(curl, CURLOPT_PROXY, "");
187
188 // Determine the proxy setting - try https first, fallback to http and use env at last
189 string UseProxy = ConfigFind("Proxy::" + ServerName.Host, "");
190 if (UseProxy.empty() == true)
191 UseProxy = ConfigFind("Proxy", "");
192 // User wants to use NO proxy, so nothing to setup
193 if (UseProxy == "DIRECT")
194 return true;
195
196 // Parse no_proxy, a comma (,) separated list of domains we don't want to use
197 // a proxy for so we stop right here if it is in the list
198 if (getenv("no_proxy") != 0 && CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
199 return true;
200
201 if (UseProxy.empty() == true)
202 {
203 const char* result = nullptr;
204 if (std::find(methodNames.begin(), methodNames.end(), "https") != methodNames.end())
205 result = getenv("https_proxy");
206 // FIXME: Fall back to http_proxy is to remain compatible with
207 // existing setups and behaviour of apt.conf. This should be
208 // deprecated in the future (including apt.conf). Most other
209 // programs do not fall back to http proxy settings and neither
210 // should Apt.
211 if (result == nullptr && std::find(methodNames.begin(), methodNames.end(), "http") != methodNames.end())
212 result = getenv("http_proxy");
213 UseProxy = result == nullptr ? "" : result;
214 }
215
216 // Determine what host and port to use based on the proxy settings
217 if (UseProxy.empty() == false)
218 {
219 Proxy = UseProxy;
220 AddProxyAuth(Proxy, ServerName);
221
222 if (Proxy.Access == "socks5h")
223 curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
224 else if (Proxy.Access == "socks5")
225 curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
226 else if (Proxy.Access == "socks4a")
227 curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4A);
228 else if (Proxy.Access == "socks")
229 curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4);
230 else if (Proxy.Access == "http" || Proxy.Access == "https")
231 curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
232 else
233 return false;
234
235 if (Proxy.Port != 1)
236 curl_easy_setopt(curl, CURLOPT_PROXYPORT, Proxy.Port);
237 curl_easy_setopt(curl, CURLOPT_PROXY, Proxy.Host.c_str());
238 if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
239 {
240 curl_easy_setopt(curl, CURLOPT_PROXYUSERNAME, Proxy.User.c_str());
241 curl_easy_setopt(curl, CURLOPT_PROXYPASSWORD, Proxy.Password.c_str());
242 }
243 }
244 return true;
245 } /*}}}*/
246 // HttpsMethod::Fetch - Fetch an item /*{{{*/
247 // ---------------------------------------------------------------------
248 /* This adds an item to the pipeline. We keep the pipeline at a fixed
249 depth. */
250 bool HttpsMethod::Fetch(FetchItem *Itm)
251 {
252 struct stat SBuf;
253 struct curl_slist *headers=NULL;
254 char curl_errorstr[CURL_ERROR_SIZE];
255 URI Uri = Itm->Uri;
256 setPostfixForMethodNames(Uri.Host.c_str());
257 AllowRedirect = ConfigFindB("AllowRedirect", true);
258 Debug = DebugEnabled();
259
260 // TODO:
261 // - http::Pipeline-Depth
262 // - error checking/reporting
263 // - more debug options? (CURLOPT_DEBUGFUNCTION?)
264 {
265 auto const plus = Binary.find('+');
266 if (plus != std::string::npos)
267 Uri.Access = Binary.substr(plus + 1);
268 }
269
270 curl_easy_reset(curl);
271 if (SetupProxy() == false)
272 return _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str());
273
274 maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
275 if (Server == nullptr || Server->Comp(Itm->Uri) == false)
276 Server = CreateServerState(Itm->Uri);
277
278 FetchResult Res;
279 RequestState Req(this, Server.get());
280 CURLUserPointer userp(this, &Res, Itm, &Req);
281 // callbacks
282 curl_easy_setopt(curl, CURLOPT_URL, static_cast<string>(Uri).c_str());
283 curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header);
284 curl_easy_setopt(curl, CURLOPT_WRITEHEADER, &userp);
285 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
286 curl_easy_setopt(curl, CURLOPT_WRITEDATA, &userp);
287 // options
288 curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
289 curl_easy_setopt(curl, CURLOPT_FILETIME, true);
290 curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0);
291
292 if (std::find(methodNames.begin(), methodNames.end(), "https") != methodNames.end())
293 {
294 curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS);
295 curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS);
296
297 // File containing the list of trusted CA.
298 std::string const cainfo = ConfigFind("CaInfo", "");
299 if(cainfo.empty() == false)
300 curl_easy_setopt(curl, CURLOPT_CAINFO, cainfo.c_str());
301 // Check server certificate against previous CA list ...
302 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, ConfigFindB("Verify-Peer", true) ? 1 : 0);
303 // ... and hostname against cert CN or subjectAltName
304 curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, ConfigFindB("Verify-Host", true) ? 2 : 0);
305 // Also enforce issuer of server certificate using its cert
306 std::string const issuercert = ConfigFind("IssuerCert", "");
307 if(issuercert.empty() == false)
308 curl_easy_setopt(curl, CURLOPT_ISSUERCERT, issuercert.c_str());
309 // For client authentication, certificate file ...
310 std::string const pem = ConfigFind("SslCert", "");
311 if(pem.empty() == false)
312 curl_easy_setopt(curl, CURLOPT_SSLCERT, pem.c_str());
313 // ... and associated key.
314 std::string const key = ConfigFind("SslKey", "");
315 if(key.empty() == false)
316 curl_easy_setopt(curl, CURLOPT_SSLKEY, key.c_str());
317 // Allow forcing SSL version to SSLv3 or TLSv1
318 long final_version = CURL_SSLVERSION_DEFAULT;
319 std::string const sslversion = ConfigFind("SslForceVersion", "");
320 if(sslversion == "TLSv1")
321 final_version = CURL_SSLVERSION_TLSv1;
322 else if(sslversion == "TLSv1.0")
323 final_version = CURL_SSLVERSION_TLSv1_0;
324 else if(sslversion == "TLSv1.1")
325 final_version = CURL_SSLVERSION_TLSv1_1;
326 else if(sslversion == "TLSv1.2")
327 final_version = CURL_SSLVERSION_TLSv1_2;
328 else if(sslversion == "SSLv3")
329 final_version = CURL_SSLVERSION_SSLv3;
330 curl_easy_setopt(curl, CURLOPT_SSLVERSION, final_version);
331 // CRL file
332 std::string const crlfile = ConfigFind("CrlFile", "");
333 if(crlfile.empty() == false)
334 curl_easy_setopt(curl, CURLOPT_CRLFILE, crlfile.c_str());
335 }
336 else
337 {
338 curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP);
339 curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP);
340 }
341 // cache-control
342 if(ConfigFindB("No-Cache", false) == false)
343 {
344 // cache enabled
345 if (ConfigFindB("No-Store", false) == true)
346 headers = curl_slist_append(headers,"Cache-Control: no-store");
347 std::string ss;
348 strprintf(ss, "Cache-Control: max-age=%u", ConfigFindI("Max-Age", 0));
349 headers = curl_slist_append(headers, ss.c_str());
350 } else {
351 // cache disabled by user
352 headers = curl_slist_append(headers, "Cache-Control: no-cache");
353 headers = curl_slist_append(headers, "Pragma: no-cache");
354 }
355 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
356 // speed limit
357 int const dlLimit = ConfigFindI("Dl-Limit", 0) * 1024;
358 if (dlLimit > 0)
359 curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, dlLimit);
360
361 // set header
362 curl_easy_setopt(curl, CURLOPT_USERAGENT, ConfigFind("User-Agent", "Debian APT-CURL/1.0 (" PACKAGE_VERSION ")").c_str());
363
364 // set timeout
365 int const timeout = ConfigFindI("Timeout", 120);
366 curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout);
367 //set really low lowspeed timeout (see #497983)
368 curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, DL_MIN_SPEED);
369 curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, timeout);
370
371 if(_config->FindB("Acquire::ForceIPv4", false) == true)
372 curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
373 else if(_config->FindB("Acquire::ForceIPv6", false) == true)
374 curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6);
375
376 // debug
377 if (Debug == true)
378 curl_easy_setopt(curl, CURLOPT_VERBOSE, true);
379
380 // error handling
381 curl_errorstr[0] = '\0';
382 curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr);
383
384 // If we ask for uncompressed files servers might respond with content-
385 // negotiation which lets us end up with compressed files we do not support,
386 // see 657029, 657560 and co, so if we have no extension on the request
387 // ask for text only. As a sidenote: If there is nothing to negotate servers
388 // seem to be nice and ignore it.
389 if (ConfigFindB("SendAccept", true))
390 {
391 size_t const filepos = Itm->Uri.find_last_of('/');
392 string const file = Itm->Uri.substr(filepos + 1);
393 if (flExtension(file) == file)
394 headers = curl_slist_append(headers, "Accept: text/*");
395 }
396
397 // if we have the file send an if-range query with a range header
398 if (Server->RangesAllowed && stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
399 {
400 std::string Buf;
401 strprintf(Buf, "Range: bytes=%lli-", (long long) SBuf.st_size);
402 headers = curl_slist_append(headers, Buf.c_str());
403 strprintf(Buf, "If-Range: %s", TimeRFC1123(SBuf.st_mtime, false).c_str());
404 headers = curl_slist_append(headers, Buf.c_str());
405 }
406 else if(Itm->LastModified > 0)
407 {
408 curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
409 curl_easy_setopt(curl, CURLOPT_TIMEVALUE, Itm->LastModified);
410 }
411
412 if (Server->InitHashes(Itm->ExpectedHashes) == false)
413 return false;
414
415 // keep apt updated
416 Res.Filename = Itm->DestFile;
417
418 // get it!
419 CURLcode success = curl_easy_perform(curl);
420
421 // If the server returns 200 OK but the If-Modified-Since condition is not
422 // met, CURLINFO_CONDITION_UNMET will be set to 1
423 long curl_condition_unmet = 0;
424 curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &curl_condition_unmet);
425 if (curl_condition_unmet == 1)
426 Req.Result = 304;
427
428 Req.File.Close();
429 curl_slist_free_all(headers);
430
431 // cleanup
432 if (success != CURLE_OK)
433 {
434 #pragma GCC diagnostic push
435 #pragma GCC diagnostic ignored "-Wswitch"
436 switch (success)
437 {
438 case CURLE_COULDNT_RESOLVE_PROXY:
439 case CURLE_COULDNT_RESOLVE_HOST:
440 SetFailReason("ResolveFailure");
441 break;
442 case CURLE_COULDNT_CONNECT:
443 SetFailReason("ConnectionRefused");
444 break;
445 case CURLE_OPERATION_TIMEDOUT:
446 SetFailReason("Timeout");
447 break;
448 }
449 #pragma GCC diagnostic pop
450 // only take curls technical errors if we haven't our own
451 // (e.g. for the maximum size limit we have and curls can be confusing)
452 if (_error->PendingError() == false)
453 _error->Error("%s", curl_errorstr);
454 else
455 _error->Warning("curl: %s", curl_errorstr);
456 return false;
457 }
458
459 switch (DealWithHeaders(Res, Req))
460 {
461 case BaseHttpMethod::IMS_HIT:
462 URIDone(Res);
463 break;
464
465 case BaseHttpMethod::ERROR_WITH_CONTENT_PAGE:
466 // unlink, no need keep 401/404 page content in partial/
467 RemoveFile(Binary.c_str(), Req.File.Name());
468 case BaseHttpMethod::ERROR_UNRECOVERABLE:
469 case BaseHttpMethod::ERROR_NOT_FROM_SERVER:
470 return false;
471
472 case BaseHttpMethod::TRY_AGAIN_OR_REDIRECT:
473 Redirect(NextURI);
474 break;
475
476 case BaseHttpMethod::FILE_IS_OPEN:
477 struct stat resultStat;
478 if (unlikely(stat(Req.File.Name().c_str(), &resultStat) != 0))
479 {
480 _error->Errno("stat", "Unable to access file %s", Req.File.Name().c_str());
481 return false;
482 }
483 Res.Size = resultStat.st_size;
484
485 // Timestamp
486 curl_easy_getinfo(curl, CURLINFO_FILETIME, &Res.LastModified);
487 if (Res.LastModified != -1)
488 {
489 struct timeval times[2];
490 times[0].tv_sec = Res.LastModified;
491 times[1].tv_sec = Res.LastModified;
492 times[0].tv_usec = times[1].tv_usec = 0;
493 utimes(Req.File.Name().c_str(), times);
494 }
495 else
496 Res.LastModified = resultStat.st_mtime;
497
498 // take hashes
499 Res.TakeHashes(*(Server->GetHashes()));
500
501 // keep apt updated
502 URIDone(Res);
503 break;
504 }
505 return true;
506 }
507 /*}}}*/
508 std::unique_ptr<ServerState> HttpsMethod::CreateServerState(URI const &uri)/*{{{*/
509 {
510 return std::unique_ptr<ServerState>(new HttpsServerState(uri, this));
511 }
512 /*}}}*/
513 HttpsMethod::HttpsMethod(std::string &&pProg) : BaseHttpMethod(std::move(pProg),"1.2",Pipeline | SendConfig)/*{{{*/
514 {
515 auto addName = std::inserter(methodNames, methodNames.begin());
516 addName = "http";
517 auto const plus = Binary.find('+');
518 if (plus != std::string::npos)
519 {
520 addName = Binary.substr(plus + 1);
521 auto base = Binary.substr(0, plus);
522 if (base != "https")
523 addName = base;
524 }
525 if (std::find(methodNames.begin(), methodNames.end(), "https") != methodNames.end())
526 curl_global_init(CURL_GLOBAL_SSL);
527 else
528 curl_global_init(CURL_GLOBAL_NOTHING);
529 curl = curl_easy_init();
530 }
531 /*}}}*/
532 HttpsMethod::~HttpsMethod() /*{{{*/
533 {
534 curl_easy_cleanup(curl);
535 }
536 /*}}}*/
537 int main(int, const char *argv[]) /*{{{*/
538 {
539 std::string Binary = flNotDir(argv[0]);
540 if (Binary.find('+') == std::string::npos && Binary != "https")
541 Binary.append("+https");
542 return HttpsMethod(std::move(Binary)).Run();
543 }
544 /*}}}*/