You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

549 lines
18 KiB

  1. //-*- mode: cpp; mode: fold -*-
  2. // Description /*{{{*/
  3. // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
  4. /* ######################################################################
  5. HTTPS Acquire Method - This is the HTTPS acquire method for APT.
  6. It uses libcurl
  7. ##################################################################### */
  8. /*}}}*/
  9. // Include Files /*{{{*/
  10. #include <config.h>
  11. #include <apt-pkg/configuration.h>
  12. #include <apt-pkg/error.h>
  13. #include <apt-pkg/fileutl.h>
  14. #include <apt-pkg/hashes.h>
  15. #include <apt-pkg/macros.h>
  16. #include <apt-pkg/netrc.h>
  17. #include <apt-pkg/proxy.h>
  18. #include <apt-pkg/strutl.h>
  19. #include <ctype.h>
  20. #include <stdio.h>
  21. #include <stdlib.h>
  22. #include <sys/stat.h>
  23. #include <sys/time.h>
  24. #include <unistd.h>
  25. #include <array>
  26. #include <iostream>
  27. #include <sstream>
  28. #include "curl.h"
  29. #include <apti18n.h>
  30. /*}}}*/
  31. using namespace std;
  32. struct APT_HIDDEN CURLUserPointer {
  33. HttpsMethod * const https;
  34. HttpsMethod::FetchResult * const Res;
  35. HttpsMethod::FetchItem const * const Itm;
  36. RequestState * const Req;
  37. CURLUserPointer(HttpsMethod * const https, HttpsMethod::FetchResult * const Res,
  38. HttpsMethod::FetchItem const * const Itm, RequestState * const Req) : https(https), Res(Res), Itm(Itm), Req(Req) {}
  39. };
  40. size_t
  41. HttpsMethod::parse_header(void *buffer, size_t size, size_t nmemb, void *userp)
  42. {
  43. size_t len = size * nmemb;
  44. CURLUserPointer *me = static_cast<CURLUserPointer *>(userp);
  45. std::string line((char*) buffer, len);
  46. for (--len; len > 0; --len)
  47. if (isspace_ascii(line[len]) == 0)
  48. {
  49. ++len;
  50. break;
  51. }
  52. line.erase(len);
  53. if (line.empty() == true)
  54. {
  55. if (me->Req->File.Open(me->Itm->DestFile, FileFd::WriteAny) == false)
  56. return ERROR_NOT_FROM_SERVER;
  57. me->Req->JunkSize = 0;
  58. if (me->Req->Result != 416 && me->Req->StartPos != 0)
  59. ;
  60. else if (me->Req->Result == 416)
  61. {
  62. bool partialHit = false;
  63. if (me->Itm->ExpectedHashes.usable() == true)
  64. {
  65. Hashes resultHashes(me->Itm->ExpectedHashes);
  66. FileFd file(me->Itm->DestFile, FileFd::ReadOnly);
  67. me->Req->TotalFileSize = file.FileSize();
  68. me->Req->Date = file.ModificationTime();
  69. resultHashes.AddFD(file);
  70. HashStringList const hashList = resultHashes.GetHashStringList();
  71. partialHit = (me->Itm->ExpectedHashes == hashList);
  72. }
  73. else if (me->Req->Result == 416 && me->Req->TotalFileSize == me->Req->File.FileSize())
  74. partialHit = true;
  75. if (partialHit == true)
  76. {
  77. me->Req->Result = 200;
  78. me->Req->StartPos = me->Req->TotalFileSize;
  79. // the actual size is not important for https as curl will deal with it
  80. // by itself and e.g. doesn't bother us with transport-encoding…
  81. me->Req->JunkSize = std::numeric_limits<unsigned long long>::max();
  82. }
  83. else
  84. me->Req->StartPos = 0;
  85. }
  86. else
  87. me->Req->StartPos = 0;
  88. me->Res->LastModified = me->Req->Date;
  89. me->Res->Size = me->Req->TotalFileSize;
  90. me->Res->ResumePoint = me->Req->StartPos;
  91. // we expect valid data, so tell our caller we get the file now
  92. if (me->Req->Result >= 200 && me->Req->Result < 300)
  93. {
  94. if (me->Res->Size != 0 && me->Res->Size > me->Res->ResumePoint)
  95. me->https->URIStart(*me->Res);
  96. if (me->Req->AddPartialFileToHashes(me->Req->File) == false)
  97. return 0;
  98. }
  99. else
  100. me->Req->JunkSize = std::numeric_limits<decltype(me->Req->JunkSize)>::max();
  101. }
  102. else if (me->Req->HeaderLine(line) == false)
  103. return 0;
  104. return size*nmemb;
  105. }
  106. size_t
  107. HttpsMethod::write_data(void *buffer, size_t size, size_t nmemb, void *userp)
  108. {
  109. CURLUserPointer *me = static_cast<CURLUserPointer *>(userp);
  110. size_t buffer_size = size * nmemb;
  111. // we don't need to count the junk here, just drop anything we get as
  112. // we don't always know how long it would be, e.g. in chunked encoding.
  113. if (me->Req->JunkSize != 0)
  114. return buffer_size;
  115. if(me->Req->File.Write(buffer, buffer_size) != true)
  116. return 0;
  117. if(me->https->Queue->MaximumSize > 0)
  118. {
  119. unsigned long long const TotalWritten = me->Req->File.Tell();
  120. if (TotalWritten > me->https->Queue->MaximumSize)
  121. {
  122. me->https->SetFailReason("MaximumSizeExceeded");
  123. _error->Error(_("File has unexpected size (%llu != %llu). Mirror sync in progress?"),
  124. TotalWritten, me->https->Queue->MaximumSize);
  125. return 0;
  126. }
  127. }
  128. if (me->https->Server->GetHashes()->Add((unsigned char const * const)buffer, buffer_size) == false)
  129. return 0;
  130. return buffer_size;
  131. }
  132. // HttpsServerState::HttpsServerState - Constructor /*{{{*/
  133. HttpsServerState::HttpsServerState(URI Srv,HttpsMethod * Owner) : ServerState(Srv, Owner), Hash(NULL)
  134. {
  135. TimeOut = Owner->ConfigFindI("Timeout", TimeOut);
  136. Reset();
  137. }
  138. /*}}}*/
  139. bool HttpsServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/
  140. {
  141. delete Hash;
  142. Hash = new Hashes(ExpectedHashes);
  143. return true;
  144. }
  145. /*}}}*/
  146. APT_PURE Hashes * HttpsServerState::GetHashes() /*{{{*/
  147. {
  148. return Hash;
  149. }
  150. /*}}}*/
  151. bool HttpsMethod::SetupProxy() /*{{{*/
  152. {
  153. URI ServerName = Queue->Uri;
  154. // Determine the proxy setting
  155. AutoDetectProxy(ServerName);
  156. // Curl should never read proxy settings from the environment, as
  157. // we determine which proxy to use. Do this for consistency among
  158. // methods and prevent an environment variable overriding a
  159. // no-proxy ("DIRECT") setting in apt.conf.
  160. curl_easy_setopt(curl, CURLOPT_PROXY, "");
  161. // Determine the proxy setting - try https first, fallback to http and use env at last
  162. string UseProxy = ConfigFind("Proxy::" + ServerName.Host, "");
  163. if (UseProxy.empty() == true)
  164. UseProxy = ConfigFind("Proxy", "");
  165. // User wants to use NO proxy, so nothing to setup
  166. if (UseProxy == "DIRECT")
  167. return true;
  168. // Parse no_proxy, a comma (,) separated list of domains we don't want to use
  169. // a proxy for so we stop right here if it is in the list
  170. if (getenv("no_proxy") != 0 && CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
  171. return true;
  172. if (UseProxy.empty() == true)
  173. {
  174. const char* result = nullptr;
  175. if (std::find(methodNames.begin(), methodNames.end(), "https") != methodNames.end())
  176. result = getenv("https_proxy");
  177. // FIXME: Fall back to http_proxy is to remain compatible with
  178. // existing setups and behaviour of apt.conf. This should be
  179. // deprecated in the future (including apt.conf). Most other
  180. // programs do not fall back to http proxy settings and neither
  181. // should Apt.
  182. if (result == nullptr && std::find(methodNames.begin(), methodNames.end(), "http") != methodNames.end())
  183. result = getenv("http_proxy");
  184. UseProxy = result == nullptr ? "" : result;
  185. }
  186. // Determine what host and port to use based on the proxy settings
  187. if (UseProxy.empty() == false)
  188. {
  189. Proxy = UseProxy;
  190. AddProxyAuth(Proxy, ServerName);
  191. if (Proxy.Access == "socks5h")
  192. curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
  193. else if (Proxy.Access == "socks5")
  194. curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
  195. else if (Proxy.Access == "socks4a")
  196. curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4A);
  197. else if (Proxy.Access == "socks")
  198. curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4);
  199. else if (Proxy.Access == "http" || Proxy.Access == "https")
  200. curl_easy_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
  201. else
  202. return false;
  203. if (Proxy.Port != 1)
  204. curl_easy_setopt(curl, CURLOPT_PROXYPORT, Proxy.Port);
  205. curl_easy_setopt(curl, CURLOPT_PROXY, Proxy.Host.c_str());
  206. if (Proxy.User.empty() == false || Proxy.Password.empty() == false)
  207. {
  208. curl_easy_setopt(curl, CURLOPT_PROXYUSERNAME, Proxy.User.c_str());
  209. curl_easy_setopt(curl, CURLOPT_PROXYPASSWORD, Proxy.Password.c_str());
  210. }
  211. }
  212. return true;
  213. } /*}}}*/
  214. // HttpsMethod::Fetch - Fetch an item /*{{{*/
  215. // ---------------------------------------------------------------------
  216. /* This adds an item to the pipeline. We keep the pipeline at a fixed
  217. depth. */
  218. bool HttpsMethod::Fetch(FetchItem *Itm)
  219. {
  220. struct stat SBuf;
  221. struct curl_slist *headers=NULL;
  222. char curl_errorstr[CURL_ERROR_SIZE];
  223. URI Uri = Itm->Uri;
  224. setPostfixForMethodNames(Uri.Host.c_str());
  225. AllowRedirect = ConfigFindB("AllowRedirect", true);
  226. Debug = DebugEnabled();
  227. // TODO:
  228. // - http::Pipeline-Depth
  229. // - error checking/reporting
  230. // - more debug options? (CURLOPT_DEBUGFUNCTION?)
  231. {
  232. auto const plus = Binary.find('+');
  233. if (plus != std::string::npos)
  234. Uri.Access = Binary.substr(plus + 1);
  235. }
  236. curl_easy_reset(curl);
  237. if (SetupProxy() == false)
  238. return _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str());
  239. MaybeAddAuthTo(Uri);
  240. if (Server == nullptr || Server->Comp(Itm->Uri) == false)
  241. Server = CreateServerState(Itm->Uri);
  242. // The "+" is encoded as a workaround for a amazon S3 bug
  243. // see LP bugs #1003633 and #1086997. (taken from http method)
  244. Uri.Path = QuoteString(Uri.Path, "+~ ");
  245. FetchResult Res;
  246. RequestState Req(this, Server.get());
  247. CURLUserPointer userp(this, &Res, Itm, &Req);
  248. // callbacks
  249. curl_easy_setopt(curl, CURLOPT_URL, static_cast<string>(Uri).c_str());
  250. curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, parse_header);
  251. curl_easy_setopt(curl, CURLOPT_WRITEHEADER, &userp);
  252. curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_data);
  253. curl_easy_setopt(curl, CURLOPT_WRITEDATA, &userp);
  254. // options
  255. curl_easy_setopt(curl, CURLOPT_NOPROGRESS, true);
  256. curl_easy_setopt(curl, CURLOPT_FILETIME, true);
  257. curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 0);
  258. if (std::find(methodNames.begin(), methodNames.end(), "https") != methodNames.end())
  259. {
  260. curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTPS);
  261. curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS);
  262. // File containing the list of trusted CA.
  263. std::string const cainfo = ConfigFind("CaInfo", "");
  264. if(cainfo.empty() == false)
  265. curl_easy_setopt(curl, CURLOPT_CAINFO, cainfo.c_str());
  266. // Check server certificate against previous CA list ...
  267. curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, ConfigFindB("Verify-Peer", true) ? 1 : 0);
  268. // ... and hostname against cert CN or subjectAltName
  269. curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, ConfigFindB("Verify-Host", true) ? 2 : 0);
  270. // Also enforce issuer of server certificate using its cert
  271. std::string const issuercert = ConfigFind("IssuerCert", "");
  272. if(issuercert.empty() == false)
  273. curl_easy_setopt(curl, CURLOPT_ISSUERCERT, issuercert.c_str());
  274. // For client authentication, certificate file ...
  275. std::string const pem = ConfigFind("SslCert", "");
  276. if(pem.empty() == false)
  277. curl_easy_setopt(curl, CURLOPT_SSLCERT, pem.c_str());
  278. // ... and associated key.
  279. std::string const key = ConfigFind("SslKey", "");
  280. if(key.empty() == false)
  281. curl_easy_setopt(curl, CURLOPT_SSLKEY, key.c_str());
  282. // Allow forcing SSL version to SSLv3 or TLSv1
  283. long final_version = CURL_SSLVERSION_DEFAULT;
  284. std::string const sslversion = ConfigFind("SslForceVersion", "");
  285. if(sslversion == "TLSv1")
  286. final_version = CURL_SSLVERSION_TLSv1;
  287. else if(sslversion == "TLSv1.0")
  288. final_version = CURL_SSLVERSION_TLSv1_0;
  289. else if(sslversion == "TLSv1.1")
  290. final_version = CURL_SSLVERSION_TLSv1_1;
  291. else if(sslversion == "TLSv1.2")
  292. final_version = CURL_SSLVERSION_TLSv1_2;
  293. else if(sslversion == "SSLv3")
  294. final_version = CURL_SSLVERSION_SSLv3;
  295. curl_easy_setopt(curl, CURLOPT_SSLVERSION, final_version);
  296. // CRL file
  297. std::string const crlfile = ConfigFind("CrlFile", "");
  298. if(crlfile.empty() == false)
  299. curl_easy_setopt(curl, CURLOPT_CRLFILE, crlfile.c_str());
  300. }
  301. else
  302. {
  303. curl_easy_setopt(curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP);
  304. curl_easy_setopt(curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP);
  305. }
  306. // cache-control
  307. if(ConfigFindB("No-Cache", false) == false)
  308. {
  309. // cache enabled
  310. if (ConfigFindB("No-Store", false) == true)
  311. headers = curl_slist_append(headers,"Cache-Control: no-store");
  312. std::string ss;
  313. strprintf(ss, "Cache-Control: max-age=%u", ConfigFindI("Max-Age", 0));
  314. headers = curl_slist_append(headers, ss.c_str());
  315. } else {
  316. // cache disabled by user
  317. headers = curl_slist_append(headers, "Cache-Control: no-cache");
  318. headers = curl_slist_append(headers, "Pragma: no-cache");
  319. }
  320. curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
  321. // speed limit
  322. int const dlLimit = ConfigFindI("Dl-Limit", 0) * 1024;
  323. if (dlLimit > 0)
  324. curl_easy_setopt(curl, CURLOPT_MAX_RECV_SPEED_LARGE, dlLimit);
  325. // set header
  326. curl_easy_setopt(curl, CURLOPT_USERAGENT, ConfigFind("User-Agent", "Debian APT-CURL/1.0 (" PACKAGE_VERSION ")").c_str());
  327. // set timeout
  328. int const timeout = ConfigFindI("Timeout", 120);
  329. curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, timeout);
  330. //set really low lowspeed timeout (see #497983)
  331. curl_easy_setopt(curl, CURLOPT_LOW_SPEED_LIMIT, DL_MIN_SPEED);
  332. curl_easy_setopt(curl, CURLOPT_LOW_SPEED_TIME, timeout);
  333. if(_config->FindB("Acquire::ForceIPv4", false) == true)
  334. curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
  335. else if(_config->FindB("Acquire::ForceIPv6", false) == true)
  336. curl_easy_setopt(curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6);
  337. // debug
  338. if (Debug == true)
  339. curl_easy_setopt(curl, CURLOPT_VERBOSE, true);
  340. // error handling
  341. curl_errorstr[0] = '\0';
  342. curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_errorstr);
  343. // If we ask for uncompressed files servers might respond with content-
  344. // negotiation which lets us end up with compressed files we do not support,
  345. // see 657029, 657560 and co, so if we have no extension on the request
  346. // ask for text only. As a sidenote: If there is nothing to negotate servers
  347. // seem to be nice and ignore it.
  348. if (ConfigFindB("SendAccept", true))
  349. {
  350. size_t const filepos = Itm->Uri.find_last_of('/');
  351. string const file = Itm->Uri.substr(filepos + 1);
  352. if (flExtension(file) == file)
  353. headers = curl_slist_append(headers, "Accept: text/*");
  354. }
  355. // if we have the file send an if-range query with a range header
  356. if (Server->RangesAllowed && stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
  357. {
  358. std::string Buf;
  359. strprintf(Buf, "Range: bytes=%lli-", (long long) SBuf.st_size);
  360. headers = curl_slist_append(headers, Buf.c_str());
  361. strprintf(Buf, "If-Range: %s", TimeRFC1123(SBuf.st_mtime, false).c_str());
  362. headers = curl_slist_append(headers, Buf.c_str());
  363. }
  364. else if(Itm->LastModified > 0)
  365. {
  366. curl_easy_setopt(curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
  367. curl_easy_setopt(curl, CURLOPT_TIMEVALUE, Itm->LastModified);
  368. }
  369. if (Server->InitHashes(Itm->ExpectedHashes) == false)
  370. return false;
  371. // keep apt updated
  372. Res.Filename = Itm->DestFile;
  373. // get it!
  374. CURLcode success = curl_easy_perform(curl);
  375. // If the server returns 200 OK but the If-Modified-Since condition is not
  376. // met, CURLINFO_CONDITION_UNMET will be set to 1
  377. long curl_condition_unmet = 0;
  378. curl_easy_getinfo(curl, CURLINFO_CONDITION_UNMET, &curl_condition_unmet);
  379. if (curl_condition_unmet == 1)
  380. Req.Result = 304;
  381. Req.File.Close();
  382. curl_slist_free_all(headers);
  383. // cleanup
  384. if (success != CURLE_OK)
  385. {
  386. #pragma GCC diagnostic push
  387. #pragma GCC diagnostic ignored "-Wswitch"
  388. switch (success)
  389. {
  390. case CURLE_COULDNT_RESOLVE_PROXY:
  391. case CURLE_COULDNT_RESOLVE_HOST:
  392. SetFailReason("ResolveFailure");
  393. break;
  394. case CURLE_COULDNT_CONNECT:
  395. SetFailReason("ConnectionRefused");
  396. break;
  397. case CURLE_OPERATION_TIMEDOUT:
  398. SetFailReason("Timeout");
  399. break;
  400. }
  401. #pragma GCC diagnostic pop
  402. // only take curls technical errors if we haven't our own
  403. // (e.g. for the maximum size limit we have and curls can be confusing)
  404. if (_error->PendingError() == false)
  405. _error->Error("%s", curl_errorstr);
  406. else
  407. _error->Warning("curl: %s", curl_errorstr);
  408. return false;
  409. }
  410. switch (DealWithHeaders(Res, Req))
  411. {
  412. case BaseHttpMethod::IMS_HIT:
  413. URIDone(Res);
  414. break;
  415. case BaseHttpMethod::ERROR_WITH_CONTENT_PAGE:
  416. // unlink, no need keep 401/404 page content in partial/
  417. RemoveFile(Binary.c_str(), Req.File.Name());
  418. // Fall through.
  419. case BaseHttpMethod::ERROR_UNRECOVERABLE:
  420. case BaseHttpMethod::ERROR_NOT_FROM_SERVER:
  421. return false;
  422. case BaseHttpMethod::TRY_AGAIN_OR_REDIRECT:
  423. Redirect(NextURI);
  424. break;
  425. case BaseHttpMethod::FILE_IS_OPEN:
  426. struct stat resultStat;
  427. if (unlikely(stat(Req.File.Name().c_str(), &resultStat) != 0))
  428. {
  429. _error->Errno("stat", "Unable to access file %s", Req.File.Name().c_str());
  430. return false;
  431. }
  432. Res.Size = resultStat.st_size;
  433. // Timestamp
  434. curl_easy_getinfo(curl, CURLINFO_FILETIME, &Res.LastModified);
  435. if (Res.LastModified != -1)
  436. {
  437. struct timeval times[2];
  438. times[0].tv_sec = Res.LastModified;
  439. times[1].tv_sec = Res.LastModified;
  440. times[0].tv_usec = times[1].tv_usec = 0;
  441. utimes(Req.File.Name().c_str(), times);
  442. }
  443. else
  444. Res.LastModified = resultStat.st_mtime;
  445. // take hashes
  446. Res.TakeHashes(*(Server->GetHashes()));
  447. // keep apt updated
  448. URIDone(Res);
  449. break;
  450. }
  451. return true;
  452. }
  453. /*}}}*/
  454. std::unique_ptr<ServerState> HttpsMethod::CreateServerState(URI const &uri)/*{{{*/
  455. {
  456. return std::unique_ptr<ServerState>(new HttpsServerState(uri, this));
  457. }
  458. /*}}}*/
  459. HttpsMethod::HttpsMethod(std::string &&pProg) : BaseHttpMethod(std::move(pProg),"1.2",Pipeline | SendConfig)/*{{{*/
  460. {
  461. auto addName = std::inserter(methodNames, methodNames.begin());
  462. addName = "http";
  463. auto const plus = Binary.find('+');
  464. if (plus != std::string::npos)
  465. {
  466. addName = Binary.substr(plus + 1);
  467. auto base = Binary.substr(0, plus);
  468. if (base != "https")
  469. addName = base;
  470. }
  471. if (std::find(methodNames.begin(), methodNames.end(), "https") != methodNames.end())
  472. curl_global_init(CURL_GLOBAL_SSL);
  473. else
  474. curl_global_init(CURL_GLOBAL_NOTHING);
  475. curl = curl_easy_init();
  476. }
  477. /*}}}*/
  478. HttpsMethod::~HttpsMethod() /*{{{*/
  479. {
  480. curl_easy_cleanup(curl);
  481. }
  482. /*}}}*/
  483. int main(int, const char *argv[]) /*{{{*/
  484. {
  485. std::string Binary = flNotDir(argv[0]);
  486. if (Binary.find('+') == std::string::npos && Binary != "https")
  487. Binary.append("+https");
  488. return HttpsMethod(std::move(Binary)).Run();
  489. }
  490. /*}}}*/