You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

987 lines
30 KiB

  1. // -*- mode: cpp; mode: fold -*-
  2. // Description /*{{{*/
  3. // $Id: http.cc,v 1.59 2004/05/08 19:42:35 mdz Exp $
  4. /* ######################################################################
  5. HTTP Acquire Method - This is the HTTP acquire method for APT.
  6. It uses HTTP/1.1 and many of the fancy options there-in, such as
  7. pipelining, range, if-range and so on.
  8. It is based on a doubly buffered select loop. A groupe of requests are
  9. fed into a single output buffer that is constantly fed out the
  10. socket. This provides ideal pipelining as in many cases all of the
  11. requests will fit into a single packet. The input socket is buffered
  12. the same way and fed into the fd for the file (may be a pipe in future).
  13. This double buffering provides fairly substantial transfer rates,
  14. compared to wget the http method is about 4% faster. Most importantly,
  15. when HTTP is compared with FTP as a protocol the speed difference is
  16. huge. In tests over the internet from two sites to llug (via ATM) this
  17. program got 230k/s sustained http transfer rates. FTP on the other
  18. hand topped out at 170k/s. That combined with the time to setup the
  19. FTP connection makes HTTP a vastly superior protocol.
  20. ##################################################################### */
  21. /*}}}*/
  22. // Include Files /*{{{*/
  23. #include <config.h>
  24. #include <apt-pkg/fileutl.h>
  25. #include <apt-pkg/configuration.h>
  26. #include <apt-pkg/error.h>
  27. #include <apt-pkg/hashes.h>
  28. #include <apt-pkg/netrc.h>
  29. #include <apt-pkg/strutl.h>
  30. #include <apt-pkg/proxy.h>
  31. #include <stddef.h>
  32. #include <stdlib.h>
  33. #include <sys/select.h>
  34. #include <cstring>
  35. #include <sys/stat.h>
  36. #include <sys/time.h>
  37. #include <unistd.h>
  38. #include <stdio.h>
  39. #include <errno.h>
  40. #include <arpa/inet.h>
  41. #include <iostream>
  42. #include <sstream>
  43. #include "config.h"
  44. #include "connect.h"
  45. #include "http.h"
  46. #include <apti18n.h>
  47. /*}}}*/
  48. using namespace std;
  49. unsigned long long CircleBuf::BwReadLimit=0;
  50. unsigned long long CircleBuf::BwTickReadData=0;
  51. struct timeval CircleBuf::BwReadTick={0,0};
  52. const unsigned int CircleBuf::BW_HZ=10;
  53. // CircleBuf::CircleBuf - Circular input buffer /*{{{*/
  54. // ---------------------------------------------------------------------
  55. /* */
  56. CircleBuf::CircleBuf(HttpMethod const * const Owner, unsigned long long Size)
  57. : Size(Size), Hash(NULL), TotalWriten(0)
  58. {
  59. Buf = new unsigned char[Size];
  60. Reset();
  61. CircleBuf::BwReadLimit = Owner->ConfigFindI("Dl-Limit", 0) * 1024;
  62. }
  63. /*}}}*/
  64. // CircleBuf::Reset - Reset to the default state /*{{{*/
  65. // ---------------------------------------------------------------------
  66. /* */
  67. void CircleBuf::Reset()
  68. {
  69. InP = 0;
  70. OutP = 0;
  71. StrPos = 0;
  72. TotalWriten = 0;
  73. MaxGet = (unsigned long long)-1;
  74. OutQueue = string();
  75. if (Hash != NULL)
  76. {
  77. delete Hash;
  78. Hash = NULL;
  79. }
  80. }
  81. /*}}}*/
  82. // CircleBuf::Read - Read from a FD into the circular buffer /*{{{*/
  83. // ---------------------------------------------------------------------
  84. /* This fills up the buffer with as much data as is in the FD, assuming it
  85. is non-blocking.. */
  86. bool CircleBuf::Read(int Fd)
  87. {
  88. while (1)
  89. {
  90. // Woops, buffer is full
  91. if (InP - OutP == Size)
  92. return true;
  93. // what's left to read in this tick
  94. unsigned long long const BwReadMax = CircleBuf::BwReadLimit/BW_HZ;
  95. if(CircleBuf::BwReadLimit) {
  96. struct timeval now;
  97. gettimeofday(&now,0);
  98. unsigned long long d = (now.tv_sec-CircleBuf::BwReadTick.tv_sec)*1000000 +
  99. now.tv_usec-CircleBuf::BwReadTick.tv_usec;
  100. if(d > 1000000/BW_HZ) {
  101. CircleBuf::BwReadTick = now;
  102. CircleBuf::BwTickReadData = 0;
  103. }
  104. if(CircleBuf::BwTickReadData >= BwReadMax) {
  105. usleep(1000000/BW_HZ);
  106. return true;
  107. }
  108. }
  109. // Write the buffer segment
  110. ssize_t Res;
  111. if(CircleBuf::BwReadLimit) {
  112. Res = read(Fd,Buf + (InP%Size),
  113. BwReadMax > LeftRead() ? LeftRead() : BwReadMax);
  114. } else
  115. Res = read(Fd,Buf + (InP%Size),LeftRead());
  116. if(Res > 0 && BwReadLimit > 0)
  117. CircleBuf::BwTickReadData += Res;
  118. if (Res == 0)
  119. return false;
  120. if (Res < 0)
  121. {
  122. if (errno == EAGAIN)
  123. return true;
  124. return false;
  125. }
  126. if (InP == 0)
  127. gettimeofday(&Start,0);
  128. InP += Res;
  129. }
  130. }
  131. /*}}}*/
  132. // CircleBuf::Read - Put the string into the buffer /*{{{*/
  133. // ---------------------------------------------------------------------
  134. /* This will hold the string in and fill the buffer with it as it empties */
  135. bool CircleBuf::Read(string const &Data)
  136. {
  137. OutQueue.append(Data);
  138. FillOut();
  139. return true;
  140. }
  141. /*}}}*/
  142. // CircleBuf::FillOut - Fill the buffer from the output queue /*{{{*/
  143. // ---------------------------------------------------------------------
  144. /* */
  145. void CircleBuf::FillOut()
  146. {
  147. if (OutQueue.empty() == true)
  148. return;
  149. while (1)
  150. {
  151. // Woops, buffer is full
  152. if (InP - OutP == Size)
  153. return;
  154. // Write the buffer segment
  155. unsigned long long Sz = LeftRead();
  156. if (OutQueue.length() - StrPos < Sz)
  157. Sz = OutQueue.length() - StrPos;
  158. memcpy(Buf + (InP%Size),OutQueue.c_str() + StrPos,Sz);
  159. // Advance
  160. StrPos += Sz;
  161. InP += Sz;
  162. if (OutQueue.length() == StrPos)
  163. {
  164. StrPos = 0;
  165. OutQueue = "";
  166. return;
  167. }
  168. }
  169. }
  170. /*}}}*/
  171. // CircleBuf::Write - Write from the buffer into a FD /*{{{*/
  172. // ---------------------------------------------------------------------
  173. /* This empties the buffer into the FD. */
  174. bool CircleBuf::Write(int Fd)
  175. {
  176. while (1)
  177. {
  178. FillOut();
  179. // Woops, buffer is empty
  180. if (OutP == InP)
  181. return true;
  182. if (OutP == MaxGet)
  183. return true;
  184. // Write the buffer segment
  185. ssize_t Res;
  186. Res = write(Fd,Buf + (OutP%Size),LeftWrite());
  187. if (Res == 0)
  188. return false;
  189. if (Res < 0)
  190. {
  191. if (errno == EAGAIN)
  192. return true;
  193. return false;
  194. }
  195. TotalWriten += Res;
  196. if (Hash != NULL)
  197. Hash->Add(Buf + (OutP%Size),Res);
  198. OutP += Res;
  199. }
  200. }
  201. /*}}}*/
  202. // CircleBuf::WriteTillEl - Write from the buffer to a string /*{{{*/
  203. // ---------------------------------------------------------------------
  204. /* This copies till the first empty line */
  205. bool CircleBuf::WriteTillEl(string &Data,bool Single)
  206. {
  207. // We cheat and assume it is unneeded to have more than one buffer load
  208. for (unsigned long long I = OutP; I < InP; I++)
  209. {
  210. if (Buf[I%Size] != '\n')
  211. continue;
  212. ++I;
  213. if (Single == false)
  214. {
  215. if (I < InP && Buf[I%Size] == '\r')
  216. ++I;
  217. if (I >= InP || Buf[I%Size] != '\n')
  218. continue;
  219. ++I;
  220. }
  221. Data = "";
  222. while (OutP < I)
  223. {
  224. unsigned long long Sz = LeftWrite();
  225. if (Sz == 0)
  226. return false;
  227. if (I - OutP < Sz)
  228. Sz = I - OutP;
  229. Data += string((char *)(Buf + (OutP%Size)),Sz);
  230. OutP += Sz;
  231. }
  232. return true;
  233. }
  234. return false;
  235. }
  236. /*}}}*/
  237. // CircleBuf::Stats - Print out stats information /*{{{*/
  238. // ---------------------------------------------------------------------
  239. /* */
  240. void CircleBuf::Stats()
  241. {
  242. if (InP == 0)
  243. return;
  244. struct timeval Stop;
  245. gettimeofday(&Stop,0);
  246. /* float Diff = Stop.tv_sec - Start.tv_sec +
  247. (float)(Stop.tv_usec - Start.tv_usec)/1000000;
  248. clog << "Got " << InP << " in " << Diff << " at " << InP/Diff << endl;*/
  249. }
  250. /*}}}*/
  251. CircleBuf::~CircleBuf()
  252. {
  253. delete [] Buf;
  254. delete Hash;
  255. }
  256. // HttpServerState::HttpServerState - Constructor /*{{{*/
  257. HttpServerState::HttpServerState(URI Srv,HttpMethod *Owner) : ServerState(Srv, Owner), In(Owner, 64*1024), Out(Owner, 4*1024)
  258. {
  259. TimeOut = Owner->ConfigFindI("Timeout", TimeOut);
  260. Reset();
  261. }
  262. /*}}}*/
  263. // HttpServerState::Open - Open a connection to the server /*{{{*/
  264. // ---------------------------------------------------------------------
  265. /* This opens a connection to the server. */
  266. static bool TalkToSocksProxy(int const ServerFd, std::string const &Proxy,
  267. char const * const type, bool const ReadWrite, uint8_t * const ToFrom,
  268. unsigned int const Size, unsigned int const Timeout)
  269. {
  270. if (WaitFd(ServerFd, ReadWrite, Timeout) == false)
  271. return _error->Error("Waiting for the SOCKS proxy %s to %s timed out", URI::SiteOnly(Proxy).c_str(), type);
  272. if (ReadWrite == false)
  273. {
  274. if (FileFd::Read(ServerFd, ToFrom, Size) == false)
  275. return _error->Error("Reading the %s from SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
  276. }
  277. else
  278. {
  279. if (FileFd::Write(ServerFd, ToFrom, Size) == false)
  280. return _error->Error("Writing the %s to SOCKS proxy %s failed", type, URI::SiteOnly(Proxy).c_str());
  281. }
  282. return true;
  283. }
  284. bool HttpServerState::Open()
  285. {
  286. // Use the already open connection if possible.
  287. if (ServerFd != -1)
  288. return true;
  289. Close();
  290. In.Reset();
  291. Out.Reset();
  292. Persistent = true;
  293. // Determine the proxy setting
  294. AutoDetectProxy(ServerName);
  295. string SpecificProxy = Owner->ConfigFind("Proxy::" + ServerName.Host, "");
  296. if (!SpecificProxy.empty())
  297. {
  298. if (SpecificProxy == "DIRECT")
  299. Proxy = "";
  300. else
  301. Proxy = SpecificProxy;
  302. }
  303. else
  304. {
  305. string DefProxy = Owner->ConfigFind("Proxy", "");
  306. if (!DefProxy.empty())
  307. {
  308. Proxy = DefProxy;
  309. }
  310. else
  311. {
  312. char* result = getenv("http_proxy");
  313. Proxy = result ? result : "";
  314. }
  315. }
  316. // Parse no_proxy, a , separated list of domains
  317. if (getenv("no_proxy") != 0)
  318. {
  319. if (CheckDomainList(ServerName.Host,getenv("no_proxy")) == true)
  320. Proxy = "";
  321. }
  322. if (Proxy.empty() == false)
  323. Owner->AddProxyAuth(Proxy, ServerName);
  324. if (Proxy.Access == "socks5h")
  325. {
  326. if (Connect(Proxy.Host, Proxy.Port, "socks", 1080, ServerFd, TimeOut, Owner) == false)
  327. return false;
  328. /* We implement a very basic SOCKS5 client here complying mostly to RFC1928 expect
  329. * for not offering GSSAPI auth which is a must (we only do no or user/pass auth).
  330. * We also expect the SOCKS5 server to do hostname lookup (aka socks5h) */
  331. std::string const ProxyInfo = URI::SiteOnly(Proxy);
  332. Owner->Status(_("Connecting to %s (%s)"),"SOCKS5h proxy",ProxyInfo.c_str());
  333. auto const Timeout = Owner->ConfigFindI("TimeOut", 120);
  334. #define APT_WriteOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, true, DATA, LENGTH, Timeout) == false) return false
  335. #define APT_ReadOrFail(TYPE, DATA, LENGTH) if (TalkToSocksProxy(ServerFd, ProxyInfo, TYPE, false, DATA, LENGTH, Timeout) == false) return false
  336. if (ServerName.Host.length() > 255)
  337. return _error->Error("Can't use SOCKS5h as hostname %s is too long!", ServerName.Host.c_str());
  338. if (Proxy.User.length() > 255 || Proxy.Password.length() > 255)
  339. return _error->Error("Can't use user&pass auth as they are too long (%lu and %lu) for the SOCKS5!", Proxy.User.length(), Proxy.Password.length());
  340. if (Proxy.User.empty())
  341. {
  342. uint8_t greeting[] = { 0x05, 0x01, 0x00 };
  343. APT_WriteOrFail("greet-1", greeting, sizeof(greeting));
  344. }
  345. else
  346. {
  347. uint8_t greeting[] = { 0x05, 0x02, 0x00, 0x02 };
  348. APT_WriteOrFail("greet-2", greeting, sizeof(greeting));
  349. }
  350. uint8_t greeting[2];
  351. APT_ReadOrFail("greet back", greeting, sizeof(greeting));
  352. if (greeting[0] != 0x05)
  353. return _error->Error("SOCKS proxy %s greets back with wrong version: %d", ProxyInfo.c_str(), greeting[0]);
  354. if (greeting[1] == 0x00)
  355. ; // no auth has no method-dependent sub-negotiations
  356. else if (greeting[1] == 0x02)
  357. {
  358. if (Proxy.User.empty())
  359. return _error->Error("SOCKS proxy %s negotiated user&pass auth, but we had not offered it!", ProxyInfo.c_str());
  360. // user&pass auth sub-negotiations are defined by RFC1929
  361. std::vector<uint8_t> auth = {{ 0x01, static_cast<uint8_t>(Proxy.User.length()) }};
  362. std::copy(Proxy.User.begin(), Proxy.User.end(), std::back_inserter(auth));
  363. auth.push_back(static_cast<uint8_t>(Proxy.Password.length()));
  364. std::copy(Proxy.Password.begin(), Proxy.Password.end(), std::back_inserter(auth));
  365. APT_WriteOrFail("user&pass auth", auth.data(), auth.size());
  366. uint8_t authstatus[2];
  367. APT_ReadOrFail("auth report", authstatus, sizeof(authstatus));
  368. if (authstatus[0] != 0x01)
  369. return _error->Error("SOCKS proxy %s auth status response with wrong version: %d", ProxyInfo.c_str(), authstatus[0]);
  370. if (authstatus[1] != 0x00)
  371. return _error->Error("SOCKS proxy %s reported authorization failure: username or password incorrect? (%d)", ProxyInfo.c_str(), authstatus[1]);
  372. }
  373. else
  374. return _error->Error("SOCKS proxy %s greets back having not found a common authorization method: %d", ProxyInfo.c_str(), greeting[1]);
  375. union { uint16_t * i; uint8_t * b; } portu;
  376. uint16_t port = htons(static_cast<uint16_t>(ServerName.Port == 0 ? 80 : ServerName.Port));
  377. portu.i = &port;
  378. std::vector<uint8_t> request = {{ 0x05, 0x01, 0x00, 0x03, static_cast<uint8_t>(ServerName.Host.length()) }};
  379. std::copy(ServerName.Host.begin(), ServerName.Host.end(), std::back_inserter(request));
  380. request.push_back(portu.b[0]);
  381. request.push_back(portu.b[1]);
  382. APT_WriteOrFail("request", request.data(), request.size());
  383. uint8_t response[4];
  384. APT_ReadOrFail("first part of response", response, sizeof(response));
  385. if (response[0] != 0x05)
  386. return _error->Error("SOCKS proxy %s response with wrong version: %d", ProxyInfo.c_str(), response[0]);
  387. if (response[2] != 0x00)
  388. return _error->Error("SOCKS proxy %s has unexpected non-zero reserved field value: %d", ProxyInfo.c_str(), response[2]);
  389. std::string bindaddr;
  390. if (response[3] == 0x01) // IPv4 address
  391. {
  392. uint8_t ip4port[6];
  393. APT_ReadOrFail("IPv4+Port of response", ip4port, sizeof(ip4port));
  394. portu.b[0] = ip4port[4];
  395. portu.b[1] = ip4port[5];
  396. port = ntohs(*portu.i);
  397. strprintf(bindaddr, "%d.%d.%d.%d:%d", ip4port[0], ip4port[1], ip4port[2], ip4port[3], port);
  398. }
  399. else if (response[3] == 0x03) // hostname
  400. {
  401. uint8_t namelength;
  402. APT_ReadOrFail("hostname length of response", &namelength, 1);
  403. uint8_t hostname[namelength + 2];
  404. APT_ReadOrFail("hostname of response", hostname, sizeof(hostname));
  405. portu.b[0] = hostname[namelength];
  406. portu.b[1] = hostname[namelength + 1];
  407. port = ntohs(*portu.i);
  408. hostname[namelength] = '\0';
  409. strprintf(bindaddr, "%s:%d", hostname, port);
  410. }
  411. else if (response[3] == 0x04) // IPv6 address
  412. {
  413. uint8_t ip6port[18];
  414. APT_ReadOrFail("IPv6+port of response", ip6port, sizeof(ip6port));
  415. portu.b[0] = ip6port[16];
  416. portu.b[1] = ip6port[17];
  417. port = ntohs(*portu.i);
  418. strprintf(bindaddr, "[%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X:%02X%02X]:%d",
  419. ip6port[0], ip6port[1], ip6port[2], ip6port[3], ip6port[4], ip6port[5], ip6port[6], ip6port[7],
  420. ip6port[8], ip6port[9], ip6port[10], ip6port[11], ip6port[12], ip6port[13], ip6port[14], ip6port[15],
  421. port);
  422. }
  423. else
  424. return _error->Error("SOCKS proxy %s destination address is of unknown type: %d",
  425. ProxyInfo.c_str(), response[3]);
  426. if (response[1] != 0x00)
  427. {
  428. char const * errstr;
  429. switch (response[1])
  430. {
  431. case 0x01: errstr = "general SOCKS server failure"; Owner->SetFailReason("SOCKS"); break;
  432. case 0x02: errstr = "connection not allowed by ruleset"; Owner->SetFailReason("SOCKS"); break;
  433. case 0x03: errstr = "Network unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break;
  434. case 0x04: errstr = "Host unreachable"; Owner->SetFailReason("ConnectionTimedOut"); break;
  435. case 0x05: errstr = "Connection refused"; Owner->SetFailReason("ConnectionRefused"); break;
  436. case 0x06: errstr = "TTL expired"; Owner->SetFailReason("Timeout"); break;
  437. case 0x07: errstr = "Command not supported"; Owner->SetFailReason("SOCKS"); break;
  438. case 0x08: errstr = "Address type not supported"; Owner->SetFailReason("SOCKS"); break;
  439. default: errstr = "Unknown error"; Owner->SetFailReason("SOCKS"); break;
  440. }
  441. return _error->Error("SOCKS proxy %s didn't grant the connect to %s due to: %s (%d)", ProxyInfo.c_str(), bindaddr.c_str(), errstr, response[1]);
  442. }
  443. else if (Owner->DebugEnabled())
  444. ioprintf(std::clog, "http: SOCKS proxy %s connection established to %s\n", ProxyInfo.c_str(), bindaddr.c_str());
  445. if (WaitFd(ServerFd, true, Timeout) == false)
  446. return _error->Error("SOCKS proxy %s reported connection, but timed out", ProxyInfo.c_str());
  447. #undef APT_ReadOrFail
  448. #undef APT_WriteOrFail
  449. }
  450. else
  451. {
  452. // Determine what host and port to use based on the proxy settings
  453. int Port = 0;
  454. string Host;
  455. if (Proxy.empty() == true || Proxy.Host.empty() == true)
  456. {
  457. if (ServerName.Port != 0)
  458. Port = ServerName.Port;
  459. Host = ServerName.Host;
  460. }
  461. else if (Proxy.Access != "http")
  462. return _error->Error("Unsupported proxy configured: %s", URI::SiteOnly(Proxy).c_str());
  463. else
  464. {
  465. if (Proxy.Port != 0)
  466. Port = Proxy.Port;
  467. Host = Proxy.Host;
  468. }
  469. return Connect(Host,Port,"http",80,ServerFd,TimeOut,Owner);
  470. }
  471. return true;
  472. }
  473. /*}}}*/
  474. // HttpServerState::Close - Close a connection to the server /*{{{*/
  475. // ---------------------------------------------------------------------
  476. /* */
  477. bool HttpServerState::Close()
  478. {
  479. close(ServerFd);
  480. ServerFd = -1;
  481. return true;
  482. }
  483. /*}}}*/
  484. // HttpServerState::RunData - Transfer the data from the socket /*{{{*/
  485. bool HttpServerState::RunData(FileFd * const File)
  486. {
  487. State = Data;
  488. // Chunked transfer encoding is fun..
  489. if (Encoding == Chunked)
  490. {
  491. while (1)
  492. {
  493. // Grab the block size
  494. bool Last = true;
  495. string Data;
  496. In.Limit(-1);
  497. do
  498. {
  499. if (In.WriteTillEl(Data,true) == true)
  500. break;
  501. }
  502. while ((Last = Go(false, File)) == true);
  503. if (Last == false)
  504. return false;
  505. // See if we are done
  506. unsigned long long Len = strtoull(Data.c_str(),0,16);
  507. if (Len == 0)
  508. {
  509. In.Limit(-1);
  510. // We have to remove the entity trailer
  511. Last = true;
  512. do
  513. {
  514. if (In.WriteTillEl(Data,true) == true && Data.length() <= 2)
  515. break;
  516. }
  517. while ((Last = Go(false, File)) == true);
  518. if (Last == false)
  519. return false;
  520. return !_error->PendingError();
  521. }
  522. // Transfer the block
  523. In.Limit(Len);
  524. while (Go(true, File) == true)
  525. if (In.IsLimit() == true)
  526. break;
  527. // Error
  528. if (In.IsLimit() == false)
  529. return false;
  530. // The server sends an extra new line before the next block specifier..
  531. In.Limit(-1);
  532. Last = true;
  533. do
  534. {
  535. if (In.WriteTillEl(Data,true) == true)
  536. break;
  537. }
  538. while ((Last = Go(false, File)) == true);
  539. if (Last == false)
  540. return false;
  541. }
  542. }
  543. else
  544. {
  545. /* Closes encoding is used when the server did not specify a size, the
  546. loss of the connection means we are done */
  547. if (JunkSize != 0)
  548. In.Limit(JunkSize);
  549. else if (DownloadSize != 0)
  550. In.Limit(DownloadSize);
  551. else if (Persistent == false)
  552. In.Limit(-1);
  553. // Just transfer the whole block.
  554. do
  555. {
  556. if (In.IsLimit() == false)
  557. continue;
  558. In.Limit(-1);
  559. return !_error->PendingError();
  560. }
  561. while (Go(true, File) == true);
  562. }
  563. return Owner->Flush() && !_error->PendingError();
  564. }
  565. /*}}}*/
  566. bool HttpServerState::RunDataToDevNull() /*{{{*/
  567. {
  568. FileFd DevNull("/dev/null", FileFd::WriteOnly);
  569. return RunData(&DevNull);
  570. }
  571. /*}}}*/
  572. bool HttpServerState::ReadHeaderLines(std::string &Data) /*{{{*/
  573. {
  574. return In.WriteTillEl(Data);
  575. }
  576. /*}}}*/
  577. bool HttpServerState::LoadNextResponse(bool const ToFile, FileFd * const File)/*{{{*/
  578. {
  579. return Go(ToFile, File);
  580. }
  581. /*}}}*/
  582. bool HttpServerState::WriteResponse(const std::string &Data) /*{{{*/
  583. {
  584. return Out.Read(Data);
  585. }
  586. /*}}}*/
  587. APT_PURE bool HttpServerState::IsOpen() /*{{{*/
  588. {
  589. return (ServerFd != -1);
  590. }
  591. /*}}}*/
  592. bool HttpServerState::InitHashes(HashStringList const &ExpectedHashes) /*{{{*/
  593. {
  594. delete In.Hash;
  595. In.Hash = new Hashes(ExpectedHashes);
  596. return true;
  597. }
  598. /*}}}*/
  599. void HttpServerState::Reset(bool const Everything) /*{{{*/
  600. {
  601. ServerState::Reset(Everything);
  602. if (Everything)
  603. ServerFd = -1;
  604. }
  605. /*}}}*/
  606. APT_PURE Hashes * HttpServerState::GetHashes() /*{{{*/
  607. {
  608. return In.Hash;
  609. }
  610. /*}}}*/
  611. // HttpServerState::Die - The server has closed the connection. /*{{{*/
  612. bool HttpServerState::Die(FileFd * const File)
  613. {
  614. unsigned int LErrno = errno;
  615. // Dump the buffer to the file
  616. if (State == ServerState::Data)
  617. {
  618. if (File == nullptr)
  619. return true;
  620. // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
  621. // can't be set
  622. if (File->Name() != "/dev/null")
  623. SetNonBlock(File->Fd(),false);
  624. while (In.WriteSpace() == true)
  625. {
  626. if (In.Write(File->Fd()) == false)
  627. return _error->Errno("write",_("Error writing to the file"));
  628. // Done
  629. if (In.IsLimit() == true)
  630. return true;
  631. }
  632. }
  633. // See if this is because the server finished the data stream
  634. if (In.IsLimit() == false && State != HttpServerState::Header &&
  635. Persistent == true)
  636. {
  637. Close();
  638. if (LErrno == 0)
  639. return _error->Error(_("Error reading from server. Remote end closed connection"));
  640. errno = LErrno;
  641. return _error->Errno("read",_("Error reading from server"));
  642. }
  643. else
  644. {
  645. In.Limit(-1);
  646. // Nothing left in the buffer
  647. if (In.WriteSpace() == false)
  648. return false;
  649. // We may have got multiple responses back in one packet..
  650. Close();
  651. return true;
  652. }
  653. return false;
  654. }
  655. /*}}}*/
  656. // HttpServerState::Flush - Dump the buffer into the file /*{{{*/
  657. // ---------------------------------------------------------------------
  658. /* This takes the current input buffer from the Server FD and writes it
  659. into the file */
  660. bool HttpServerState::Flush(FileFd * const File)
  661. {
  662. if (File != NULL)
  663. {
  664. // on GNU/kFreeBSD, apt dies on /dev/null because non-blocking
  665. // can't be set
  666. if (File->Name() != "/dev/null")
  667. SetNonBlock(File->Fd(),false);
  668. if (In.WriteSpace() == false)
  669. return true;
  670. while (In.WriteSpace() == true)
  671. {
  672. if (In.Write(File->Fd()) == false)
  673. return _error->Errno("write",_("Error writing to file"));
  674. if (In.IsLimit() == true)
  675. return true;
  676. }
  677. if (In.IsLimit() == true || Persistent == false)
  678. return true;
  679. }
  680. return false;
  681. }
  682. /*}}}*/
  683. // HttpServerState::Go - Run a single loop /*{{{*/
  684. // ---------------------------------------------------------------------
  685. /* This runs the select loop over the server FDs, Output file FDs and
  686. stdin. */
  687. bool HttpServerState::Go(bool ToFile, FileFd * const File)
  688. {
  689. // Server has closed the connection
  690. if (ServerFd == -1 && (In.WriteSpace() == false ||
  691. ToFile == false))
  692. return false;
  693. fd_set rfds,wfds;
  694. FD_ZERO(&rfds);
  695. FD_ZERO(&wfds);
  696. /* Add the server. We only send more requests if the connection will
  697. be persisting */
  698. if (Out.WriteSpace() == true && ServerFd != -1
  699. && Persistent == true)
  700. FD_SET(ServerFd,&wfds);
  701. if (In.ReadSpace() == true && ServerFd != -1)
  702. FD_SET(ServerFd,&rfds);
  703. // Add the file
  704. int FileFD = -1;
  705. if (File != NULL)
  706. FileFD = File->Fd();
  707. if (In.WriteSpace() == true && ToFile == true && FileFD != -1)
  708. FD_SET(FileFD,&wfds);
  709. // Add stdin
  710. if (Owner->ConfigFindB("DependOnSTDIN", true) == true)
  711. FD_SET(STDIN_FILENO,&rfds);
  712. // Figure out the max fd
  713. int MaxFd = FileFD;
  714. if (MaxFd < ServerFd)
  715. MaxFd = ServerFd;
  716. // Select
  717. struct timeval tv;
  718. tv.tv_sec = TimeOut;
  719. tv.tv_usec = 0;
  720. int Res = 0;
  721. if ((Res = select(MaxFd+1,&rfds,&wfds,0,&tv)) < 0)
  722. {
  723. if (errno == EINTR)
  724. return true;
  725. return _error->Errno("select",_("Select failed"));
  726. }
  727. if (Res == 0)
  728. {
  729. _error->Error(_("Connection timed out"));
  730. return Die(File);
  731. }
  732. // Handle server IO
  733. if (ServerFd != -1 && FD_ISSET(ServerFd,&rfds))
  734. {
  735. errno = 0;
  736. if (In.Read(ServerFd) == false)
  737. return Die(File);
  738. }
  739. if (ServerFd != -1 && FD_ISSET(ServerFd,&wfds))
  740. {
  741. errno = 0;
  742. if (Out.Write(ServerFd) == false)
  743. return Die(File);
  744. }
  745. // Send data to the file
  746. if (FileFD != -1 && FD_ISSET(FileFD,&wfds))
  747. {
  748. if (In.Write(FileFD) == false)
  749. return _error->Errno("write",_("Error writing to output file"));
  750. }
  751. if (MaximumSize > 0 && File && File->Tell() > MaximumSize)
  752. {
  753. Owner->SetFailReason("MaximumSizeExceeded");
  754. return _error->Error("Writing more data than expected (%llu > %llu)",
  755. File->Tell(), MaximumSize);
  756. }
  757. // Handle commands from APT
  758. if (FD_ISSET(STDIN_FILENO,&rfds))
  759. {
  760. if (Owner->Run(true) != -1)
  761. exit(100);
  762. }
  763. return true;
  764. }
  765. /*}}}*/
  766. // HttpMethod::SendReq - Send the HTTP request /*{{{*/
  767. // ---------------------------------------------------------------------
  768. /* This places the http request in the outbound buffer */
  769. void HttpMethod::SendReq(FetchItem *Itm)
  770. {
  771. URI Uri = Itm->Uri;
  772. {
  773. auto const plus = Binary.find('+');
  774. if (plus != std::string::npos)
  775. Uri.Access = Binary.substr(plus + 1);
  776. }
  777. // The HTTP server expects a hostname with a trailing :port
  778. std::stringstream Req;
  779. string ProperHost;
  780. if (Uri.Host.find(':') != string::npos)
  781. ProperHost = '[' + Uri.Host + ']';
  782. else
  783. ProperHost = Uri.Host;
  784. /* RFC 2616 ยง5.1.2 requires absolute URIs for requests to proxies,
  785. but while its a must for all servers to accept absolute URIs,
  786. it is assumed clients will sent an absolute path for non-proxies */
  787. std::string requesturi;
  788. if (Server->Proxy.Access != "http" || Server->Proxy.empty() == true || Server->Proxy.Host.empty())
  789. requesturi = Uri.Path;
  790. else
  791. requesturi = Uri;
  792. // The "+" is encoded as a workaround for a amazon S3 bug
  793. // see LP bugs #1003633 and #1086997.
  794. requesturi = QuoteString(requesturi, "+~ ");
  795. /* Build the request. No keep-alive is included as it is the default
  796. in 1.1, can cause problems with proxies, and we are an HTTP/1.1
  797. client anyway.
  798. C.f. https://tools.ietf.org/wg/httpbis/trac/ticket/158 */
  799. Req << "GET " << requesturi << " HTTP/1.1\r\n";
  800. if (Uri.Port != 0)
  801. Req << "Host: " << ProperHost << ":" << std::to_string(Uri.Port) << "\r\n";
  802. else
  803. Req << "Host: " << ProperHost << "\r\n";
  804. // generate a cache control header (if needed)
  805. if (ConfigFindB("No-Cache",false) == true)
  806. Req << "Cache-Control: no-cache\r\n"
  807. << "Pragma: no-cache\r\n";
  808. else if (Itm->IndexFile == true)
  809. Req << "Cache-Control: max-age=" << std::to_string(ConfigFindI("Max-Age", 0)) << "\r\n";
  810. else if (ConfigFindB("No-Store", false) == true)
  811. Req << "Cache-Control: no-store\r\n";
  812. // If we ask for uncompressed files servers might respond with content-
  813. // negotiation which lets us end up with compressed files we do not support,
  814. // see 657029, 657560 and co, so if we have no extension on the request
  815. // ask for text only. As a sidenote: If there is nothing to negotate servers
  816. // seem to be nice and ignore it.
  817. if (ConfigFindB("SendAccept", true) == true)
  818. {
  819. size_t const filepos = Itm->Uri.find_last_of('/');
  820. string const file = Itm->Uri.substr(filepos + 1);
  821. if (flExtension(file) == file)
  822. Req << "Accept: text/*\r\n";
  823. }
  824. // Check for a partial file and send if-queries accordingly
  825. struct stat SBuf;
  826. if (Server->RangesAllowed && stat(Itm->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
  827. Req << "Range: bytes=" << std::to_string(SBuf.st_size) << "-\r\n"
  828. << "If-Range: " << TimeRFC1123(SBuf.st_mtime, false) << "\r\n";
  829. else if (Itm->LastModified != 0)
  830. Req << "If-Modified-Since: " << TimeRFC1123(Itm->LastModified, false).c_str() << "\r\n";
  831. if (Server->Proxy.Access == "http" &&
  832. (Server->Proxy.User.empty() == false || Server->Proxy.Password.empty() == false))
  833. Req << "Proxy-Authorization: Basic "
  834. << Base64Encode(Server->Proxy.User + ":" + Server->Proxy.Password) << "\r\n";
  835. maybe_add_auth (Uri, _config->FindFile("Dir::Etc::netrc"));
  836. if (Uri.User.empty() == false || Uri.Password.empty() == false)
  837. Req << "Authorization: Basic "
  838. << Base64Encode(Uri.User + ":" + Uri.Password) << "\r\n";
  839. Req << "User-Agent: " << ConfigFind("User-Agent",
  840. "Debian APT-HTTP/1.3 (" PACKAGE_VERSION ")") << "\r\n";
  841. Req << "\r\n";
  842. if (Debug == true)
  843. cerr << Req.str() << endl;
  844. Server->WriteResponse(Req.str());
  845. }
  846. /*}}}*/
  847. std::unique_ptr<ServerState> HttpMethod::CreateServerState(URI const &uri)/*{{{*/
  848. {
  849. return std::unique_ptr<ServerState>(new HttpServerState(uri, this));
  850. }
  851. /*}}}*/
  852. void HttpMethod::RotateDNS() /*{{{*/
  853. {
  854. ::RotateDNS();
  855. }
  856. /*}}}*/
  857. ServerMethod::DealWithHeadersResult HttpMethod::DealWithHeaders(FetchResult &Res)/*{{{*/
  858. {
  859. auto ret = ServerMethod::DealWithHeaders(Res);
  860. if (ret != ServerMethod::FILE_IS_OPEN)
  861. return ret;
  862. // Open the file
  863. delete File;
  864. File = new FileFd(Queue->DestFile,FileFd::WriteAny);
  865. if (_error->PendingError() == true)
  866. return ERROR_NOT_FROM_SERVER;
  867. FailFile = Queue->DestFile;
  868. FailFile.c_str(); // Make sure we don't do a malloc in the signal handler
  869. FailFd = File->Fd();
  870. FailTime = Server->Date;
  871. if (Server->InitHashes(Queue->ExpectedHashes) == false || Server->AddPartialFileToHashes(*File) == false)
  872. {
  873. _error->Errno("read",_("Problem hashing file"));
  874. return ERROR_NOT_FROM_SERVER;
  875. }
  876. if (Server->StartPos > 0)
  877. Res.ResumePoint = Server->StartPos;
  878. SetNonBlock(File->Fd(),true);
  879. return FILE_IS_OPEN;
  880. }
  881. /*}}}*/
  882. HttpMethod::HttpMethod(std::string &&pProg) : ServerMethod(pProg.c_str(), "1.2", Pipeline | SendConfig)/*{{{*/
  883. {
  884. auto addName = std::inserter(methodNames, methodNames.begin());
  885. if (Binary != "http")
  886. addName = "http";
  887. auto const plus = Binary.find('+');
  888. if (plus != std::string::npos)
  889. addName = Binary.substr(0, plus);
  890. File = 0;
  891. Server = 0;
  892. }
  893. /*}}}*/