You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

665 lines
17 KiB

  1. // -*- mode: cpp; mode: fold -*-
  2. // Description /*{{{*/
  3. /* ######################################################################
  4. HTTP and HTTPS share a lot of common code and these classes are
  5. exactly the dumping ground for this common code
  6. ##################################################################### */
  7. /*}}}*/
  8. // Include Files /*{{{*/
  9. #include <config.h>
  10. #include <apt-pkg/fileutl.h>
  11. #include <apt-pkg/acquire-method.h>
  12. #include <apt-pkg/configuration.h>
  13. #include <apt-pkg/error.h>
  14. #include <apt-pkg/hashes.h>
  15. #include <apt-pkg/netrc.h>
  16. #include <fcntl.h>
  17. #include <sys/stat.h>
  18. #include <sys/time.h>
  19. #include <unistd.h>
  20. #include <signal.h>
  21. #include <stdio.h>
  22. #include <errno.h>
  23. #include <string.h>
  24. #include <climits>
  25. #include <iostream>
  26. #include <map>
  27. // Internet stuff
  28. #include <netdb.h>
  29. #include "config.h"
  30. #include "connect.h"
  31. #include "rfc2553emu.h"
  32. #include "http.h"
  33. #include <apti18n.h>
  34. /*}}}*/
  35. using namespace std;
  36. string ServerMethod::FailFile;
  37. int ServerMethod::FailFd = -1;
  38. time_t ServerMethod::FailTime = 0;
  39. // ServerState::RunHeaders - Get the headers before the data /*{{{*/
  40. // ---------------------------------------------------------------------
  41. /* Returns 0 if things are OK, 1 if an IO error occurred and 2 if a header
  42. parse error occurred */
  43. ServerState::RunHeadersResult ServerState::RunHeaders(FileFd * const File)
  44. {
  45. State = Header;
  46. Owner->Status(_("Waiting for headers"));
  47. Major = 0;
  48. Minor = 0;
  49. Result = 0;
  50. Size = 0;
  51. StartPos = 0;
  52. Encoding = Closes;
  53. HaveContent = false;
  54. time(&Date);
  55. do
  56. {
  57. string Data;
  58. if (ReadHeaderLines(Data) == false)
  59. continue;
  60. if (Owner->Debug == true)
  61. clog << Data;
  62. for (string::const_iterator I = Data.begin(); I < Data.end(); ++I)
  63. {
  64. string::const_iterator J = I;
  65. for (; J != Data.end() && *J != '\n' && *J != '\r'; ++J);
  66. if (HeaderLine(string(I,J)) == false)
  67. return RUN_HEADERS_PARSE_ERROR;
  68. I = J;
  69. }
  70. // 100 Continue is a Nop...
  71. if (Result == 100)
  72. continue;
  73. // Tidy up the connection persistance state.
  74. if (Encoding == Closes && HaveContent == true)
  75. Persistent = false;
  76. return RUN_HEADERS_OK;
  77. }
  78. while (LoadNextResponse(false, File) == true);
  79. return RUN_HEADERS_IO_ERROR;
  80. }
  81. /*}}}*/
  82. // ServerState::HeaderLine - Process a header line /*{{{*/
  83. // ---------------------------------------------------------------------
  84. /* */
  85. bool ServerState::HeaderLine(string Line)
  86. {
  87. if (Line.empty() == true)
  88. return true;
  89. string::size_type Pos = Line.find(' ');
  90. if (Pos == string::npos || Pos+1 > Line.length())
  91. {
  92. // Blah, some servers use "connection:closes", evil.
  93. Pos = Line.find(':');
  94. if (Pos == string::npos || Pos + 2 > Line.length())
  95. return _error->Error(_("Bad header line"));
  96. Pos++;
  97. }
  98. // Parse off any trailing spaces between the : and the next word.
  99. string::size_type Pos2 = Pos;
  100. while (Pos2 < Line.length() && isspace(Line[Pos2]) != 0)
  101. Pos2++;
  102. string Tag = string(Line,0,Pos);
  103. string Val = string(Line,Pos2);
  104. if (stringcasecmp(Tag.c_str(),Tag.c_str()+4,"HTTP") == 0)
  105. {
  106. // Evil servers return no version
  107. if (Line[4] == '/')
  108. {
  109. int const elements = sscanf(Line.c_str(),"HTTP/%3u.%3u %3u%359[^\n]",&Major,&Minor,&Result,Code);
  110. if (elements == 3)
  111. {
  112. Code[0] = '\0';
  113. if (Owner->Debug == true)
  114. clog << "HTTP server doesn't give Reason-Phrase for " << Result << std::endl;
  115. }
  116. else if (elements != 4)
  117. return _error->Error(_("The HTTP server sent an invalid reply header"));
  118. }
  119. else
  120. {
  121. Major = 0;
  122. Minor = 9;
  123. if (sscanf(Line.c_str(),"HTTP %3u%359[^\n]",&Result,Code) != 2)
  124. return _error->Error(_("The HTTP server sent an invalid reply header"));
  125. }
  126. /* Check the HTTP response header to get the default persistance
  127. state. */
  128. if (Major < 1)
  129. Persistent = false;
  130. else
  131. {
  132. if (Major == 1 && Minor == 0)
  133. Persistent = false;
  134. else
  135. Persistent = true;
  136. }
  137. return true;
  138. }
  139. if (stringcasecmp(Tag,"Content-Length:") == 0)
  140. {
  141. if (Encoding == Closes)
  142. Encoding = Stream;
  143. HaveContent = true;
  144. // The length is already set from the Content-Range header
  145. if (StartPos != 0)
  146. return true;
  147. Size = strtoull(Val.c_str(), NULL, 10);
  148. if (Size >= std::numeric_limits<unsigned long long>::max())
  149. return _error->Errno("HeaderLine", _("The HTTP server sent an invalid Content-Length header"));
  150. else if (Size == 0)
  151. HaveContent = false;
  152. return true;
  153. }
  154. if (stringcasecmp(Tag,"Content-Type:") == 0)
  155. {
  156. HaveContent = true;
  157. return true;
  158. }
  159. if (stringcasecmp(Tag,"Content-Range:") == 0)
  160. {
  161. HaveContent = true;
  162. // ยง14.16 says 'byte-range-resp-spec' should be a '*' in case of 416
  163. if (Result == 416 && sscanf(Val.c_str(), "bytes */%llu",&Size) == 1)
  164. {
  165. StartPos = 1; // ignore Content-Length, it would override Size
  166. HaveContent = false;
  167. }
  168. else if (sscanf(Val.c_str(),"bytes %llu-%*u/%llu",&StartPos,&Size) != 2)
  169. return _error->Error(_("The HTTP server sent an invalid Content-Range header"));
  170. if ((unsigned long long)StartPos > Size)
  171. return _error->Error(_("This HTTP server has broken range support"));
  172. return true;
  173. }
  174. if (stringcasecmp(Tag,"Transfer-Encoding:") == 0)
  175. {
  176. HaveContent = true;
  177. if (stringcasecmp(Val,"chunked") == 0)
  178. Encoding = Chunked;
  179. return true;
  180. }
  181. if (stringcasecmp(Tag,"Connection:") == 0)
  182. {
  183. if (stringcasecmp(Val,"close") == 0)
  184. Persistent = false;
  185. if (stringcasecmp(Val,"keep-alive") == 0)
  186. Persistent = true;
  187. return true;
  188. }
  189. if (stringcasecmp(Tag,"Last-Modified:") == 0)
  190. {
  191. if (RFC1123StrToTime(Val.c_str(), Date) == false)
  192. return _error->Error(_("Unknown date format"));
  193. return true;
  194. }
  195. if (stringcasecmp(Tag,"Location:") == 0)
  196. {
  197. Location = Val;
  198. return true;
  199. }
  200. return true;
  201. }
  202. /*}}}*/
  203. // ServerState::ServerState - Constructor /*{{{*/
  204. ServerState::ServerState(URI Srv, ServerMethod *Owner) : ServerName(Srv), TimeOut(120), Owner(Owner)
  205. {
  206. Reset();
  207. }
  208. /*}}}*/
  209. bool ServerMethod::Configuration(string Message) /*{{{*/
  210. {
  211. return pkgAcqMethod::Configuration(Message);
  212. }
  213. /*}}}*/
  214. // ServerMethod::DealWithHeaders - Handle the retrieved header data /*{{{*/
  215. // ---------------------------------------------------------------------
  216. /* We look at the header data we got back from the server and decide what
  217. to do. Returns DealWithHeadersResult (see http.h for details).
  218. */
  219. ServerMethod::DealWithHeadersResult
  220. ServerMethod::DealWithHeaders(FetchResult &Res)
  221. {
  222. // Not Modified
  223. if (Server->Result == 304)
  224. {
  225. unlink(Queue->DestFile.c_str());
  226. Res.IMSHit = true;
  227. Res.LastModified = Queue->LastModified;
  228. return IMS_HIT;
  229. }
  230. /* Redirect
  231. *
  232. * Note that it is only OK for us to treat all redirection the same
  233. * because we *always* use GET, not other HTTP methods. There are
  234. * three redirection codes for which it is not appropriate that we
  235. * redirect. Pass on those codes so the error handling kicks in.
  236. */
  237. if (AllowRedirect
  238. && (Server->Result > 300 && Server->Result < 400)
  239. && (Server->Result != 300 // Multiple Choices
  240. && Server->Result != 304 // Not Modified
  241. && Server->Result != 306)) // (Not part of HTTP/1.1, reserved)
  242. {
  243. if (Server->Location.empty() == true);
  244. else if (Server->Location[0] == '/' && Queue->Uri.empty() == false)
  245. {
  246. URI Uri = Queue->Uri;
  247. if (Uri.Host.empty() == false)
  248. NextURI = URI::SiteOnly(Uri);
  249. else
  250. NextURI.clear();
  251. NextURI.append(DeQuoteString(Server->Location));
  252. return TRY_AGAIN_OR_REDIRECT;
  253. }
  254. else
  255. {
  256. NextURI = DeQuoteString(Server->Location);
  257. URI tmpURI = NextURI;
  258. // Do not allow a redirection to switch protocol
  259. if (tmpURI.Access == "http")
  260. return TRY_AGAIN_OR_REDIRECT;
  261. }
  262. /* else pass through for error message */
  263. }
  264. // retry after an invalid range response without partial data
  265. else if (Server->Result == 416)
  266. {
  267. struct stat SBuf;
  268. if (stat(Queue->DestFile.c_str(),&SBuf) >= 0 && SBuf.st_size > 0)
  269. {
  270. if ((unsigned long long)SBuf.st_size == Server->Size)
  271. {
  272. // the file is completely downloaded, but was not moved
  273. Server->StartPos = Server->Size;
  274. Server->Result = 200;
  275. Server->HaveContent = false;
  276. }
  277. else if (unlink(Queue->DestFile.c_str()) == 0)
  278. {
  279. NextURI = Queue->Uri;
  280. return TRY_AGAIN_OR_REDIRECT;
  281. }
  282. }
  283. }
  284. /* We have a reply we dont handle. This should indicate a perm server
  285. failure */
  286. if (Server->Result < 200 || Server->Result >= 300)
  287. {
  288. char err[255];
  289. snprintf(err,sizeof(err)-1,"HttpError%i",Server->Result);
  290. SetFailReason(err);
  291. _error->Error("%u %s",Server->Result,Server->Code);
  292. if (Server->HaveContent == true)
  293. return ERROR_WITH_CONTENT_PAGE;
  294. return ERROR_UNRECOVERABLE;
  295. }
  296. // This is some sort of 2xx 'data follows' reply
  297. Res.LastModified = Server->Date;
  298. Res.Size = Server->Size;
  299. // Open the file
  300. delete File;
  301. File = new FileFd(Queue->DestFile,FileFd::WriteAny);
  302. if (_error->PendingError() == true)
  303. return ERROR_NOT_FROM_SERVER;
  304. FailFile = Queue->DestFile;
  305. FailFile.c_str(); // Make sure we dont do a malloc in the signal handler
  306. FailFd = File->Fd();
  307. FailTime = Server->Date;
  308. if (Server->InitHashes(*File) == false)
  309. {
  310. _error->Errno("read",_("Problem hashing file"));
  311. return ERROR_NOT_FROM_SERVER;
  312. }
  313. if (Server->StartPos > 0)
  314. Res.ResumePoint = Server->StartPos;
  315. SetNonBlock(File->Fd(),true);
  316. return FILE_IS_OPEN;
  317. }
  318. /*}}}*/
  319. // ServerMethod::SigTerm - Handle a fatal signal /*{{{*/
  320. // ---------------------------------------------------------------------
  321. /* This closes and timestamps the open file. This is neccessary to get
  322. resume behavoir on user abort */
  323. void ServerMethod::SigTerm(int)
  324. {
  325. if (FailFd == -1)
  326. _exit(100);
  327. struct timeval times[2];
  328. times[0].tv_sec = FailTime;
  329. times[1].tv_sec = FailTime;
  330. times[0].tv_usec = times[1].tv_usec = 0;
  331. utimes(FailFile.c_str(), times);
  332. close(FailFd);
  333. _exit(100);
  334. }
  335. /*}}}*/
  336. // ServerMethod::Fetch - Fetch an item /*{{{*/
  337. // ---------------------------------------------------------------------
  338. /* This adds an item to the pipeline. We keep the pipeline at a fixed
  339. depth. */
  340. bool ServerMethod::Fetch(FetchItem *)
  341. {
  342. if (Server == 0)
  343. return true;
  344. // Queue the requests
  345. int Depth = -1;
  346. for (FetchItem *I = Queue; I != 0 && Depth < (signed)PipelineDepth;
  347. I = I->Next, Depth++)
  348. {
  349. // If pipelining is disabled, we only queue 1 request
  350. if (Server->Pipeline == false && Depth >= 0)
  351. break;
  352. // Make sure we stick with the same server
  353. if (Server->Comp(I->Uri) == false)
  354. break;
  355. if (QueueBack == I)
  356. {
  357. QueueBack = I->Next;
  358. SendReq(I);
  359. continue;
  360. }
  361. }
  362. return true;
  363. };
  364. /*}}}*/
  365. // ServerMethod::Loop - Main loop /*{{{*/
  366. int ServerMethod::Loop()
  367. {
  368. typedef vector<string> StringVector;
  369. typedef vector<string>::iterator StringVectorIterator;
  370. map<string, StringVector> Redirected;
  371. signal(SIGTERM,SigTerm);
  372. signal(SIGINT,SigTerm);
  373. Server = 0;
  374. int FailCounter = 0;
  375. while (1)
  376. {
  377. // We have no commands, wait for some to arrive
  378. if (Queue == 0)
  379. {
  380. if (WaitFd(STDIN_FILENO) == false)
  381. return 0;
  382. }
  383. /* Run messages, we can accept 0 (no message) if we didn't
  384. do a WaitFd above.. Otherwise the FD is closed. */
  385. int Result = Run(true);
  386. if (Result != -1 && (Result != 0 || Queue == 0))
  387. {
  388. if(FailReason.empty() == false ||
  389. _config->FindB("Acquire::http::DependOnSTDIN", true) == true)
  390. return 100;
  391. else
  392. return 0;
  393. }
  394. if (Queue == 0)
  395. continue;
  396. // Connect to the server
  397. if (Server == 0 || Server->Comp(Queue->Uri) == false)
  398. {
  399. delete Server;
  400. Server = CreateServerState(Queue->Uri);
  401. }
  402. /* If the server has explicitly said this is the last connection
  403. then we pre-emptively shut down the pipeline and tear down
  404. the connection. This will speed up HTTP/1.0 servers a tad
  405. since we don't have to wait for the close sequence to
  406. complete */
  407. if (Server->Persistent == false)
  408. Server->Close();
  409. // Reset the pipeline
  410. if (Server->IsOpen() == false)
  411. QueueBack = Queue;
  412. // Connnect to the host
  413. if (Server->Open() == false)
  414. {
  415. Fail(true);
  416. delete Server;
  417. Server = 0;
  418. continue;
  419. }
  420. // Fill the pipeline.
  421. Fetch(0);
  422. // Fetch the next URL header data from the server.
  423. switch (Server->RunHeaders(File))
  424. {
  425. case ServerState::RUN_HEADERS_OK:
  426. break;
  427. // The header data is bad
  428. case ServerState::RUN_HEADERS_PARSE_ERROR:
  429. {
  430. _error->Error(_("Bad header data"));
  431. Fail(true);
  432. RotateDNS();
  433. continue;
  434. }
  435. // The server closed a connection during the header get..
  436. default:
  437. case ServerState::RUN_HEADERS_IO_ERROR:
  438. {
  439. FailCounter++;
  440. _error->Discard();
  441. Server->Close();
  442. Server->Pipeline = false;
  443. if (FailCounter >= 2)
  444. {
  445. Fail(_("Connection failed"),true);
  446. FailCounter = 0;
  447. }
  448. RotateDNS();
  449. continue;
  450. }
  451. };
  452. // Decide what to do.
  453. FetchResult Res;
  454. Res.Filename = Queue->DestFile;
  455. switch (DealWithHeaders(Res))
  456. {
  457. // Ok, the file is Open
  458. case FILE_IS_OPEN:
  459. {
  460. URIStart(Res);
  461. // Run the data
  462. bool Result = true;
  463. if (Server->HaveContent)
  464. Result = Server->RunData(File);
  465. /* If the server is sending back sizeless responses then fill in
  466. the size now */
  467. if (Res.Size == 0)
  468. Res.Size = File->Size();
  469. // Close the file, destroy the FD object and timestamp it
  470. FailFd = -1;
  471. delete File;
  472. File = 0;
  473. // Timestamp
  474. struct timeval times[2];
  475. times[0].tv_sec = times[1].tv_sec = Server->Date;
  476. times[0].tv_usec = times[1].tv_usec = 0;
  477. utimes(Queue->DestFile.c_str(), times);
  478. // Send status to APT
  479. if (Result == true)
  480. {
  481. Res.TakeHashes(*Server->GetHashes());
  482. URIDone(Res);
  483. }
  484. else
  485. {
  486. if (Server->IsOpen() == false)
  487. {
  488. FailCounter++;
  489. _error->Discard();
  490. Server->Close();
  491. if (FailCounter >= 2)
  492. {
  493. Fail(_("Connection failed"),true);
  494. FailCounter = 0;
  495. }
  496. QueueBack = Queue;
  497. }
  498. else
  499. Fail(true);
  500. }
  501. break;
  502. }
  503. // IMS hit
  504. case IMS_HIT:
  505. {
  506. URIDone(Res);
  507. break;
  508. }
  509. // Hard server error, not found or something
  510. case ERROR_UNRECOVERABLE:
  511. {
  512. Fail();
  513. break;
  514. }
  515. // Hard internal error, kill the connection and fail
  516. case ERROR_NOT_FROM_SERVER:
  517. {
  518. delete File;
  519. File = 0;
  520. Fail();
  521. RotateDNS();
  522. Server->Close();
  523. break;
  524. }
  525. // We need to flush the data, the header is like a 404 w/ error text
  526. case ERROR_WITH_CONTENT_PAGE:
  527. {
  528. Fail();
  529. // Send to content to dev/null
  530. File = new FileFd("/dev/null",FileFd::WriteExists);
  531. Server->RunData(File);
  532. delete File;
  533. File = 0;
  534. break;
  535. }
  536. // Try again with a new URL
  537. case TRY_AGAIN_OR_REDIRECT:
  538. {
  539. // Clear rest of response if there is content
  540. if (Server->HaveContent)
  541. {
  542. File = new FileFd("/dev/null",FileFd::WriteExists);
  543. Server->RunData(File);
  544. delete File;
  545. File = 0;
  546. }
  547. /* Detect redirect loops. No more redirects are allowed
  548. after the same URI is seen twice in a queue item. */
  549. StringVector &R = Redirected[Queue->DestFile];
  550. bool StopRedirects = false;
  551. if (R.empty() == true)
  552. R.push_back(Queue->Uri);
  553. else if (R[0] == "STOP" || R.size() > 10)
  554. StopRedirects = true;
  555. else
  556. {
  557. for (StringVectorIterator I = R.begin(); I != R.end(); ++I)
  558. if (Queue->Uri == *I)
  559. {
  560. R[0] = "STOP";
  561. break;
  562. }
  563. R.push_back(Queue->Uri);
  564. }
  565. if (StopRedirects == false)
  566. Redirect(NextURI);
  567. else
  568. Fail();
  569. break;
  570. }
  571. default:
  572. Fail(_("Internal error"));
  573. break;
  574. }
  575. FailCounter = 0;
  576. }
  577. return 0;
  578. }
  579. /*}}}*/