You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

cachedb.cc 16 KiB

15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
15 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582
  1. // -*- mode: cpp; mode: fold -*-
  2. // Description /*{{{*/
  3. /* ######################################################################
  4. CacheDB
  5. Simple uniform interface to a cache database.
  6. ##################################################################### */
  7. /*}}}*/
  8. // Include Files /*{{{*/
  9. #include <config.h>
  10. #include <apt-pkg/configuration.h>
  11. #include <apt-pkg/debfile.h>
  12. #include <apt-pkg/error.h>
  13. #include <apt-pkg/fileutl.h>
  14. #include <apt-pkg/gpgv.h>
  15. #include <apt-pkg/hashes.h>
  16. #include <apt-pkg/md5.h>
  17. #include <apt-pkg/sha1.h>
  18. #include <apt-pkg/sha2.h>
  19. #include <apt-pkg/strutl.h>
  20. #include <ctype.h>
  21. #include <netinet/in.h> // htonl, etc
  22. #include <stddef.h>
  23. #include <strings.h>
  24. #include <sys/stat.h>
  25. #include "cachedb.h"
  26. #include <apti18n.h>
  27. /*}}}*/
  28. CacheDB::CacheDB(std::string const &DB)
  29. : Dbp(0), Fd(NULL), DebFile(0)
  30. {
  31. TmpKey[0]='\0';
  32. ReadyDB(DB);
  33. }
  34. CacheDB::~CacheDB()
  35. {
  36. ReadyDB();
  37. delete DebFile;
  38. CloseFile();
  39. }
  40. // CacheDB::ReadyDB - Ready the DB2 /*{{{*/
  41. // ---------------------------------------------------------------------
  42. /* This opens the DB2 file for caching package information */
  43. bool CacheDB::ReadyDB(std::string const &DB)
  44. {
  45. int err;
  46. ReadOnly = _config->FindB("APT::FTPArchive::ReadOnlyDB",false);
  47. // Close the old DB
  48. if (Dbp != 0)
  49. Dbp->close(Dbp,0);
  50. /* Check if the DB was disabled while running and deal with a
  51. corrupted DB */
  52. if (DBFailed() == true)
  53. {
  54. _error->Warning(_("DB was corrupted, file renamed to %s.old"),DBFile.c_str());
  55. rename(DBFile.c_str(),(DBFile+".old").c_str());
  56. }
  57. DBLoaded = false;
  58. Dbp = 0;
  59. DBFile = std::string();
  60. if (DB.empty())
  61. return true;
  62. db_create(&Dbp, NULL, 0);
  63. if ((err = Dbp->open(Dbp, NULL, DB.c_str(), NULL, DB_BTREE,
  64. (ReadOnly?DB_RDONLY:DB_CREATE),
  65. 0644)) != 0)
  66. {
  67. if (err == DB_OLD_VERSION)
  68. {
  69. _error->Warning(_("DB is old, attempting to upgrade %s"),DBFile.c_str());
  70. err = Dbp->upgrade(Dbp, DB.c_str(), 0);
  71. if (!err)
  72. err = Dbp->open(Dbp, NULL, DB.c_str(), NULL, DB_HASH,
  73. (ReadOnly?DB_RDONLY:DB_CREATE), 0644);
  74. }
  75. // the database format has changed from DB_HASH to DB_BTREE in
  76. // apt 0.6.44
  77. if (err == EINVAL)
  78. {
  79. _error->Error(_("DB format is invalid. If you upgraded from an older version of apt, please remove and re-create the database."));
  80. }
  81. if (err)
  82. {
  83. Dbp = 0;
  84. return _error->Error(_("Unable to open DB file %s: %s"),DB.c_str(), db_strerror(err));
  85. }
  86. }
  87. DBFile = DB;
  88. DBLoaded = true;
  89. return true;
  90. }
  91. /*}}}*/
  92. // CacheDB::OpenFile - Open the file /*{{{*/
  93. // ---------------------------------------------------------------------
  94. /* */
  95. bool CacheDB::OpenFile()
  96. {
  97. // always close existing file first
  98. CloseFile();
  99. // open a new file
  100. Fd = new FileFd(FileName,FileFd::ReadOnly);
  101. if (_error->PendingError() == true)
  102. {
  103. CloseFile();
  104. return false;
  105. }
  106. return true;
  107. }
  108. /*}}}*/
  109. // CacheDB::CloseFile - Close the file /*{{{*/
  110. void CacheDB::CloseFile()
  111. {
  112. if(Fd != NULL)
  113. {
  114. delete Fd;
  115. Fd = NULL;
  116. }
  117. }
  118. /*}}}*/
  119. // CacheDB::OpenDebFile - Open a debfile /*{{{*/
  120. bool CacheDB::OpenDebFile()
  121. {
  122. // always close existing file first
  123. CloseDebFile();
  124. // first open the fd, then pass it to the debDebFile
  125. if(OpenFile() == false)
  126. return false;
  127. DebFile = new debDebFile(*Fd);
  128. if (_error->PendingError() == true)
  129. return false;
  130. return true;
  131. }
  132. /*}}}*/
  133. // CacheDB::CloseDebFile - Close a debfile again /*{{{*/
  134. void CacheDB::CloseDebFile()
  135. {
  136. CloseFile();
  137. if(DebFile != NULL)
  138. {
  139. delete DebFile;
  140. DebFile = NULL;
  141. }
  142. }
  143. /*}}}*/
  144. // CacheDB::GetFileStat - Get stats from the file /*{{{*/
  145. // ---------------------------------------------------------------------
  146. /* This gets the size from the database if it's there. If we need
  147. * to look at the file, also get the mtime from the file. */
  148. bool CacheDB::GetFileStat(bool const &doStat)
  149. {
  150. if ((CurStat.Flags & FlSize) == FlSize && doStat == false)
  151. return true;
  152. /* Get it from the file. */
  153. if (OpenFile() == false)
  154. return false;
  155. // Stat the file
  156. struct stat St;
  157. if (fstat(Fd->Fd(),&St) != 0)
  158. {
  159. CloseFile();
  160. return _error->Errno("fstat",
  161. _("Failed to stat %s"),FileName.c_str());
  162. }
  163. CurStat.FileSize = St.st_size;
  164. CurStat.mtime = htonl(St.st_mtime);
  165. CurStat.Flags |= FlSize;
  166. return true;
  167. }
  168. /*}}}*/
  169. // CacheDB::GetCurStatCompatOldFormat /*{{{*/
  170. // ---------------------------------------------------------------------
  171. /* Read the old (32bit FileSize) StateStore format from disk */
  172. bool CacheDB::GetCurStatCompatOldFormat()
  173. {
  174. InitQueryStats();
  175. Data.data = &CurStatOldFormat;
  176. Data.flags = DB_DBT_USERMEM;
  177. Data.ulen = sizeof(CurStatOldFormat);
  178. if (Get() == false)
  179. {
  180. CurStat.Flags = 0;
  181. } else {
  182. CurStat.Flags = CurStatOldFormat.Flags;
  183. CurStat.mtime = CurStatOldFormat.mtime;
  184. CurStat.FileSize = CurStatOldFormat.FileSize;
  185. memcpy(CurStat.MD5, CurStatOldFormat.MD5, sizeof(CurStat.MD5));
  186. memcpy(CurStat.SHA1, CurStatOldFormat.SHA1, sizeof(CurStat.SHA1));
  187. memcpy(CurStat.SHA256, CurStatOldFormat.SHA256, sizeof(CurStat.SHA256));
  188. }
  189. return true;
  190. }
  191. /*}}}*/
  192. // CacheDB::GetCurStatCompatOldFormat /*{{{*/
  193. // ---------------------------------------------------------------------
  194. /* Read the new (64bit FileSize) StateStore format from disk */
  195. bool CacheDB::GetCurStatCompatNewFormat()
  196. {
  197. InitQueryStats();
  198. Data.data = &CurStat;
  199. Data.flags = DB_DBT_USERMEM;
  200. Data.ulen = sizeof(CurStat);
  201. if (Get() == false)
  202. {
  203. CurStat.Flags = 0;
  204. }
  205. return true;
  206. }
  207. /*}}}*/
  208. // CacheDB::GetCurStat - Set the CurStat variable. /*{{{*/
  209. // ---------------------------------------------------------------------
  210. /* Sets the CurStat variable. Either to 0 if no database is used
  211. * or to the value in the database if one is used */
  212. bool CacheDB::GetCurStat()
  213. {
  214. memset(&CurStat,0,sizeof(CurStat));
  215. if (DBLoaded)
  216. {
  217. // do a first query to just get the size of the data on disk
  218. InitQueryStats();
  219. Data.data = &CurStat;
  220. Data.flags = DB_DBT_USERMEM;
  221. Data.ulen = 0;
  222. Get();
  223. if (Data.size == 0)
  224. {
  225. // nothing needs to be done, we just have not data for this deb
  226. }
  227. // check if the record is written in the old format (32bit filesize)
  228. else if(Data.size == sizeof(CurStatOldFormat))
  229. {
  230. GetCurStatCompatOldFormat();
  231. }
  232. else if(Data.size == sizeof(CurStat))
  233. {
  234. GetCurStatCompatNewFormat();
  235. } else {
  236. return _error->Error("Cache record size mismatch (%ul)", Data.size);
  237. }
  238. CurStat.Flags = ntohl(CurStat.Flags);
  239. CurStat.FileSize = ntohl(CurStat.FileSize);
  240. }
  241. return true;
  242. }
  243. /*}}}*/
  244. // CacheDB::GetFileInfo - Get all the info about the file /*{{{*/
  245. // ---------------------------------------------------------------------
  246. bool CacheDB::GetFileInfo(std::string const &FileName, bool const &DoControl, bool const &DoContents,
  247. bool const &GenContentsOnly, bool const DoSource, unsigned int const DoHashes,
  248. bool const &checkMtime)
  249. {
  250. this->FileName = FileName;
  251. if (GetCurStat() == false)
  252. return false;
  253. OldStat = CurStat;
  254. if (GetFileStat(checkMtime) == false)
  255. return false;
  256. /* if mtime changed, update CurStat from disk */
  257. if (checkMtime == true && OldStat.mtime != CurStat.mtime)
  258. CurStat.Flags = FlSize;
  259. Stats.Bytes += CurStat.FileSize;
  260. ++Stats.Packages;
  261. if ((DoControl && LoadControl() == false)
  262. || (DoContents && LoadContents(GenContentsOnly) == false)
  263. || (DoSource && LoadSource() == false)
  264. || (DoHashes != 0 && GetHashes(false, DoHashes) == false)
  265. )
  266. {
  267. return false;
  268. }
  269. return true;
  270. }
  271. /*}}}*/
  272. bool CacheDB::LoadSource() /*{{{*/
  273. {
  274. // Try to read the control information out of the DB.
  275. if ((CurStat.Flags & FlSource) == FlSource)
  276. {
  277. // Lookup the control information
  278. InitQuerySource();
  279. if (Get() == true && Dsc.TakeDsc(Data.data, Data.size) == true)
  280. {
  281. return true;
  282. }
  283. CurStat.Flags &= ~FlSource;
  284. }
  285. if (OpenFile() == false)
  286. return false;
  287. Stats.Misses++;
  288. if (Dsc.Read(FileName) == false)
  289. return false;
  290. if (Dsc.Length == 0)
  291. return _error->Error(_("Failed to read .dsc"));
  292. // Write back the control information
  293. InitQuerySource();
  294. if (Put(Dsc.Data.c_str(), Dsc.Length) == true)
  295. CurStat.Flags |= FlSource;
  296. return true;
  297. }
  298. /*}}}*/
  299. // CacheDB::LoadControl - Load Control information /*{{{*/
  300. // ---------------------------------------------------------------------
  301. /* */
  302. bool CacheDB::LoadControl()
  303. {
  304. // Try to read the control information out of the DB.
  305. if ((CurStat.Flags & FlControl) == FlControl)
  306. {
  307. // Lookup the control information
  308. InitQueryControl();
  309. if (Get() == true && Control.TakeControl(Data.data,Data.size) == true)
  310. return true;
  311. CurStat.Flags &= ~FlControl;
  312. }
  313. if(OpenDebFile() == false)
  314. return false;
  315. Stats.Misses++;
  316. if (Control.Read(*DebFile) == false)
  317. return false;
  318. if (Control.Control == 0)
  319. return _error->Error(_("Archive has no control record"));
  320. // Write back the control information
  321. InitQueryControl();
  322. if (Put(Control.Control,Control.Length) == true)
  323. CurStat.Flags |= FlControl;
  324. return true;
  325. }
  326. /*}}}*/
  327. // CacheDB::LoadContents - Load the File Listing /*{{{*/
  328. // ---------------------------------------------------------------------
  329. /* */
  330. bool CacheDB::LoadContents(bool const &GenOnly)
  331. {
  332. // Try to read the control information out of the DB.
  333. if ((CurStat.Flags & FlContents) == FlContents)
  334. {
  335. if (GenOnly == true)
  336. return true;
  337. // Lookup the contents information
  338. InitQueryContent();
  339. if (Get() == true)
  340. {
  341. if (Contents.TakeContents(Data.data,Data.size) == true)
  342. return true;
  343. }
  344. CurStat.Flags &= ~FlContents;
  345. }
  346. if(OpenDebFile() == false)
  347. return false;
  348. Stats.Misses++;
  349. if (Contents.Read(*DebFile) == false)
  350. return false;
  351. // Write back the control information
  352. InitQueryContent();
  353. if (Put(Contents.Data,Contents.CurSize) == true)
  354. CurStat.Flags |= FlContents;
  355. return true;
  356. }
  357. /*}}}*/
  358. // CacheDB::GetHashes - Get the hashes /*{{{*/
  359. static std::string bytes2hex(uint8_t *bytes, size_t length) {
  360. char buf[3];
  361. std::string space;
  362. space.reserve(length*2 + 1);
  363. for (size_t i = 0; i < length; i++) {
  364. snprintf(buf, sizeof(buf), "%02x", bytes[i]);
  365. space.append(buf);
  366. }
  367. return space;
  368. }
  369. static inline unsigned char xdig2num(char const &dig) {
  370. if (isdigit(dig)) return dig - '0';
  371. if ('a' <= dig && dig <= 'f') return dig - 'a' + 10;
  372. if ('A' <= dig && dig <= 'F') return dig - 'A' + 10;
  373. return 0;
  374. }
  375. static void hex2bytes(uint8_t *bytes, const char *hex, int length) {
  376. while (length-- > 0) {
  377. *bytes = 0;
  378. if (isxdigit(hex[0]) && isxdigit(hex[1])) {
  379. *bytes = xdig2num(hex[0]) * 16 + xdig2num(hex[1]);
  380. hex += 2;
  381. }
  382. bytes++;
  383. }
  384. }
  385. bool CacheDB::GetHashes(bool const GenOnly, unsigned int const DoHashes)
  386. {
  387. unsigned int notCachedHashes = 0;
  388. if ((CurStat.Flags & FlMD5) != FlMD5)
  389. {
  390. notCachedHashes = notCachedHashes | Hashes::MD5SUM;
  391. }
  392. if ((CurStat.Flags & FlSHA1) != FlSHA1)
  393. {
  394. notCachedHashes = notCachedHashes | Hashes::SHA1SUM;
  395. }
  396. if ((CurStat.Flags & FlSHA256) != FlSHA256)
  397. {
  398. notCachedHashes = notCachedHashes | Hashes::SHA256SUM;
  399. }
  400. if ((CurStat.Flags & FlSHA512) != FlSHA512)
  401. {
  402. notCachedHashes = notCachedHashes | Hashes::SHA512SUM;
  403. }
  404. unsigned int FlHashes = DoHashes & notCachedHashes;
  405. HashesList.clear();
  406. if (FlHashes != 0)
  407. {
  408. if (OpenFile() == false)
  409. return false;
  410. Hashes hashes(FlHashes);
  411. if (Fd->Seek(0) == false || hashes.AddFD(*Fd, CurStat.FileSize) == false)
  412. return false;
  413. HashStringList hl = hashes.GetHashStringList();
  414. for (HashStringList::const_iterator hs = hl.begin(); hs != hl.end(); ++hs)
  415. {
  416. HashesList.push_back(*hs);
  417. if (strcasecmp(hs->HashType().c_str(), "SHA512") == 0)
  418. {
  419. Stats.SHA512Bytes += CurStat.FileSize;
  420. hex2bytes(CurStat.SHA512, hs->HashValue().data(), sizeof(CurStat.SHA512));
  421. CurStat.Flags |= FlSHA512;
  422. }
  423. else if (strcasecmp(hs->HashType().c_str(), "SHA256") == 0)
  424. {
  425. Stats.SHA256Bytes += CurStat.FileSize;
  426. hex2bytes(CurStat.SHA256, hs->HashValue().data(), sizeof(CurStat.SHA256));
  427. CurStat.Flags |= FlSHA256;
  428. }
  429. else if (strcasecmp(hs->HashType().c_str(), "SHA1") == 0)
  430. {
  431. Stats.SHA1Bytes += CurStat.FileSize;
  432. hex2bytes(CurStat.SHA1, hs->HashValue().data(), sizeof(CurStat.SHA1));
  433. CurStat.Flags |= FlSHA1;
  434. }
  435. else if (strcasecmp(hs->HashType().c_str(), "MD5Sum") == 0)
  436. {
  437. Stats.MD5Bytes += CurStat.FileSize;
  438. hex2bytes(CurStat.MD5, hs->HashValue().data(), sizeof(CurStat.MD5));
  439. CurStat.Flags |= FlMD5;
  440. }
  441. else if (strcasecmp(hs->HashType().c_str(), "Checksum-FileSize") == 0)
  442. {
  443. // we store it in a different field already
  444. }
  445. else
  446. return _error->Error("Got unknown unrequested hashtype %s", hs->HashType().c_str());
  447. }
  448. }
  449. if (GenOnly == true)
  450. return true;
  451. bool ret = true;
  452. #define PUSH_BACK_HASH(FLAG, TYPE, VALUE) \
  453. if ((CurStat.Flags & FLAG) == FLAG) \
  454. ret &= HashesList.push_back(HashString(TYPE, bytes2hex(VALUE, sizeof(VALUE))));
  455. PUSH_BACK_HASH(FlMD5, "MD5Sum", CurStat.MD5);
  456. PUSH_BACK_HASH(FlSHA1, "SHA1", CurStat.SHA1);
  457. PUSH_BACK_HASH(FlSHA256, "SHA256", CurStat.SHA256);
  458. PUSH_BACK_HASH(FlSHA512, "SHA512", CurStat.SHA512);
  459. return ret;
  460. }
  461. /*}}}*/
  462. // CacheDB::Finish - Write back the cache structure /*{{{*/
  463. // ---------------------------------------------------------------------
  464. /* */
  465. bool CacheDB::Finish()
  466. {
  467. // Optimize away some writes.
  468. if (CurStat.Flags == OldStat.Flags &&
  469. CurStat.mtime == OldStat.mtime)
  470. return true;
  471. // Write the stat information
  472. CurStat.Flags = htonl(CurStat.Flags);
  473. CurStat.FileSize = htonl(CurStat.FileSize);
  474. InitQueryStats();
  475. Put(&CurStat,sizeof(CurStat));
  476. CurStat.Flags = ntohl(CurStat.Flags);
  477. CurStat.FileSize = ntohl(CurStat.FileSize);
  478. return true;
  479. }
  480. /*}}}*/
  481. // CacheDB::Clean - Clean the Database /*{{{*/
  482. // ---------------------------------------------------------------------
  483. /* Tidy the database by removing files that no longer exist at all. */
  484. bool CacheDB::Clean()
  485. {
  486. if (DBLoaded == false)
  487. return true;
  488. /* I'm not sure what VERSION_MINOR should be here.. 2.4.14 certainly
  489. needs the lower one and 2.7.7 needs the upper.. */
  490. DBC *Cursor;
  491. if ((errno = Dbp->cursor(Dbp, NULL, &Cursor, 0)) != 0)
  492. return _error->Error(_("Unable to get a cursor"));
  493. DBT Key;
  494. DBT Data;
  495. memset(&Key,0,sizeof(Key));
  496. memset(&Data,0,sizeof(Data));
  497. while ((errno = Cursor->c_get(Cursor,&Key,&Data,DB_NEXT)) == 0)
  498. {
  499. const char *Colon = (char*)memrchr(Key.data, ':', Key.size);
  500. if (Colon)
  501. {
  502. if (stringcmp(Colon + 1, (char *)Key.data+Key.size,"st") == 0 ||
  503. stringcmp(Colon + 1, (char *)Key.data+Key.size,"cl") == 0 ||
  504. stringcmp(Colon + 1, (char *)Key.data+Key.size,"cs") == 0 ||
  505. stringcmp(Colon + 1, (char *)Key.data+Key.size,"cn") == 0)
  506. {
  507. std::string FileName = std::string((const char *)Key.data,Colon);
  508. if (FileExists(FileName) == true) {
  509. continue;
  510. }
  511. }
  512. }
  513. Cursor->c_del(Cursor,0);
  514. }
  515. int res = Dbp->compact(Dbp, NULL, NULL, NULL, NULL, DB_FREE_SPACE, NULL);
  516. if (res < 0)
  517. _error->Warning("compact failed with result %i", res);
  518. if(_config->FindB("Debug::APT::FTPArchive::Clean", false) == true)
  519. Dbp->stat_print(Dbp, 0);
  520. return true;
  521. }
  522. /*}}}*/