You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

1059 lines
35 KiB

  1. // -*- mode: cpp; mode: fold -*-
  2. // Description /*{{{*/
  3. /* ######################################################################
  4. apt-ftparchive - Efficient work-alike for dpkg-scanpackages
  5. Let contents be disabled from the conf
  6. ##################################################################### */
  7. /*}}}*/
  8. // Include Files /*{{{*/
  9. #include <config.h>
  10. #include <apt-pkg/cmndline.h>
  11. #include <apt-pkg/configuration.h>
  12. #include <apt-pkg/error.h>
  13. #include <apt-pkg/fileutl.h>
  14. #include <apt-pkg/init.h>
  15. #include <apt-pkg/strutl.h>
  16. #include <apt-private/private-cmndline.h>
  17. #include <apt-private/private-main.h>
  18. #include <apt-private/private-output.h>
  19. #include <algorithm>
  20. #include <chrono>
  21. #include <climits>
  22. #include <functional>
  23. #include <iostream>
  24. #include <string>
  25. #include <vector>
  26. #include <locale.h>
  27. #include <stdio.h>
  28. #include <sys/stat.h>
  29. #include <sys/time.h>
  30. #include <time.h>
  31. #include <math.h>
  32. #include "apt-ftparchive.h"
  33. #include "cachedb.h"
  34. #include "multicompress.h"
  35. #include "override.h"
  36. #include "writer.h"
  37. #include <apti18n.h>
  38. /*}}}*/
  39. using namespace std;
  40. unsigned Quiet = 0;
  41. static struct timeval GetTimevalFromSteadyClock() /*{{{*/
  42. {
  43. auto const Time = std::chrono::steady_clock::now().time_since_epoch();
  44. auto const Time_sec = std::chrono::duration_cast<std::chrono::seconds>(Time);
  45. auto const Time_usec = std::chrono::duration_cast<std::chrono::microseconds>(Time - Time_sec);
  46. return { Time_sec.count(), Time_usec.count() };
  47. }
  48. /*}}}*/
  49. // struct PackageMap - List of all package files in the config file /*{{{*/
  50. // ---------------------------------------------------------------------
  51. /* */
  52. struct PackageMap
  53. {
  54. // General Stuff
  55. string BaseDir;
  56. string InternalPrefix;
  57. string FLFile;
  58. string PkgExt;
  59. string SrcExt;
  60. // Stuff for the Package File
  61. string PkgFile;
  62. string BinCacheDB;
  63. string SrcCacheDB;
  64. string BinOverride;
  65. string ExtraOverride;
  66. // We generate for this given arch
  67. string Arch;
  68. bool IncludeArchAll;
  69. // Stuff for the Source File
  70. string SrcFile;
  71. string SrcOverride;
  72. string SrcExtraOverride;
  73. // Translation master file
  74. bool LongDesc;
  75. TranslationWriter *TransWriter;
  76. // Contents
  77. string Contents;
  78. string ContentsHead;
  79. // Random things
  80. string Tag;
  81. string PkgCompress;
  82. string CntCompress;
  83. string SrcCompress;
  84. string PathPrefix;
  85. unsigned int DeLinkLimit;
  86. mode_t Permissions;
  87. bool ContentsDone;
  88. bool PkgDone;
  89. bool SrcDone;
  90. time_t ContentsMTime;
  91. struct ContentsCompare : public binary_function<PackageMap,PackageMap,bool>
  92. {
  93. inline bool operator() (const PackageMap &x,const PackageMap &y)
  94. {return x.ContentsMTime < y.ContentsMTime;};
  95. };
  96. struct DBCompare : public binary_function<PackageMap,PackageMap,bool>
  97. {
  98. inline bool operator() (const PackageMap &x,const PackageMap &y)
  99. {return x.BinCacheDB < y.BinCacheDB;};
  100. };
  101. struct SrcDBCompare : public binary_function<PackageMap,PackageMap,bool>
  102. {
  103. inline bool operator() (const PackageMap &x,const PackageMap &y)
  104. {return x.SrcCacheDB < y.SrcCacheDB;};
  105. };
  106. void GetGeneral(Configuration &Setup,Configuration &Block);
  107. bool GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats);
  108. bool GenSources(Configuration &Setup,struct CacheDB::Stats &Stats);
  109. bool GenContents(Configuration &Setup,
  110. vector<PackageMap>::iterator Begin,
  111. vector<PackageMap>::iterator End,
  112. unsigned long &Left);
  113. PackageMap() : IncludeArchAll(true), LongDesc(true), TransWriter(NULL),
  114. DeLinkLimit(0), Permissions(1), ContentsDone(false),
  115. PkgDone(false), SrcDone(false), ContentsMTime(0) {};
  116. };
  117. /*}}}*/
  118. // PackageMap::GetGeneral - Common per-section definitions /*{{{*/
  119. // ---------------------------------------------------------------------
  120. /* */
  121. void PackageMap::GetGeneral(Configuration &Setup,Configuration &Block)
  122. {
  123. PathPrefix = Block.Find("PathPrefix");
  124. if (Block.FindB("External-Links",true) == false)
  125. DeLinkLimit = Setup.FindI("Default::DeLinkLimit", std::numeric_limits<unsigned int>::max());
  126. else
  127. DeLinkLimit = 0;
  128. PkgCompress = Block.Find("Packages::Compress",
  129. Setup.Find("Default::Packages::Compress",". gzip").c_str());
  130. CntCompress = Block.Find("Contents::Compress",
  131. Setup.Find("Default::Contents::Compress",". gzip").c_str());
  132. SrcCompress = Block.Find("Sources::Compress",
  133. Setup.Find("Default::Sources::Compress",". gzip").c_str());
  134. SrcExt = Block.Find("Sources::Extensions",
  135. Setup.Find("Default::Sources::Extensions",".dsc").c_str());
  136. PkgExt = Block.Find("Packages::Extensions",
  137. Setup.Find("Default::Packages::Extensions",".deb").c_str());
  138. Permissions = Setup.FindI("Default::FileMode",0644);
  139. if (FLFile.empty() == false)
  140. FLFile = flCombine(Setup.Find("Dir::FileListDir"),FLFile);
  141. if (Contents == " ")
  142. Contents= string();
  143. }
  144. /*}}}*/
  145. // PackageMap::GenPackages - Actually generate a Package file /*{{{*/
  146. // ---------------------------------------------------------------------
  147. /* This generates the Package File described by this object. */
  148. bool PackageMap::GenPackages(Configuration &Setup,struct CacheDB::Stats &Stats)
  149. {
  150. if (PkgFile.empty() == true)
  151. return true;
  152. string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
  153. string OverrideDir = Setup.FindDir("Dir::OverrideDir");
  154. string CacheDir = Setup.FindDir("Dir::CacheDir");
  155. struct timeval StartTime = GetTimevalFromSteadyClock();
  156. PkgDone = true;
  157. // Create a package writer object.
  158. MultiCompress Comp(flCombine(ArchiveDir,PkgFile),
  159. PkgCompress,Permissions);
  160. PackagesWriter Packages(&Comp.Input, TransWriter, flCombine(CacheDir,BinCacheDB),
  161. flCombine(OverrideDir,BinOverride),
  162. flCombine(OverrideDir,ExtraOverride),
  163. Arch, IncludeArchAll);
  164. if (PkgExt.empty() == false && Packages.SetExts(PkgExt) == false)
  165. return _error->Error(_("Package extension list is too long"));
  166. if (_error->PendingError() == true)
  167. return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
  168. Packages.PathPrefix = PathPrefix;
  169. Packages.DirStrip = ArchiveDir;
  170. Packages.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
  171. Packages.LongDescription = LongDesc;
  172. Packages.Stats.DeLinkBytes = Stats.DeLinkBytes;
  173. Packages.DeLinkLimit = DeLinkLimit;
  174. if (_error->PendingError() == true)
  175. return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
  176. c0out << ' ' << BaseDir << ":" << flush;
  177. // Do recursive directory searching
  178. if (FLFile.empty() == true)
  179. {
  180. if (Packages.RecursiveScan(flCombine(ArchiveDir,BaseDir)) == false)
  181. return false;
  182. }
  183. else
  184. {
  185. if (Packages.LoadFileList(ArchiveDir,FLFile) == false)
  186. return false;
  187. }
  188. Packages.Output = 0; // Just in case
  189. // Finish compressing
  190. unsigned long long Size;
  191. if (Comp.Finalize(Size) == false)
  192. {
  193. c0out << endl;
  194. return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
  195. }
  196. if (Size != 0)
  197. c0out << " New "
  198. << SizeToStr(Size) << "B ";
  199. else
  200. c0out << ' ';
  201. struct timeval NewTime = GetTimevalFromSteadyClock();
  202. std::chrono::duration<double> Delta =
  203. std::chrono::seconds(NewTime.tv_sec - StartTime.tv_sec) +
  204. std::chrono::microseconds(NewTime.tv_sec - StartTime.tv_usec);
  205. c0out << Packages.Stats.Packages << " files " <<
  206. /* SizeToStr(Packages.Stats.MD5Bytes) << "B/" << */
  207. SizeToStr(Packages.Stats.Bytes) << "B " <<
  208. TimeToStr(llround(Delta.count())) << endl;
  209. if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
  210. c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
  211. Stats.Add(Packages.Stats);
  212. Stats.DeLinkBytes = Packages.Stats.DeLinkBytes;
  213. return !_error->PendingError();
  214. }
  215. /*}}}*/
  216. // PackageMap::GenSources - Actually generate a Source file /*{{{*/
  217. // ---------------------------------------------------------------------
  218. /* This generates the Sources File described by this object. */
  219. bool PackageMap::GenSources(Configuration &Setup,struct CacheDB::Stats &Stats)
  220. {
  221. if (SrcFile.empty() == true)
  222. return true;
  223. string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
  224. string OverrideDir = Setup.FindDir("Dir::OverrideDir");
  225. string CacheDir = Setup.FindDir("Dir::CacheDir");
  226. struct timeval StartTime = GetTimevalFromSteadyClock();
  227. SrcDone = true;
  228. // Create a package writer object.
  229. MultiCompress Comp(flCombine(ArchiveDir,SrcFile),
  230. SrcCompress,Permissions);
  231. SourcesWriter Sources(&Comp.Input, flCombine(CacheDir, SrcCacheDB),
  232. flCombine(OverrideDir,BinOverride),
  233. flCombine(OverrideDir,SrcOverride),
  234. flCombine(OverrideDir,SrcExtraOverride));
  235. if (SrcExt.empty() == false && Sources.SetExts(SrcExt) == false)
  236. return _error->Error(_("Source extension list is too long"));
  237. if (_error->PendingError() == true)
  238. return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
  239. Sources.PathPrefix = PathPrefix;
  240. Sources.DirStrip = ArchiveDir;
  241. Sources.InternalPrefix = flCombine(ArchiveDir,InternalPrefix);
  242. Sources.DeLinkLimit = DeLinkLimit;
  243. Sources.Stats.DeLinkBytes = Stats.DeLinkBytes;
  244. if (_error->PendingError() == true)
  245. return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
  246. c0out << ' ' << BaseDir << ":" << flush;
  247. // Do recursive directory searching
  248. if (FLFile.empty() == true)
  249. {
  250. if (Sources.RecursiveScan(flCombine(ArchiveDir,BaseDir))== false)
  251. return false;
  252. }
  253. else
  254. {
  255. if (Sources.LoadFileList(ArchiveDir,FLFile) == false)
  256. return false;
  257. }
  258. Sources.Output = 0; // Just in case
  259. // Finish compressing
  260. unsigned long long Size;
  261. if (Comp.Finalize(Size) == false)
  262. {
  263. c0out << endl;
  264. return _error->Error(_("Error processing directory %s"),BaseDir.c_str());
  265. }
  266. if (Size != 0)
  267. c0out << " New "
  268. << SizeToStr(Size) << "B ";
  269. else
  270. c0out << ' ';
  271. struct timeval NewTime = GetTimevalFromSteadyClock();
  272. std::chrono::duration<double> Delta =
  273. std::chrono::seconds(NewTime.tv_sec - StartTime.tv_sec) +
  274. std::chrono::microseconds(NewTime.tv_sec - StartTime.tv_usec);
  275. c0out << Sources.Stats.Packages << " pkgs in " <<
  276. TimeToStr(llround(Delta.count())) << endl;
  277. if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
  278. c0out << " Misses in Cache: " << Sources.Stats.Misses << endl;
  279. Stats.Add(Sources.Stats);
  280. Stats.DeLinkBytes = Sources.Stats.DeLinkBytes;
  281. return !_error->PendingError();
  282. }
  283. /*}}}*/
  284. // PackageMap::GenContents - Actually generate a Contents file /*{{{*/
  285. // ---------------------------------------------------------------------
  286. /* This generates the contents file partially described by this object.
  287. It searches the given iterator range for other package files that map
  288. into this contents file and includes their data as well when building. */
  289. bool PackageMap::GenContents(Configuration &Setup,
  290. vector<PackageMap>::iterator Begin,
  291. vector<PackageMap>::iterator End,
  292. unsigned long &Left)
  293. {
  294. if (Contents.empty() == true)
  295. return true;
  296. if (Left == 0)
  297. return true;
  298. string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
  299. string CacheDir = Setup.FindDir("Dir::CacheDir");
  300. string OverrideDir = Setup.FindDir("Dir::OverrideDir");
  301. struct timeval StartTime = GetTimevalFromSteadyClock();
  302. // Create a package writer object.
  303. MultiCompress Comp(flCombine(ArchiveDir,this->Contents),
  304. CntCompress,Permissions);
  305. Comp.UpdateMTime = Setup.FindI("Default::ContentsAge",10)*24*60*60;
  306. ContentsWriter Contents(&Comp.Input, "", Arch, IncludeArchAll);
  307. if (PkgExt.empty() == false && Contents.SetExts(PkgExt) == false)
  308. return _error->Error(_("Package extension list is too long"));
  309. if (_error->PendingError() == true)
  310. return false;
  311. if (_error->PendingError() == true)
  312. return false;
  313. // Write the header out.
  314. if (ContentsHead.empty() == false)
  315. {
  316. FileFd Head(flCombine(OverrideDir,ContentsHead),FileFd::ReadOnly);
  317. if (_error->PendingError() == true)
  318. return false;
  319. unsigned long long Size = Head.Size();
  320. unsigned char Buf[4096];
  321. while (Size != 0)
  322. {
  323. unsigned long long ToRead = Size;
  324. if (Size > sizeof(Buf))
  325. ToRead = sizeof(Buf);
  326. if (Head.Read(Buf,ToRead) == false)
  327. return false;
  328. if (Comp.Input.Write(Buf, ToRead) == false)
  329. return _error->Errno("fwrite",_("Error writing header to contents file"));
  330. Size -= ToRead;
  331. }
  332. }
  333. /* Go over all the package file records and parse all the package
  334. files associated with this contents file into one great big honking
  335. memory structure, then dump the sorted version */
  336. c0out << ' ' << this->Contents << ":" << flush;
  337. for (vector<PackageMap>::iterator I = Begin; I != End; ++I)
  338. {
  339. if (I->Contents != this->Contents)
  340. continue;
  341. Contents.Prefix = ArchiveDir;
  342. Contents.ReadyDB(flCombine(CacheDir,I->BinCacheDB));
  343. Contents.ReadFromPkgs(flCombine(ArchiveDir,I->PkgFile),
  344. I->PkgCompress);
  345. I->ContentsDone = true;
  346. }
  347. Contents.Finish();
  348. // Finish compressing
  349. unsigned long long Size;
  350. if (Comp.Finalize(Size) == false || _error->PendingError() == true)
  351. {
  352. c0out << endl;
  353. return _error->Error(_("Error processing contents %s"),
  354. this->Contents.c_str());
  355. }
  356. if (Size != 0)
  357. {
  358. c0out << " New " << SizeToStr(Size) << "B ";
  359. if (Left > Size)
  360. Left -= Size;
  361. else
  362. Left = 0;
  363. }
  364. else
  365. c0out << ' ';
  366. struct timeval NewTime = GetTimevalFromSteadyClock();
  367. std::chrono::duration<double> Delta =
  368. std::chrono::seconds(NewTime.tv_sec - StartTime.tv_sec) +
  369. std::chrono::microseconds(NewTime.tv_sec - StartTime.tv_usec);
  370. if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
  371. c0out << " Misses in Cache: " << Contents.Stats.Misses<< endl;
  372. c0out << Contents.Stats.Packages << " files " <<
  373. SizeToStr(Contents.Stats.Bytes) << "B " <<
  374. TimeToStr(llround(Delta.count())) << endl;
  375. return true;
  376. }
  377. /*}}}*/
  378. // LoadTree - Load a 'tree' section from the Generate Config /*{{{*/
  379. // ---------------------------------------------------------------------
  380. /* This populates the PkgList with all the possible permutations of the
  381. section/arch lists. */
  382. static void LoadTree(vector<PackageMap> &PkgList, std::vector<TranslationWriter*> &TransList, Configuration &Setup)
  383. {
  384. // Load the defaults
  385. string DDir = Setup.Find("TreeDefault::Directory",
  386. "$(DIST)/$(SECTION)/binary-$(ARCH)/");
  387. string DSDir = Setup.Find("TreeDefault::SrcDirectory",
  388. "$(DIST)/$(SECTION)/source/");
  389. string DPkg = Setup.Find("TreeDefault::Packages",
  390. "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages");
  391. string DTrans = Setup.Find("TreeDefault::Translation",
  392. "$(DIST)/$(SECTION)/i18n/Translation-en");
  393. string DIPrfx = Setup.Find("TreeDefault::InternalPrefix",
  394. "$(DIST)/$(SECTION)/");
  395. string DContents = Setup.Find("TreeDefault::Contents",
  396. "$(DIST)/$(SECTION)/Contents-$(ARCH)");
  397. string DContentsH = Setup.Find("TreeDefault::Contents::Header","");
  398. string DBCache = Setup.Find("TreeDefault::BinCacheDB",
  399. "packages-$(ARCH).db");
  400. string SrcDBCache = Setup.Find("TreeDefault::SrcCacheDB",
  401. "sources-$(SECTION).db");
  402. string DSources = Setup.Find("TreeDefault::Sources",
  403. "$(DIST)/$(SECTION)/source/Sources");
  404. string DFLFile = Setup.Find("TreeDefault::FileList", "");
  405. string DSFLFile = Setup.Find("TreeDefault::SourceFileList", "");
  406. mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
  407. bool const LongDescription = Setup.FindB("Default::LongDescription",
  408. _config->FindB("APT::FTPArchive::LongDescription", true));
  409. string const TranslationCompress = Setup.Find("Default::Translation::Compress",". gzip").c_str();
  410. bool const ConfIncludeArchAllExists = _config->Exists("APT::FTPArchive::IncludeArchitectureAll");
  411. bool const ConfIncludeArchAll = _config->FindB("APT::FTPArchive::IncludeArchitectureAll", true);
  412. // Process 'tree' type sections
  413. const Configuration::Item *Top = Setup.Tree("tree");
  414. for (Top = (Top == 0?0:Top->Child); Top != 0;)
  415. {
  416. Configuration Block(Top);
  417. string Dist = Top->Tag;
  418. // Parse the sections
  419. string Tmp = Block.Find("Sections");
  420. const char *Sections = Tmp.c_str();
  421. string Section;
  422. while (ParseQuoteWord(Sections,Section) == true)
  423. {
  424. struct SubstVar Vars[] = {{"$(DIST)",&Dist},
  425. {"$(SECTION)",&Section},
  426. {"$(ARCH)",nullptr},
  427. {nullptr, nullptr}};
  428. mode_t const Perms = Block.FindI("FileMode", Permissions);
  429. bool const LongDesc = Block.FindB("LongDescription", LongDescription);
  430. TranslationWriter *TransWriter = nullptr;
  431. std::string Tmp2 = Block.Find("Architectures");
  432. std::transform(Tmp2.begin(), Tmp2.end(), Tmp2.begin(), ::tolower);
  433. std::vector<std::string> const Archs = VectorizeString(Tmp2, ' ');
  434. bool IncludeArchAll;
  435. if (ConfIncludeArchAllExists == true)
  436. IncludeArchAll = ConfIncludeArchAll;
  437. else
  438. IncludeArchAll = std::find(Archs.begin(), Archs.end(), "all") == Archs.end();
  439. for (auto const& Arch: Archs)
  440. {
  441. if (Arch.empty()) continue;
  442. Vars[2].Contents = &Arch;
  443. PackageMap Itm;
  444. Itm.Permissions = Perms;
  445. Itm.BinOverride = SubstVar(Block.Find("BinOverride"),Vars);
  446. Itm.InternalPrefix = SubstVar(Block.Find("InternalPrefix",DIPrfx.c_str()),Vars);
  447. if (Arch == "source")
  448. {
  449. Itm.SrcOverride = SubstVar(Block.Find("SrcOverride"),Vars);
  450. Itm.BaseDir = SubstVar(Block.Find("SrcDirectory",DSDir.c_str()),Vars);
  451. Itm.SrcFile = SubstVar(Block.Find("Sources",DSources.c_str()),Vars);
  452. Itm.Tag = SubstVar("$(DIST)/$(SECTION)/source",Vars);
  453. Itm.FLFile = SubstVar(Block.Find("SourceFileList",DSFLFile.c_str()),Vars);
  454. Itm.SrcExtraOverride = SubstVar(Block.Find("SrcExtraOverride"),Vars);
  455. Itm.SrcCacheDB = SubstVar(Block.Find("SrcCacheDB",SrcDBCache.c_str()),Vars);
  456. }
  457. else
  458. {
  459. Itm.BinCacheDB = SubstVar(Block.Find("BinCacheDB",DBCache.c_str()),Vars);
  460. Itm.BaseDir = SubstVar(Block.Find("Directory",DDir.c_str()),Vars);
  461. Itm.PkgFile = SubstVar(Block.Find("Packages",DPkg.c_str()),Vars);
  462. Itm.Tag = SubstVar("$(DIST)/$(SECTION)/$(ARCH)",Vars);
  463. Itm.Arch = Arch;
  464. Itm.IncludeArchAll = IncludeArchAll;
  465. Itm.LongDesc = LongDesc;
  466. if (TransWriter == NULL && DTrans.empty() == false && LongDesc == false && DTrans != "/dev/null")
  467. {
  468. string const TranslationFile = flCombine(Setup.FindDir("Dir::ArchiveDir"),
  469. SubstVar(Block.Find("Translation", DTrans.c_str()), Vars));
  470. string const TransCompress = Block.Find("Translation::Compress", TranslationCompress);
  471. TransWriter = new TranslationWriter(TranslationFile, TransCompress, Perms);
  472. TransList.push_back(TransWriter);
  473. }
  474. Itm.TransWriter = TransWriter;
  475. Itm.Contents = SubstVar(Block.Find("Contents",DContents.c_str()),Vars);
  476. Itm.ContentsHead = SubstVar(Block.Find("Contents::Header",DContentsH.c_str()),Vars);
  477. Itm.FLFile = SubstVar(Block.Find("FileList",DFLFile.c_str()),Vars);
  478. Itm.ExtraOverride = SubstVar(Block.Find("ExtraOverride"),Vars);
  479. }
  480. Itm.GetGeneral(Setup,Block);
  481. PkgList.push_back(Itm);
  482. }
  483. }
  484. Top = Top->Next;
  485. }
  486. }
  487. /*}}}*/
  488. static void UnloadTree(std::vector<TranslationWriter*> const &Trans) /*{{{*/
  489. {
  490. for (std::vector<TranslationWriter*>::const_reverse_iterator T = Trans.rbegin(); T != Trans.rend(); ++T)
  491. delete *T;
  492. }
  493. /*}}}*/
  494. // LoadBinDir - Load a 'bindirectory' section from the Generate Config /*{{{*/
  495. // ---------------------------------------------------------------------
  496. /* */
  497. static void LoadBinDir(vector<PackageMap> &PkgList,Configuration &Setup)
  498. {
  499. mode_t const Permissions = Setup.FindI("Default::FileMode",0644);
  500. // Process 'bindirectory' type sections
  501. const Configuration::Item *Top = Setup.Tree("bindirectory");
  502. for (Top = (Top == 0?0:Top->Child); Top != 0;)
  503. {
  504. Configuration Block(Top);
  505. PackageMap Itm;
  506. Itm.PkgFile = Block.Find("Packages");
  507. Itm.SrcFile = Block.Find("Sources");
  508. Itm.BinCacheDB = Block.Find("BinCacheDB");
  509. Itm.SrcCacheDB = Block.Find("SrcCacheDB");
  510. Itm.BinOverride = Block.Find("BinOverride");
  511. Itm.ExtraOverride = Block.Find("ExtraOverride");
  512. Itm.SrcExtraOverride = Block.Find("SrcExtraOverride");
  513. Itm.SrcOverride = Block.Find("SrcOverride");
  514. Itm.BaseDir = Top->Tag;
  515. Itm.FLFile = Block.Find("FileList");
  516. Itm.InternalPrefix = Block.Find("InternalPrefix",Top->Tag.c_str());
  517. Itm.Contents = Block.Find("Contents");
  518. Itm.ContentsHead = Block.Find("Contents::Header");
  519. Itm.Permissions = Block.FindI("FileMode", Permissions);
  520. Itm.GetGeneral(Setup,Block);
  521. PkgList.push_back(Itm);
  522. Top = Top->Next;
  523. }
  524. }
  525. /*}}}*/
  526. static bool ShowHelp(CommandLine &) /*{{{*/
  527. {
  528. std::cout <<
  529. _("Usage: apt-ftparchive [options] command\n"
  530. "Commands: packages binarypath [overridefile [pathprefix]]\n"
  531. " sources srcpath [overridefile [pathprefix]]\n"
  532. " contents path\n"
  533. " release path\n"
  534. " generate config [groups]\n"
  535. " clean config\n"
  536. "\n"
  537. "apt-ftparchive generates index files for Debian archives. It supports\n"
  538. "many styles of generation from fully automated to functional replacements\n"
  539. "for dpkg-scanpackages and dpkg-scansources\n"
  540. "\n"
  541. "apt-ftparchive generates Package files from a tree of .debs. The\n"
  542. "Package file contains the contents of all the control fields from\n"
  543. "each package as well as the MD5 hash and filesize. An override file\n"
  544. "is supported to force the value of Priority and Section.\n"
  545. "\n"
  546. "Similarly apt-ftparchive generates Sources files from a tree of .dscs.\n"
  547. "The --source-override option can be used to specify a src override file\n"
  548. "\n"
  549. "The 'packages' and 'sources' command should be run in the root of the\n"
  550. "tree. BinaryPath should point to the base of the recursive search and \n"
  551. "override file should contain the override flags. Pathprefix is\n"
  552. "appended to the filename fields if present. Example usage from the \n"
  553. "Debian archive:\n"
  554. " apt-ftparchive packages dists/potato/main/binary-i386/ > \\\n"
  555. " dists/potato/main/binary-i386/Packages\n"
  556. "\n"
  557. "Options:\n"
  558. " -h This help text\n"
  559. " --md5 Control MD5 generation\n"
  560. " -s=? Source override file\n"
  561. " -q Quiet\n"
  562. " -d=? Select the optional caching database\n"
  563. " --no-delink Enable delinking debug mode\n"
  564. " --contents Control contents file generation\n"
  565. " -c=? Read this configuration file\n"
  566. " -o=? Set an arbitrary configuration option") << endl;
  567. return true;
  568. }
  569. /*}}}*/
  570. // SimpleGenPackages - Generate a Packages file for a directory tree /*{{{*/
  571. // ---------------------------------------------------------------------
  572. /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
  573. static bool SimpleGenPackages(CommandLine &CmdL)
  574. {
  575. if (CmdL.FileSize() < 2)
  576. return ShowHelp(CmdL);
  577. string Override;
  578. if (CmdL.FileSize() >= 3)
  579. Override = CmdL.FileList[2];
  580. // Create a package writer object.
  581. PackagesWriter Packages(NULL, NULL, _config->Find("APT::FTPArchive::DB"),
  582. Override, "", _config->Find("APT::FTPArchive::Architecture"),
  583. _config->FindB("APT::FTPArchive::IncludeArchitectureAll", true));
  584. if (_error->PendingError() == true)
  585. return false;
  586. if (CmdL.FileSize() >= 4)
  587. Packages.PathPrefix = CmdL.FileList[3];
  588. // Do recursive directory searching
  589. if (Packages.RecursiveScan(CmdL.FileList[1]) == false)
  590. return false;
  591. // Give some stats if asked for
  592. if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
  593. c0out << " Misses in Cache: " << Packages.Stats.Misses<< endl;
  594. return true;
  595. }
  596. /*}}}*/
  597. // SimpleGenContents - Generate a Contents listing /*{{{*/
  598. // ---------------------------------------------------------------------
  599. /* */
  600. static bool SimpleGenContents(CommandLine &CmdL)
  601. {
  602. if (CmdL.FileSize() < 2)
  603. return ShowHelp(CmdL);
  604. // Create a package writer object.
  605. ContentsWriter Contents(NULL, _config->Find("APT::FTPArchive::DB"), _config->Find("APT::FTPArchive::Architecture"));
  606. if (_error->PendingError() == true)
  607. return false;
  608. // Do recursive directory searching
  609. if (Contents.RecursiveScan(CmdL.FileList[1]) == false)
  610. return false;
  611. Contents.Finish();
  612. return true;
  613. }
  614. /*}}}*/
  615. // SimpleGenSources - Generate a Sources file for a directory tree /*{{{*/
  616. // ---------------------------------------------------------------------
  617. /* This emulates dpkg-scanpackages's command line interface. 'mostly' */
  618. static bool SimpleGenSources(CommandLine &CmdL)
  619. {
  620. if (CmdL.FileSize() < 2)
  621. return ShowHelp(CmdL);
  622. string Override;
  623. if (CmdL.FileSize() >= 3)
  624. Override = CmdL.FileList[2];
  625. string SOverride;
  626. if (Override.empty() == false)
  627. SOverride = Override + ".src";
  628. SOverride = _config->Find("APT::FTPArchive::SourceOverride",
  629. SOverride.c_str());
  630. // Create a package writer object.
  631. SourcesWriter Sources(NULL, _config->Find("APT::FTPArchive::DB"),Override,SOverride);
  632. if (_error->PendingError() == true)
  633. return false;
  634. if (CmdL.FileSize() >= 4)
  635. Sources.PathPrefix = CmdL.FileList[3];
  636. // Do recursive directory searching
  637. if (Sources.RecursiveScan(CmdL.FileList[1]) == false)
  638. return false;
  639. // Give some stats if asked for
  640. if(_config->FindB("APT::FTPArchive::ShowCacheMisses", false) == true)
  641. c0out << " Misses in Cache: " << Sources.Stats.Misses<< endl;
  642. return true;
  643. }
  644. /*}}}*/
  645. // SimpleGenRelease - Generate a Release file for a directory tree /*{{{*/
  646. // ---------------------------------------------------------------------
  647. static bool SimpleGenRelease(CommandLine &CmdL)
  648. {
  649. if (CmdL.FileSize() < 2)
  650. return ShowHelp(CmdL);
  651. string Dir = CmdL.FileList[1];
  652. ReleaseWriter Release(NULL, "");
  653. Release.DirStrip = Dir;
  654. if (_error->PendingError() == true)
  655. return false;
  656. if (Release.RecursiveScan(Dir) == false)
  657. return false;
  658. Release.Finish();
  659. return true;
  660. }
  661. /*}}}*/
  662. // DoGeneratePackagesAndSources - Helper for Generate /*{{{*/
  663. // ---------------------------------------------------------------------
  664. static bool DoGeneratePackagesAndSources(Configuration &Setup,
  665. vector<PackageMap> &PkgList,
  666. struct CacheDB::Stats &SrcStats,
  667. struct CacheDB::Stats &Stats,
  668. CommandLine &CmdL)
  669. {
  670. if (CmdL.FileSize() <= 2)
  671. {
  672. for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
  673. if (I->GenPackages(Setup,Stats) == false)
  674. _error->DumpErrors();
  675. for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
  676. if (I->GenSources(Setup,SrcStats) == false)
  677. _error->DumpErrors();
  678. }
  679. else
  680. {
  681. // Make a choice list out of the package list..
  682. RxChoiceList *List = new RxChoiceList[2*PkgList.size()+1];
  683. RxChoiceList *End = List;
  684. for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
  685. {
  686. End->UserData = &(*I);
  687. End->Str = I->BaseDir.c_str();
  688. End++;
  689. End->UserData = &(*I);
  690. End->Str = I->Tag.c_str();
  691. End++;
  692. }
  693. End->Str = 0;
  694. // Regex it
  695. if (RegexChoice(List,CmdL.FileList + 2,CmdL.FileList + CmdL.FileSize()) == 0)
  696. {
  697. delete [] List;
  698. return _error->Error(_("No selections matched"));
  699. }
  700. _error->DumpErrors();
  701. // Do the generation for Packages
  702. for (End = List; End->Str != 0; ++End)
  703. {
  704. if (End->Hit == false)
  705. continue;
  706. PackageMap * const I = static_cast<PackageMap *>(End->UserData);
  707. if (I->PkgDone == true)
  708. continue;
  709. if (I->GenPackages(Setup,Stats) == false)
  710. _error->DumpErrors();
  711. }
  712. // Do the generation for Sources
  713. for (End = List; End->Str != 0; ++End)
  714. {
  715. if (End->Hit == false)
  716. continue;
  717. PackageMap * const I = static_cast<PackageMap *>(End->UserData);
  718. if (I->SrcDone == true)
  719. continue;
  720. if (I->GenSources(Setup,SrcStats) == false)
  721. _error->DumpErrors();
  722. }
  723. delete [] List;
  724. }
  725. return true;
  726. }
  727. /*}}}*/
  728. // DoGenerateContents - Helper for Generate to generate the Contents /*{{{*/
  729. // ---------------------------------------------------------------------
  730. static bool DoGenerateContents(Configuration &Setup,
  731. vector<PackageMap> &PkgList,
  732. CommandLine &CmdL)
  733. {
  734. c1out << "Packages done, Starting contents." << endl;
  735. // Sort the contents file list by date
  736. string ArchiveDir = Setup.FindDir("Dir::ArchiveDir");
  737. for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
  738. {
  739. struct stat A;
  740. if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),
  741. I->CntCompress,A) == false)
  742. time(&I->ContentsMTime);
  743. else
  744. I->ContentsMTime = A.st_mtime;
  745. }
  746. stable_sort(PkgList.begin(),PkgList.end(),PackageMap::ContentsCompare());
  747. /* Now for Contents.. The process here is to do a make-like dependency
  748. check. Each contents file is verified to be newer than the package files
  749. that describe the debs it indexes. Since the package files contain
  750. hashes of the .debs this means they have not changed either so the
  751. contents must be up to date. */
  752. unsigned long MaxContentsChange = Setup.FindI("Default::MaxContentsChange",
  753. std::numeric_limits<unsigned int>::max())*1024;
  754. for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); ++I)
  755. {
  756. // This record is not relevant
  757. if (I->ContentsDone == true ||
  758. I->Contents.empty() == true)
  759. continue;
  760. // Do not do everything if the user specified sections.
  761. if (CmdL.FileSize() > 2 && I->PkgDone == false)
  762. continue;
  763. struct stat A,B;
  764. if (MultiCompress::GetStat(flCombine(ArchiveDir,I->Contents),I->CntCompress,A) == true)
  765. {
  766. if (MultiCompress::GetStat(flCombine(ArchiveDir,I->PkgFile),I->PkgCompress,B) == false)
  767. {
  768. _error->Warning(_("Some files are missing in the package file group `%s'"),I->PkgFile.c_str());
  769. continue;
  770. }
  771. if (A.st_mtime > B.st_mtime)
  772. continue;
  773. }
  774. if (I->GenContents(Setup,PkgList.begin(),PkgList.end(),
  775. MaxContentsChange) == false)
  776. _error->DumpErrors();
  777. // Hit the limit?
  778. if (MaxContentsChange == 0)
  779. {
  780. c1out << "Hit contents update byte limit" << endl;
  781. break;
  782. }
  783. }
  784. return true;
  785. }
  786. /*}}}*/
  787. // Generate - Full generate, using a config file /*{{{*/
  788. // ---------------------------------------------------------------------
  789. /* */
  790. static bool Generate(CommandLine &CmdL)
  791. {
  792. struct CacheDB::Stats SrcStats;
  793. if (CmdL.FileSize() < 2)
  794. return ShowHelp(CmdL);
  795. struct timeval StartTime = GetTimevalFromSteadyClock();
  796. struct CacheDB::Stats Stats;
  797. // Read the configuration file.
  798. Configuration Setup;
  799. if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
  800. return false;
  801. vector<PackageMap> PkgList;
  802. std::vector<TranslationWriter*> TransList;
  803. LoadTree(PkgList, TransList, Setup);
  804. LoadBinDir(PkgList,Setup);
  805. // Sort by cache DB to improve IO locality.
  806. stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
  807. stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
  808. // Generate packages
  809. if (_config->FindB("APT::FTPArchive::ContentsOnly", false) == false)
  810. {
  811. if(DoGeneratePackagesAndSources(Setup, PkgList, SrcStats, Stats, CmdL) == false)
  812. {
  813. UnloadTree(TransList);
  814. return false;
  815. }
  816. } else {
  817. c1out << "Skipping Packages/Sources generation" << endl;
  818. }
  819. // do Contents if needed
  820. if (_config->FindB("APT::FTPArchive::Contents", true) == true)
  821. if (DoGenerateContents(Setup, PkgList, CmdL) == false)
  822. {
  823. UnloadTree(TransList);
  824. return false;
  825. }
  826. struct timeval NewTime = GetTimevalFromSteadyClock();
  827. std::chrono::duration<double> Delta =
  828. std::chrono::seconds(NewTime.tv_sec - StartTime.tv_sec) +
  829. std::chrono::microseconds(NewTime.tv_sec - StartTime.tv_usec);
  830. c1out << "Done. " << SizeToStr(Stats.Bytes) << "B in " << Stats.Packages
  831. << " archives. Took " << TimeToStr(llround(Delta.count())) << endl;
  832. UnloadTree(TransList);
  833. return true;
  834. }
  835. /*}}}*/
  836. // Clean - Clean out the databases /*{{{*/
  837. // ---------------------------------------------------------------------
  838. /* */
  839. static bool Clean(CommandLine &CmdL)
  840. {
  841. if (CmdL.FileSize() != 2)
  842. return ShowHelp(CmdL);
  843. // Read the configuration file.
  844. Configuration Setup;
  845. if (ReadConfigFile(Setup,CmdL.FileList[1],true) == false)
  846. return false;
  847. // we don't need translation creation here
  848. Setup.Set("TreeDefault::Translation", "/dev/null");
  849. vector<PackageMap> PkgList;
  850. std::vector<TranslationWriter*> TransList;
  851. LoadTree(PkgList, TransList, Setup);
  852. LoadBinDir(PkgList,Setup);
  853. // Sort by cache DB to improve IO locality.
  854. stable_sort(PkgList.begin(),PkgList.end(),PackageMap::DBCompare());
  855. stable_sort(PkgList.begin(),PkgList.end(),PackageMap::SrcDBCompare());
  856. string CacheDir = Setup.FindDir("Dir::CacheDir");
  857. for (vector<PackageMap>::iterator I = PkgList.begin(); I != PkgList.end(); )
  858. {
  859. if(I->BinCacheDB != "")
  860. c0out << I->BinCacheDB << endl;
  861. if(I->SrcCacheDB != "")
  862. c0out << I->SrcCacheDB << endl;
  863. CacheDB DB(flCombine(CacheDir,I->BinCacheDB));
  864. CacheDB DB_SRC(flCombine(CacheDir,I->SrcCacheDB));
  865. if (DB.Clean() == false)
  866. _error->DumpErrors();
  867. if (DB_SRC.Clean() == false)
  868. _error->DumpErrors();
  869. I = std::find_if(I, PkgList.end(),
  870. [&](PackageMap const &PM) { return PM.BinCacheDB != I->BinCacheDB || PM.SrcCacheDB != I->SrcCacheDB;
  871. });
  872. }
  873. return true;
  874. }
  875. /*}}}*/
  876. static std::vector<aptDispatchWithHelp> GetCommands() /*{{{*/
  877. {
  878. return {
  879. {"packages",&SimpleGenPackages, nullptr},
  880. {"contents",&SimpleGenContents, nullptr},
  881. {"sources",&SimpleGenSources, nullptr},
  882. {"release",&SimpleGenRelease, nullptr},
  883. {"generate",&Generate, nullptr},
  884. {"clean",&Clean, nullptr},
  885. {nullptr, nullptr, nullptr}
  886. };
  887. }
  888. /*}}}*/
  889. int main(int argc, const char *argv[]) /*{{{*/
  890. {
  891. // Parse the command line and initialize the package library
  892. CommandLine CmdL;
  893. auto const Cmds = ParseCommandLine(CmdL, APT_CMD::APT_FTPARCHIVE, &_config, NULL, argc, argv, ShowHelp, &GetCommands);
  894. _config->CndSet("quiet",0);
  895. Quiet = _config->FindI("quiet",0);
  896. InitOutput(clog.rdbuf());
  897. return DispatchCommandLine(CmdL, Cmds);
  898. }
  899. /*}}}*/