You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

1206 lines
39 KiB

  1. // -*- mode: cpp; mode: fold -*-
  2. // Description /*{{{*/
  3. /* ######################################################################
  4. Writer
  5. The file writer classes. These write various types of output, sources,
  6. packages and contents.
  7. ##################################################################### */
  8. /*}}}*/
  9. // Include Files /*{{{*/
  10. #include <config.h>
  11. #include <apt-pkg/configuration.h>
  12. #include <apt-pkg/debfile.h>
  13. #include <apt-pkg/deblistparser.h>
  14. #include <apt-pkg/error.h>
  15. #include <apt-pkg/fileutl.h>
  16. #include <apt-pkg/gpgv.h>
  17. #include <apt-pkg/hashes.h>
  18. #include <apt-pkg/md5.h>
  19. #include <apt-pkg/pkgcache.h>
  20. #include <apt-pkg/sha1.h>
  21. #include <apt-pkg/sha2.h>
  22. #include <apt-pkg/strutl.h>
  23. #include <apt-pkg/tagfile.h>
  24. #include <algorithm>
  25. #include <ctime>
  26. #include <iomanip>
  27. #include <iostream>
  28. #include <memory>
  29. #include <sstream>
  30. #include <utility>
  31. #include <ctype.h>
  32. #include <fnmatch.h>
  33. #include <ftw.h>
  34. #include <locale.h>
  35. #include <string.h>
  36. #include <sys/stat.h>
  37. #include <sys/types.h>
  38. #include <unistd.h>
  39. #include "apt-ftparchive.h"
  40. #include "byhash.h"
  41. #include "cachedb.h"
  42. #include "multicompress.h"
  43. #include "writer.h"
  44. #include <apti18n.h>
  45. /*}}}*/
  46. using namespace std;
  47. FTWScanner *FTWScanner::Owner;
  48. // ConfigToDoHashes - which hashes to generate /*{{{*/
  49. static void SingleConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf, unsigned int const Flag)
  50. {
  51. if (_config->FindB(Conf, (DoHashes & Flag) == Flag) == true)
  52. DoHashes |= Flag;
  53. else
  54. DoHashes &= ~Flag;
  55. }
  56. static void ConfigToDoHashes(unsigned int &DoHashes, std::string const &Conf)
  57. {
  58. SingleConfigToDoHashes(DoHashes, Conf + "::MD5", Hashes::MD5SUM);
  59. SingleConfigToDoHashes(DoHashes, Conf + "::SHA1", Hashes::SHA1SUM);
  60. SingleConfigToDoHashes(DoHashes, Conf + "::SHA256", Hashes::SHA256SUM);
  61. SingleConfigToDoHashes(DoHashes, Conf + "::SHA512", Hashes::SHA512SUM);
  62. }
  63. /*}}}*/
  64. // FTWScanner::FTWScanner - Constructor /*{{{*/
  65. FTWScanner::FTWScanner(FileFd * const GivenOutput, string const &Arch, bool const IncludeArchAll)
  66. : Arch(Arch), IncludeArchAll(IncludeArchAll), DoHashes(~0)
  67. {
  68. if (GivenOutput == NULL)
  69. {
  70. Output = new FileFd;
  71. OwnsOutput = true;
  72. Output->OpenDescriptor(STDOUT_FILENO, FileFd::WriteOnly, false);
  73. }
  74. else
  75. {
  76. Output = GivenOutput;
  77. OwnsOutput = false;
  78. }
  79. ErrorPrinted = false;
  80. NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
  81. ConfigToDoHashes(DoHashes, "APT::FTPArchive");
  82. }
  83. /*}}}*/
  84. FTWScanner::~FTWScanner()
  85. {
  86. if (Output != NULL && OwnsOutput)
  87. delete Output;
  88. }
  89. // FTWScanner::Scanner - FTW Scanner /*{{{*/
  90. // ---------------------------------------------------------------------
  91. /* This is the FTW scanner, it processes each directory element in the
  92. directory tree. */
  93. int FTWScanner::ScannerFTW(const char *File,const struct stat * /*sb*/,int Flag)
  94. {
  95. if (Flag == FTW_DNR)
  96. {
  97. Owner->NewLine(1);
  98. ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
  99. }
  100. if (Flag == FTW_NS)
  101. {
  102. Owner->NewLine(1);
  103. ioprintf(c1out, _("W: Unable to stat %s\n"), File);
  104. }
  105. if (Flag != FTW_F)
  106. return 0;
  107. return ScannerFile(File, true);
  108. }
  109. /*}}}*/
  110. static bool FileMatchesPatterns(char const *const File, std::vector<std::string> const &Patterns) /*{{{*/
  111. {
  112. const char *LastComponent = strrchr(File, '/');
  113. if (LastComponent == nullptr)
  114. LastComponent = File;
  115. else
  116. ++LastComponent;
  117. return std::any_of(Patterns.cbegin(), Patterns.cend(), [&](std::string const &pattern) {
  118. return fnmatch(pattern.c_str(), LastComponent, 0) == 0;
  119. });
  120. }
  121. /*}}}*/
  122. int FTWScanner::ScannerFile(const char *const File, bool const ReadLink) /*{{{*/
  123. {
  124. if (FileMatchesPatterns(File, Owner->Patterns) == false)
  125. return 0;
  126. Owner->FilesToProcess.emplace_back(File, ReadLink);
  127. return 0;
  128. }
  129. /*}}}*/
  130. int FTWScanner::ProcessFile(const char *const File, bool const ReadLink) /*{{{*/
  131. {
  132. /* Process it. If the file is a link then resolve it into an absolute
  133. name.. This works best if the directory components the scanner are
  134. given are not links themselves. */
  135. char Jnk[2];
  136. char *RealPath = NULL;
  137. Owner->OriginalPath = File;
  138. if (ReadLink &&
  139. readlink(File,Jnk,sizeof(Jnk)) != -1 &&
  140. (RealPath = realpath(File,NULL)) != 0)
  141. {
  142. Owner->DoPackage(RealPath);
  143. free(RealPath);
  144. }
  145. else
  146. Owner->DoPackage(File);
  147. if (_error->empty() == false)
  148. {
  149. // Print any errors or warnings found
  150. string Err;
  151. bool SeenPath = false;
  152. while (_error->empty() == false)
  153. {
  154. Owner->NewLine(1);
  155. bool const Type = _error->PopMessage(Err);
  156. if (Type == true)
  157. cerr << _("E: ") << Err << endl;
  158. else
  159. cerr << _("W: ") << Err << endl;
  160. if (Err.find(File) != string::npos)
  161. SeenPath = true;
  162. }
  163. if (SeenPath == false)
  164. cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
  165. return 0;
  166. }
  167. return 0;
  168. }
  169. /*}}}*/
  170. // FTWScanner::RecursiveScan - Just scan a directory tree /*{{{*/
  171. // ---------------------------------------------------------------------
  172. /* */
  173. bool FTWScanner::RecursiveScan(string const &Dir)
  174. {
  175. /* If noprefix is set then jam the scan root in, so we don't generate
  176. link followed paths out of control */
  177. if (InternalPrefix.empty() == true)
  178. {
  179. char *RealPath = nullptr;
  180. if ((RealPath = realpath(Dir.c_str(), nullptr)) == 0)
  181. return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
  182. InternalPrefix = RealPath;
  183. free(RealPath);
  184. }
  185. // Do recursive directory searching
  186. Owner = this;
  187. int const Res = ftw(Dir.c_str(),ScannerFTW,30);
  188. // Error treewalking?
  189. if (Res != 0)
  190. {
  191. if (_error->PendingError() == false)
  192. _error->Errno("ftw",_("Tree walking failed"));
  193. return false;
  194. }
  195. using PairType = decltype(*FilesToProcess.cbegin());
  196. std::sort(FilesToProcess.begin(), FilesToProcess.end(), [](PairType a, PairType b) {
  197. return a.first < b.first;
  198. });
  199. for (PairType it : FilesToProcess)
  200. if (ProcessFile(it.first.c_str(), it.second) != 0)
  201. return false;
  202. FilesToProcess.clear();
  203. return true;
  204. }
  205. /*}}}*/
  206. // FTWScanner::LoadFileList - Load the file list from a file /*{{{*/
  207. // ---------------------------------------------------------------------
  208. /* This is an alternative to using FTW to locate files, it reads the list
  209. of files from another file. */
  210. bool FTWScanner::LoadFileList(string const &Dir, string const &File)
  211. {
  212. /* If noprefix is set then jam the scan root in, so we don't generate
  213. link followed paths out of control */
  214. if (InternalPrefix.empty() == true)
  215. {
  216. char *RealPath = nullptr;
  217. if ((RealPath = realpath(Dir.c_str(), nullptr)) == 0)
  218. return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
  219. InternalPrefix = RealPath;
  220. free(RealPath);
  221. }
  222. Owner = this;
  223. FILE *List = fopen(File.c_str(),"r");
  224. if (List == 0)
  225. return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
  226. /* We are a tad tricky here.. We prefix the buffer with the directory
  227. name, that way if we need a full path with just use line.. Sneaky and
  228. fully evil. */
  229. char Line[1000];
  230. char *FileStart;
  231. if (Dir.empty() == true || Dir.end()[-1] != '/')
  232. FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
  233. else
  234. FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
  235. while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
  236. {
  237. char *FileName = _strstrip(FileStart);
  238. if (FileName[0] == 0)
  239. continue;
  240. if (FileName[0] != '/')
  241. {
  242. if (FileName != FileStart)
  243. memmove(FileStart,FileName,strlen(FileStart));
  244. FileName = Line;
  245. }
  246. #if 0
  247. struct stat St;
  248. int Flag = FTW_F;
  249. if (stat(FileName,&St) != 0)
  250. Flag = FTW_NS;
  251. #endif
  252. if (FileMatchesPatterns(FileName, Patterns) == false)
  253. continue;
  254. if (ProcessFile(FileName, false) != 0)
  255. break;
  256. }
  257. fclose(List);
  258. return true;
  259. }
  260. /*}}}*/
  261. // FTWScanner::Delink - Delink symlinks /*{{{*/
  262. // ---------------------------------------------------------------------
  263. /* */
  264. bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
  265. unsigned long long &DeLinkBytes,
  266. unsigned long long const &FileSize)
  267. {
  268. // See if this isn't an internally prefix'd file name.
  269. if (InternalPrefix.empty() == false &&
  270. InternalPrefix.length() < FileName.length() &&
  271. stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
  272. InternalPrefix.begin(),InternalPrefix.end()) != 0)
  273. {
  274. if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
  275. {
  276. // Tidy up the display
  277. if (DeLinkBytes == 0)
  278. cout << endl;
  279. NewLine(1);
  280. ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
  281. SizeToStr(FileSize).c_str());
  282. c1out << flush;
  283. if (NoLinkAct == false)
  284. {
  285. char OldLink[400];
  286. if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
  287. _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
  288. else
  289. {
  290. if (RemoveFile("FTWScanner::Delink", OriginalPath))
  291. {
  292. if (link(FileName.c_str(),OriginalPath) != 0)
  293. {
  294. // Panic! Restore the symlink
  295. if (symlink(OldLink,OriginalPath) != 0)
  296. _error->Errno("symlink", "failed to restore symlink");
  297. return _error->Errno("link",_("*** Failed to link %s to %s"),
  298. FileName.c_str(),
  299. OriginalPath);
  300. }
  301. }
  302. }
  303. }
  304. DeLinkBytes += FileSize;
  305. if (DeLinkBytes/1024 >= DeLinkLimit)
  306. ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
  307. }
  308. FileName = OriginalPath;
  309. }
  310. return true;
  311. }
  312. /*}}}*/
  313. // FTWScanner::SetExts - Set extensions to support /*{{{*/
  314. // ---------------------------------------------------------------------
  315. /* */
  316. bool FTWScanner::SetExts(string const &Vals)
  317. {
  318. ClearPatterns();
  319. string::size_type Start = 0;
  320. while (Start <= Vals.length()-1)
  321. {
  322. string::size_type const Space = Vals.find(' ',Start);
  323. string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
  324. if ( Arch.empty() == false )
  325. {
  326. AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
  327. if (IncludeArchAll == true && Arch != "all")
  328. AddPattern(string("*_all") + Vals.substr(Start, Length));
  329. }
  330. else
  331. AddPattern(string("*") + Vals.substr(Start, Length));
  332. Start += Length + 1;
  333. }
  334. return true;
  335. }
  336. /*}}}*/
  337. // PackagesWriter::PackagesWriter - Constructor /*{{{*/
  338. // ---------------------------------------------------------------------
  339. /* */
  340. PackagesWriter::PackagesWriter(FileFd * const GivenOutput, TranslationWriter * const transWriter,
  341. string const &DB,string const &Overrides,string const &ExtOverrides,
  342. string const &Arch, bool const IncludeArchAll) :
  343. FTWScanner(GivenOutput, Arch, IncludeArchAll), Db(DB), Stats(Db.Stats), TransWriter(transWriter)
  344. {
  345. SetExts(".deb .ddeb .udeb");
  346. DeLinkLimit = 0;
  347. // Process the command line options
  348. ConfigToDoHashes(DoHashes, "APT::FTPArchive::Packages");
  349. DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
  350. DoContents = _config->FindB("APT::FTPArchive::Contents",true);
  351. NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
  352. LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
  353. if (Db.Loaded() == false)
  354. DoContents = false;
  355. // Read the override file
  356. if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
  357. return;
  358. else
  359. NoOverride = true;
  360. if (ExtOverrides.empty() == false)
  361. Over.ReadExtraOverride(ExtOverrides);
  362. _error->DumpErrors();
  363. }
  364. /*}}}*/
  365. // PackagesWriter::DoPackage - Process a single package /*{{{*/
  366. // ---------------------------------------------------------------------
  367. /* This method takes a package and gets its control information and
  368. MD5, SHA1 and SHA256 then writes out a control record with the proper fields
  369. rewritten and the path/size/hash appended. */
  370. bool PackagesWriter::DoPackage(string FileName)
  371. {
  372. // Pull all the data we need form the DB
  373. if (Db.GetFileInfo(FileName,
  374. true, /* DoControl */
  375. DoContents,
  376. true, /* GenContentsOnly */
  377. false, /* DoSource */
  378. DoHashes, DoAlwaysStat) == false)
  379. {
  380. return false;
  381. }
  382. unsigned long long FileSize = Db.GetFileSize();
  383. if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
  384. return false;
  385. // Lookup the override information
  386. pkgTagSection &Tags = Db.Control.Section;
  387. string Package = Tags.FindS("Package");
  388. string Architecture;
  389. // if we generate a Packages file for a given arch, we use it to
  390. // look for overrides. if we run in "simple" mode without the
  391. // "Architectures" variable in the config we use the architecture value
  392. // from the deb file
  393. if(Arch != "")
  394. Architecture = Arch;
  395. else
  396. Architecture = Tags.FindS("Architecture");
  397. unique_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
  398. if (Package.empty() == true)
  399. return _error->Error(_("Archive had no package field"));
  400. // If we need to do any rewriting of the header do it now..
  401. if (OverItem.get() == 0)
  402. {
  403. if (NoOverride == false)
  404. {
  405. NewLine(1);
  406. ioprintf(c1out, _(" %s has no override entry\n"), Package.c_str());
  407. }
  408. OverItem = unique_ptr<Override::Item>(new Override::Item);
  409. OverItem->FieldOverride["Section"] = Tags.FindS("Section");
  410. OverItem->Priority = Tags.FindS("Priority");
  411. }
  412. // Strip the DirStrip prefix from the FileName and add the PathPrefix
  413. string NewFileName;
  414. if (DirStrip.empty() == false &&
  415. FileName.length() > DirStrip.length() &&
  416. stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
  417. DirStrip.begin(),DirStrip.end()) == 0)
  418. NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
  419. else
  420. NewFileName = FileName;
  421. if (PathPrefix.empty() == false)
  422. NewFileName = flCombine(PathPrefix,NewFileName);
  423. /* Configuration says we don't want to include the long Description
  424. in the package file - instead we want to ship a separated file */
  425. string desc;
  426. if (LongDescription == false) {
  427. desc = Tags.FindS("Description").append("\n");
  428. OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
  429. }
  430. // This lists all the changes to the fields we are going to make.
  431. std::vector<pkgTagSection::Tag> Changes;
  432. Changes.push_back(pkgTagSection::Tag::Rewrite("Size", std::to_string(FileSize)));
  433. for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
  434. {
  435. if (hs->HashType() == "MD5Sum")
  436. Changes.push_back(pkgTagSection::Tag::Rewrite("MD5sum", hs->HashValue()));
  437. else if (hs->HashType() == "Checksum-FileSize")
  438. continue;
  439. else
  440. Changes.push_back(pkgTagSection::Tag::Rewrite(hs->HashType(), hs->HashValue()));
  441. }
  442. Changes.push_back(pkgTagSection::Tag::Rewrite("Filename", NewFileName));
  443. Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", OverItem->Priority));
  444. Changes.push_back(pkgTagSection::Tag::Remove("Status"));
  445. Changes.push_back(pkgTagSection::Tag::Remove("Optional"));
  446. string DescriptionMd5;
  447. if (LongDescription == false) {
  448. MD5Summation descmd5;
  449. descmd5.Add(desc.c_str());
  450. DescriptionMd5 = descmd5.Result().Value();
  451. Changes.push_back(pkgTagSection::Tag::Rewrite("Description-md5", DescriptionMd5));
  452. if (TransWriter != NULL)
  453. TransWriter->DoPackage(Package, desc, DescriptionMd5);
  454. }
  455. // Rewrite the maintainer field if necessary
  456. bool MaintFailed;
  457. string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
  458. if (MaintFailed == true)
  459. {
  460. if (NoOverride == false)
  461. {
  462. NewLine(1);
  463. ioprintf(c1out, _(" %s maintainer is %s not %s\n"),
  464. Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
  465. }
  466. }
  467. if (NewMaint.empty() == false)
  468. Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint));
  469. /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
  470. dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
  471. but dpkg does this append bit. So we do the append bit, at least that way the
  472. status file and package file will remain similar. There are other transforms
  473. but optional is the only legacy one still in use for some lazy reason. */
  474. string OptionalStr = Tags.FindS("Optional");
  475. if (OptionalStr.empty() == false)
  476. {
  477. if (Tags.FindS("Suggests").empty() == false)
  478. OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
  479. Changes.push_back(pkgTagSection::Tag::Rewrite("Suggests", OptionalStr));
  480. }
  481. for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
  482. I != OverItem->FieldOverride.end(); ++I)
  483. Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
  484. // Rewrite and store the fields.
  485. if (Tags.Write(*Output, TFRewritePackageOrder, Changes) == false ||
  486. Output->Write("\n", 1) == false)
  487. return false;
  488. return Db.Finish();
  489. }
  490. /*}}}*/
  491. PackagesWriter::~PackagesWriter() /*{{{*/
  492. {
  493. }
  494. /*}}}*/
  495. // TranslationWriter::TranslationWriter - Constructor /*{{{*/
  496. // ---------------------------------------------------------------------
  497. /* Create a Translation-Master file for this Packages file */
  498. TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
  499. mode_t const &Permissions) : Comp(NULL), Output(NULL)
  500. {
  501. if (File.empty() == true)
  502. return;
  503. Comp = new MultiCompress(File, TransCompress, Permissions);
  504. Output = &Comp->Input;
  505. }
  506. /*}}}*/
  507. // TranslationWriter::DoPackage - Process a single package /*{{{*/
  508. // ---------------------------------------------------------------------
  509. /* Create a Translation-Master file for this Packages file */
  510. bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
  511. string const &MD5)
  512. {
  513. if (Output == NULL)
  514. return true;
  515. // Different archs can include different versions and therefore
  516. // different descriptions - so we need to check for both name and md5.
  517. string const Record = Pkg + ":" + MD5;
  518. if (Included.find(Record) != Included.end())
  519. return true;
  520. std::string out;
  521. strprintf(out, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
  522. Pkg.c_str(), MD5.c_str(), Desc.c_str());
  523. Output->Write(out.c_str(), out.length());
  524. Included.insert(Record);
  525. return true;
  526. }
  527. /*}}}*/
  528. // TranslationWriter::~TranslationWriter - Destructor /*{{{*/
  529. // ---------------------------------------------------------------------
  530. /* */
  531. TranslationWriter::~TranslationWriter()
  532. {
  533. if (Comp != NULL)
  534. delete Comp;
  535. }
  536. /*}}}*/
  537. // SourcesWriter::SourcesWriter - Constructor /*{{{*/
  538. // ---------------------------------------------------------------------
  539. /* */
  540. SourcesWriter::SourcesWriter(FileFd * const GivenOutput, string const &DB, string const &BOverrides,string const &SOverrides,
  541. string const &ExtOverrides) :
  542. FTWScanner(GivenOutput), Db(DB), Stats(Db.Stats)
  543. {
  544. AddPattern("*.dsc");
  545. DeLinkLimit = 0;
  546. Buffer = 0;
  547. BufSize = 0;
  548. // Process the command line options
  549. ConfigToDoHashes(DoHashes, "APT::FTPArchive::Sources");
  550. NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
  551. DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
  552. // Read the override file
  553. if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
  554. return;
  555. else
  556. NoOverride = true;
  557. // WTF?? The logic above: if we can't read binary overrides, don't even try
  558. // reading source overrides. if we can read binary overrides, then say there
  559. // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
  560. if (ExtOverrides.empty() == false)
  561. SOver.ReadExtraOverride(ExtOverrides);
  562. if (SOverrides.empty() == false && FileExists(SOverrides) == true)
  563. SOver.ReadOverride(SOverrides,true);
  564. }
  565. /*}}}*/
  566. // SourcesWriter::DoPackage - Process a single package /*{{{*/
  567. static std::string getDscHash(unsigned int const DoHashes,
  568. Hashes::SupportedHashes const DoIt, pkgTagSection &Tags, char const * const FieldName,
  569. HashString const * const Hash, unsigned long long Size, std::string const &FileName)
  570. {
  571. if ((DoHashes & DoIt) != DoIt || Tags.Exists(FieldName) == false || Hash == NULL)
  572. return "";
  573. std::ostringstream out;
  574. out << "\n " << Hash->HashValue() << " " << std::to_string(Size) << " " << FileName
  575. << "\n " << Tags.FindS(FieldName);
  576. return out.str();
  577. }
  578. bool SourcesWriter::DoPackage(string FileName)
  579. {
  580. // Pull all the data we need form the DB
  581. if (Db.GetFileInfo(FileName,
  582. false, /* DoControl */
  583. false, /* DoContents */
  584. false, /* GenContentsOnly */
  585. true, /* DoSource */
  586. DoHashes, DoAlwaysStat) == false)
  587. {
  588. return false;
  589. }
  590. // we need to perform a "write" here (this is what finish is doing)
  591. // because the call to Db.GetFileInfo() in the loop will change
  592. // the "db cursor"
  593. Db.Finish();
  594. pkgTagSection Tags;
  595. if (Tags.Scan(Db.Dsc.Data.c_str(), Db.Dsc.Data.length()) == false)
  596. return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
  597. if (Tags.Exists("Source") == false)
  598. return _error->Error("Could not find a Source entry in the DSC '%s'",FileName.c_str());
  599. Tags.Trim();
  600. // Lookup the override information, finding first the best priority.
  601. string BestPrio;
  602. string Bins = Tags.FindS("Binary");
  603. char Buffer[Bins.length() + 1];
  604. unique_ptr<Override::Item> OverItem(nullptr);
  605. if (Bins.empty() == false)
  606. {
  607. strcpy(Buffer,Bins.c_str());
  608. // Ignore too-long errors.
  609. char *BinList[400];
  610. TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
  611. // Look at all the binaries
  612. unsigned char BestPrioV = pkgCache::State::Extra;
  613. for (unsigned I = 0; BinList[I] != 0; I++)
  614. {
  615. unique_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
  616. if (Itm.get() == 0)
  617. continue;
  618. unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
  619. if (NewPrioV < BestPrioV || BestPrio.empty() == true)
  620. {
  621. BestPrioV = NewPrioV;
  622. BestPrio = Itm->Priority;
  623. }
  624. if (OverItem.get() == 0)
  625. OverItem = std::move(Itm);
  626. }
  627. }
  628. // If we need to do any rewriting of the header do it now..
  629. if (OverItem.get() == 0)
  630. {
  631. if (NoOverride == false)
  632. {
  633. NewLine(1);
  634. ioprintf(c1out, _(" %s has no override entry\n"), Tags.FindS("Source").c_str());
  635. }
  636. OverItem.reset(new Override::Item);
  637. }
  638. struct stat St;
  639. if (stat(FileName.c_str(), &St) != 0)
  640. return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
  641. unique_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
  642. // const unique_ptr<Override::Item> autoSOverItem(SOverItem);
  643. if (SOverItem.get() == 0)
  644. {
  645. ioprintf(c1out, _(" %s has no source override entry\n"), Tags.FindS("Source").c_str());
  646. SOverItem = unique_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
  647. if (SOverItem.get() == 0)
  648. {
  649. ioprintf(c1out, _(" %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
  650. SOverItem = unique_ptr<Override::Item>(new Override::Item);
  651. *SOverItem = *OverItem;
  652. }
  653. }
  654. // Add the dsc to the files hash list
  655. string const strippedName = flNotDir(FileName);
  656. std::string const Files = getDscHash(DoHashes, Hashes::MD5SUM, Tags, "Files", Db.HashesList.find("MD5Sum"), St.st_size, strippedName);
  657. std::string ChecksumsSha1 = getDscHash(DoHashes, Hashes::SHA1SUM, Tags, "Checksums-Sha1", Db.HashesList.find("SHA1"), St.st_size, strippedName);
  658. std::string ChecksumsSha256 = getDscHash(DoHashes, Hashes::SHA256SUM, Tags, "Checksums-Sha256", Db.HashesList.find("SHA256"), St.st_size, strippedName);
  659. std::string ChecksumsSha512 = getDscHash(DoHashes, Hashes::SHA512SUM, Tags, "Checksums-Sha512", Db.HashesList.find("SHA512"), St.st_size, strippedName);
  660. // Strip the DirStrip prefix from the FileName and add the PathPrefix
  661. string NewFileName;
  662. if (DirStrip.empty() == false &&
  663. FileName.length() > DirStrip.length() &&
  664. stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
  665. NewFileName = string(OriginalPath + DirStrip.length());
  666. else
  667. NewFileName = OriginalPath;
  668. if (PathPrefix.empty() == false)
  669. NewFileName = flCombine(PathPrefix,NewFileName);
  670. string Directory = flNotFile(OriginalPath);
  671. string Package = Tags.FindS("Source");
  672. // Perform operation over all of the files
  673. string ParseJnk;
  674. const char *C = Files.c_str();
  675. char *RealPath = NULL;
  676. for (;isspace(*C); C++);
  677. while (*C != 0)
  678. {
  679. // Parse each of the elements
  680. if (ParseQuoteWord(C,ParseJnk) == false ||
  681. ParseQuoteWord(C,ParseJnk) == false ||
  682. ParseQuoteWord(C,ParseJnk) == false)
  683. return _error->Error("Error parsing file record");
  684. string OriginalPath = Directory + ParseJnk;
  685. // Add missing hashes to source files
  686. if (((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM && !Tags.Exists("Checksums-Sha1")) ||
  687. ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM && !Tags.Exists("Checksums-Sha256")) ||
  688. ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM && !Tags.Exists("Checksums-Sha512")))
  689. {
  690. if (Db.GetFileInfo(OriginalPath,
  691. false, /* DoControl */
  692. false, /* DoContents */
  693. false, /* GenContentsOnly */
  694. false, /* DoSource */
  695. DoHashes,
  696. DoAlwaysStat) == false)
  697. {
  698. return _error->Error("Error getting file info");
  699. }
  700. for (HashStringList::const_iterator hs = Db.HashesList.begin(); hs != Db.HashesList.end(); ++hs)
  701. {
  702. if (hs->HashType() == "MD5Sum" || hs->HashType() == "Checksum-FileSize")
  703. continue;
  704. char const * fieldname;
  705. std::string * out;
  706. if (hs->HashType() == "SHA1")
  707. {
  708. fieldname = "Checksums-Sha1";
  709. out = &ChecksumsSha1;
  710. }
  711. else if (hs->HashType() == "SHA256")
  712. {
  713. fieldname = "Checksums-Sha256";
  714. out = &ChecksumsSha256;
  715. }
  716. else if (hs->HashType() == "SHA512")
  717. {
  718. fieldname = "Checksums-Sha512";
  719. out = &ChecksumsSha512;
  720. }
  721. else
  722. {
  723. _error->Warning("Ignoring unknown Checksumtype %s in SourcesWriter::DoPackages", hs->HashType().c_str());
  724. continue;
  725. }
  726. if (Tags.Exists(fieldname) == true)
  727. continue;
  728. std::ostringstream streamout;
  729. streamout << "\n " << hs->HashValue() << " " << std::to_string(Db.GetFileSize()) << " " << ParseJnk;
  730. out->append(streamout.str());
  731. }
  732. // write back the GetFileInfo() stats data
  733. Db.Finish();
  734. }
  735. // Perform the delinking operation
  736. char Jnk[2];
  737. if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
  738. (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
  739. {
  740. string RP = RealPath;
  741. free(RealPath);
  742. if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
  743. return false;
  744. }
  745. }
  746. Directory = flNotFile(NewFileName);
  747. if (Directory.length() > 2)
  748. Directory.erase(Directory.end()-1);
  749. // This lists all the changes to the fields we are going to make.
  750. // (5 hardcoded + checksums + maintainer + end marker)
  751. std::vector<pkgTagSection::Tag> Changes;
  752. Changes.push_back(pkgTagSection::Tag::Remove("Source"));
  753. Changes.push_back(pkgTagSection::Tag::Rewrite("Package", Package));
  754. if (Files.empty() == false)
  755. Changes.push_back(pkgTagSection::Tag::Rewrite("Files", Files));
  756. else
  757. Changes.push_back(pkgTagSection::Tag::Remove("Files"));
  758. if (ChecksumsSha1.empty() == false)
  759. Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha1", ChecksumsSha1));
  760. else
  761. Changes.push_back(pkgTagSection::Tag::Remove("Checksums-Sha1"));
  762. if (ChecksumsSha256.empty() == false)
  763. Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha256", ChecksumsSha256));
  764. else
  765. Changes.push_back(pkgTagSection::Tag::Remove("Checksums-Sha256"));
  766. if (ChecksumsSha512.empty() == false)
  767. Changes.push_back(pkgTagSection::Tag::Rewrite("Checksums-Sha512", ChecksumsSha512));
  768. else
  769. Changes.push_back(pkgTagSection::Tag::Remove("Checksums-Sha512"));
  770. if (Directory != "./")
  771. Changes.push_back(pkgTagSection::Tag::Rewrite("Directory", Directory));
  772. Changes.push_back(pkgTagSection::Tag::Rewrite("Priority", BestPrio));
  773. Changes.push_back(pkgTagSection::Tag::Remove("Status"));
  774. // Rewrite the maintainer field if necessary
  775. bool MaintFailed;
  776. string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"), MaintFailed);
  777. if (MaintFailed == true)
  778. {
  779. if (NoOverride == false)
  780. {
  781. NewLine(1);
  782. ioprintf(c1out, _(" %s maintainer is %s not %s\n"), Package.c_str(),
  783. Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
  784. }
  785. }
  786. if (NewMaint.empty() == false)
  787. Changes.push_back(pkgTagSection::Tag::Rewrite("Maintainer", NewMaint.c_str()));
  788. for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
  789. I != SOverItem->FieldOverride.end(); ++I)
  790. Changes.push_back(pkgTagSection::Tag::Rewrite(I->first, I->second));
  791. // Rewrite and store the fields.
  792. if (Tags.Write(*Output, TFRewriteSourceOrder, Changes) == false ||
  793. Output->Write("\n", 1) == false)
  794. return false;
  795. Stats.Packages++;
  796. return true;
  797. }
  798. /*}}}*/
  799. // ContentsWriter::ContentsWriter - Constructor /*{{{*/
  800. // ---------------------------------------------------------------------
  801. /* */
  802. ContentsWriter::ContentsWriter(FileFd * const GivenOutput, string const &DB,
  803. string const &Arch, bool const IncludeArchAll) :
  804. FTWScanner(GivenOutput, Arch, IncludeArchAll), Db(DB), Stats(Db.Stats)
  805. {
  806. SetExts(".deb");
  807. }
  808. /*}}}*/
  809. // ContentsWriter::DoPackage - Process a single package /*{{{*/
  810. // ---------------------------------------------------------------------
  811. /* If Package is the empty string the control record will be parsed to
  812. determine what the package name is. */
  813. bool ContentsWriter::DoPackage(string FileName, string Package)
  814. {
  815. if (!Db.GetFileInfo(FileName,
  816. Package.empty(), /* DoControl */
  817. true, /* DoContents */
  818. false, /* GenContentsOnly */
  819. false, /* DoSource */
  820. 0, /* DoHashes */
  821. false /* checkMtime */))
  822. {
  823. return false;
  824. }
  825. // Parse the package name
  826. if (Package.empty() == true)
  827. {
  828. Package = Db.Control.Section.FindS("Package");
  829. }
  830. Db.Contents.Add(Gen,Package);
  831. return Db.Finish();
  832. }
  833. /*}}}*/
  834. // ContentsWriter::ReadFromPkgs - Read from a packages file /*{{{*/
  835. // ---------------------------------------------------------------------
  836. /* */
  837. bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
  838. {
  839. MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
  840. if (_error->PendingError() == true)
  841. return false;
  842. // Open the package file
  843. FileFd Fd;
  844. if (Pkgs.OpenOld(Fd) == false)
  845. return false;
  846. pkgTagFile Tags(&Fd);
  847. if (_error->PendingError() == true)
  848. return false;
  849. // Parse.
  850. pkgTagSection Section;
  851. while (Tags.Step(Section) == true)
  852. {
  853. string File = flCombine(Prefix,Section.FindS("FileName"));
  854. string Package = Section.FindS("Section");
  855. if (Package.empty() == false && Package.end()[-1] != '/')
  856. {
  857. Package += '/';
  858. Package += Section.FindS("Package");
  859. }
  860. else
  861. Package += Section.FindS("Package");
  862. DoPackage(File,Package);
  863. if (_error->empty() == false)
  864. {
  865. _error->Error("Errors apply to file '%s'",File.c_str());
  866. _error->DumpErrors();
  867. }
  868. }
  869. // Tidy the compressor
  870. Fd.Close();
  871. return true;
  872. }
  873. /*}}}*/
  874. // ReleaseWriter::ReleaseWriter - Constructor /*{{{*/
  875. // ---------------------------------------------------------------------
  876. /* */
  877. static std::string formatUTCDateTime(time_t const now)
  878. {
  879. bool const NumericTimezone = _config->FindB("APT::FTPArchive::Release::NumericTimezone", true);
  880. // TimeRFC1123 uses GMT to satisfy HTTP/1.1
  881. std::string datetime = TimeRFC1123(now, NumericTimezone);
  882. if (NumericTimezone == false)
  883. {
  884. auto const lastspace = datetime.rfind(' ');
  885. if (likely(lastspace != std::string::npos))
  886. datetime.replace(lastspace + 1, 3, "UTC");
  887. }
  888. return datetime;
  889. }
  890. ReleaseWriter::ReleaseWriter(FileFd * const GivenOutput, string const &/*DB*/) : FTWScanner(GivenOutput)
  891. {
  892. if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
  893. {
  894. AddPattern("Packages");
  895. AddPattern("Packages.*");
  896. AddPattern("Translation-*");
  897. AddPattern("Sources");
  898. AddPattern("Sources.*");
  899. AddPattern("Release");
  900. AddPattern("Contents-*");
  901. AddPattern("Index");
  902. AddPattern("Index.*");
  903. AddPattern("icons-*.tar");
  904. AddPattern("icons-*.tar.*");
  905. AddPattern("Components-*.yml");
  906. AddPattern("Components-*.yml.*");
  907. AddPattern("md5sum.txt");
  908. }
  909. AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
  910. time_t const now = time(NULL);
  911. time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
  912. map<string,bool> BoolFields;
  913. map<string,string> Fields;
  914. Fields["Origin"] = "";
  915. Fields["Label"] = "";
  916. Fields["Suite"] = "";
  917. Fields["Version"] = "";
  918. Fields["Codename"] = "";
  919. Fields["Date"] = formatUTCDateTime(now);
  920. if (validuntil != now)
  921. Fields["Valid-Until"] = formatUTCDateTime(validuntil);
  922. Fields["Architectures"] = "";
  923. Fields["Components"] = "";
  924. Fields["Description"] = "";
  925. Fields["Signed-By"] = "";
  926. BoolFields["Acquire-By-Hash"] = _config->FindB("APT::FTPArchive::DoByHash", false);
  927. BoolFields["NotAutomatic"] = false;
  928. BoolFields["ButAutomaticUpgrades"] = false;
  929. // Read configuration for string fields, but don't output them
  930. for (auto &&I : Fields)
  931. {
  932. string Config = string("APT::FTPArchive::Release::") + I.first;
  933. I.second = _config->Find(Config, I.second);
  934. }
  935. // Read configuration for bool fields, and add them to Fields if true
  936. for (auto &&I : BoolFields)
  937. {
  938. string Config = string("APT::FTPArchive::Release::") + I.first;
  939. I.second = _config->FindB(Config, I.second);
  940. if (I.second)
  941. Fields[I.first] = "yes";
  942. }
  943. // All configuration read and stored in Fields; output
  944. for (auto &&I : Fields)
  945. {
  946. if (I.second.empty())
  947. continue;
  948. std::string const out = I.first + ": " + I.second + "\n";
  949. Output->Write(out.c_str(), out.length());
  950. }
  951. ConfigToDoHashes(DoHashes, "APT::FTPArchive::Release");
  952. }
  953. /*}}}*/
  954. // ReleaseWriter::DoPackage - Process a single package /*{{{*/
  955. // ---------------------------------------------------------------------
  956. bool ReleaseWriter::DoPackage(string FileName)
  957. {
  958. // Strip the DirStrip prefix from the FileName and add the PathPrefix
  959. string NewFileName;
  960. if (DirStrip.empty() == false &&
  961. FileName.length() > DirStrip.length() &&
  962. stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
  963. DirStrip.begin(),DirStrip.end()) == 0)
  964. {
  965. NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
  966. while (NewFileName[0] == '/')
  967. NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
  968. }
  969. else
  970. NewFileName = FileName;
  971. if (PathPrefix.empty() == false)
  972. NewFileName = flCombine(PathPrefix,NewFileName);
  973. FileFd fd(FileName, FileFd::ReadOnly);
  974. if (!fd.IsOpen())
  975. {
  976. return false;
  977. }
  978. CheckSums[NewFileName].size = fd.Size();
  979. Hashes hs(DoHashes);
  980. hs.AddFD(fd);
  981. CheckSums[NewFileName].Hashes = hs.GetHashStringList();
  982. fd.Close();
  983. // FIXME: wrong layer in the code(?)
  984. // FIXME2: symlink instead of create a copy
  985. if (_config->FindB("APT::FTPArchive::DoByHash", false) == true)
  986. {
  987. std::string Input = FileName;
  988. HashStringList hsl = hs.GetHashStringList();
  989. for(HashStringList::const_iterator h = hsl.begin();
  990. h != hsl.end(); ++h)
  991. {
  992. if (!h->usable())
  993. continue;
  994. if (flNotDir(FileName) == "Release" || flNotDir(FileName) == "InRelease")
  995. continue;
  996. std::string ByHashOutputFile = GenByHashFilename(Input, *h);
  997. std::string ByHashOutputDir = flNotFile(ByHashOutputFile);
  998. if(!CreateDirectory(flNotFile(Input), ByHashOutputDir))
  999. return _error->Warning("can not create dir %s", flNotFile(ByHashOutputFile).c_str());
  1000. // write new hashes
  1001. FileFd In(Input, FileFd::ReadOnly);
  1002. FileFd Out(ByHashOutputFile, FileFd::WriteEmpty);
  1003. if(!CopyFile(In, Out))
  1004. return _error->Warning("failed to copy %s %s", Input.c_str(), ByHashOutputFile.c_str());
  1005. }
  1006. }
  1007. return true;
  1008. }
  1009. /*}}}*/
  1010. // ReleaseWriter::Finish - Output the checksums /*{{{*/
  1011. // ---------------------------------------------------------------------
  1012. static void printChecksumTypeRecord(FileFd &Output, char const * const Type, map<string, ReleaseWriter::CheckSum> const &CheckSums)
  1013. {
  1014. {
  1015. std::string out;
  1016. strprintf(out, "%s:\n", Type);
  1017. Output.Write(out.c_str(), out.length());
  1018. }
  1019. for(map<string,ReleaseWriter::CheckSum>::const_iterator I = CheckSums.begin();
  1020. I != CheckSums.end(); ++I)
  1021. {
  1022. HashString const * const hs = I->second.Hashes.find(Type);
  1023. if (hs == NULL)
  1024. continue;
  1025. std::string out;
  1026. strprintf(out, " %s %16llu %s\n",
  1027. hs->HashValue().c_str(),
  1028. (*I).second.size,
  1029. (*I).first.c_str());
  1030. Output.Write(out.c_str(), out.length());
  1031. }
  1032. }
  1033. void ReleaseWriter::Finish()
  1034. {
  1035. if ((DoHashes & Hashes::MD5SUM) == Hashes::MD5SUM)
  1036. printChecksumTypeRecord(*Output, "MD5Sum", CheckSums);
  1037. if ((DoHashes & Hashes::SHA1SUM) == Hashes::SHA1SUM)
  1038. printChecksumTypeRecord(*Output, "SHA1", CheckSums);
  1039. if ((DoHashes & Hashes::SHA256SUM) == Hashes::SHA256SUM)
  1040. printChecksumTypeRecord(*Output, "SHA256", CheckSums);
  1041. if ((DoHashes & Hashes::SHA512SUM) == Hashes::SHA512SUM)
  1042. printChecksumTypeRecord(*Output, "SHA512", CheckSums);
  1043. // go by-hash cleanup
  1044. map<string,ReleaseWriter::CheckSum>::const_iterator prev = CheckSums.begin();
  1045. if (_config->FindB("APT::FTPArchive::DoByHash", false) == true)
  1046. {
  1047. for(map<string,ReleaseWriter::CheckSum>::const_iterator I = CheckSums.begin();
  1048. I != CheckSums.end(); ++I)
  1049. {
  1050. if (I->first == "Release" || I->first == "InRelease")
  1051. continue;
  1052. // keep iterating until we find a new subdir
  1053. if(flNotFile(I->first) == flNotFile(prev->first))
  1054. continue;
  1055. // clean that subdir up
  1056. int keepFiles = _config->FindI("APT::FTPArchive::By-Hash-Keep", 3);
  1057. // calculate how many compressors are used (the amount of files
  1058. // in that subdir generated for this run)
  1059. keepFiles *= std::distance(prev, I);
  1060. prev = I;
  1061. HashStringList hsl = prev->second.Hashes;
  1062. for(HashStringList::const_iterator h = hsl.begin();
  1063. h != hsl.end(); ++h)
  1064. {
  1065. if (!h->usable())
  1066. continue;
  1067. std::string RealFilename = DirStrip+"/"+prev->first;
  1068. std::string ByHashOutputFile = GenByHashFilename(RealFilename, *h);
  1069. DeleteAllButMostRecent(flNotFile(ByHashOutputFile), keepFiles);
  1070. }
  1071. }
  1072. }
  1073. }