You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

537 lines
18 KiB

  1. // -*- mode: cpp; mode: fold -*-
  2. // Description /*{{{*/
  3. // $Id: sourcelist.cc,v 1.3 2002/08/15 20:51:37 niemeyer Exp $
  4. /* ######################################################################
  5. List of Sources
  6. ##################################################################### */
  7. /*}}}*/
  8. // Include Files /*{{{*/
  9. #include<config.h>
  10. #include <apt-pkg/sourcelist.h>
  11. #include <apt-pkg/error.h>
  12. #include <apt-pkg/fileutl.h>
  13. #include <apt-pkg/strutl.h>
  14. #include <apt-pkg/configuration.h>
  15. #include <apt-pkg/metaindex.h>
  16. #include <apt-pkg/indexfile.h>
  17. #include <apt-pkg/tagfile.h>
  18. #include <apt-pkg/pkgcache.h>
  19. #include <apt-pkg/cacheiterators.h>
  20. #include <ctype.h>
  21. #include <stddef.h>
  22. #include <time.h>
  23. #include <cstring>
  24. #include <map>
  25. #include <string>
  26. #include <vector>
  27. #include <fstream>
  28. #include <algorithm>
  29. #include <apti18n.h>
  30. /*}}}*/
  31. using namespace std;
  32. // Global list of Items supported
  33. static pkgSourceList::Type *ItmList[10];
  34. pkgSourceList::Type **pkgSourceList::Type::GlobalList = ItmList;
  35. unsigned long pkgSourceList::Type::GlobalListLen = 0;
  36. // Type::Type - Constructor /*{{{*/
  37. // ---------------------------------------------------------------------
  38. /* Link this to the global list of items*/
  39. pkgSourceList::Type::Type(char const * const pName, char const * const pLabel) : Name(pName), Label(pLabel)
  40. {
  41. ItmList[GlobalListLen] = this;
  42. ++GlobalListLen;
  43. }
  44. pkgSourceList::Type::~Type() {}
  45. /*}}}*/
  46. // Type::GetType - Get a specific meta for a given type /*{{{*/
  47. // ---------------------------------------------------------------------
  48. /* */
  49. pkgSourceList::Type *pkgSourceList::Type::GetType(const char *Type)
  50. {
  51. for (unsigned I = 0; I != GlobalListLen; ++I)
  52. if (strcmp(GlobalList[I]->Name,Type) == 0)
  53. return GlobalList[I];
  54. return 0;
  55. }
  56. /*}}}*/
  57. // Type::FixupURI - Normalize the URI and check it.. /*{{{*/
  58. // ---------------------------------------------------------------------
  59. /* */
  60. bool pkgSourceList::Type::FixupURI(string &URI) const
  61. {
  62. if (URI.empty() == true)
  63. return false;
  64. if (URI.find(':') == string::npos)
  65. return false;
  66. URI = SubstVar(URI,"$(ARCH)",_config->Find("APT::Architecture"));
  67. // Make sure that the URI is / postfixed
  68. if (URI[URI.size() - 1] != '/')
  69. URI += '/';
  70. return true;
  71. }
  72. /*}}}*/
  73. bool pkgSourceList::Type::ParseStanza(vector<metaIndex *> &List, /*{{{*/
  74. pkgTagSection &Tags,
  75. unsigned int const i,
  76. FileFd &Fd)
  77. {
  78. map<string, string> Options;
  79. string Enabled = Tags.FindS("Enabled");
  80. if (Enabled.empty() == false && StringToBool(Enabled) == false)
  81. return true;
  82. std::map<char const * const, std::pair<char const * const, bool> > mapping;
  83. #define APT_PLUSMINUS(X, Y) \
  84. mapping.insert(std::make_pair(X, std::make_pair(Y, true))); \
  85. mapping.insert(std::make_pair(X "Add", std::make_pair(Y "+", true))); \
  86. mapping.insert(std::make_pair(X "Remove", std::make_pair(Y "-", true)))
  87. APT_PLUSMINUS("Architectures", "arch");
  88. APT_PLUSMINUS("Languages", "lang");
  89. APT_PLUSMINUS("Targets", "target");
  90. #undef APT_PLUSMINUS
  91. mapping.insert(std::make_pair("Trusted", std::make_pair("trusted", false)));
  92. mapping.insert(std::make_pair("Check-Valid-Until", std::make_pair("check-valid-until", false)));
  93. mapping.insert(std::make_pair("Valid-Until-Min", std::make_pair("valid-until-min", false)));
  94. mapping.insert(std::make_pair("Valid-Until-Max", std::make_pair("valid-until-max", false)));
  95. mapping.insert(std::make_pair("Signed-By", std::make_pair("signed-by", false)));
  96. for (std::map<char const * const, std::pair<char const * const, bool> >::const_iterator m = mapping.begin(); m != mapping.end(); ++m)
  97. if (Tags.Exists(m->first))
  98. {
  99. std::string option = Tags.FindS(m->first);
  100. // for deb822 the " " is the delimiter, but the backend expects ","
  101. if (m->second.second == true)
  102. std::replace(option.begin(), option.end(), ' ', ',');
  103. Options[m->second.first] = option;
  104. }
  105. // now create one item per suite/section
  106. string Suite = Tags.FindS("Suites");
  107. Suite = SubstVar(Suite,"$(ARCH)",_config->Find("APT::Architecture"));
  108. string const Component = Tags.FindS("Components");
  109. string const URIS = Tags.FindS("URIs");
  110. std::vector<std::string> const list_uris = VectorizeString(URIS, ' ');
  111. std::vector<std::string> const list_suite = VectorizeString(Suite, ' ');
  112. std::vector<std::string> const list_comp = VectorizeString(Component, ' ');
  113. if (list_uris.empty())
  114. // TRANSLATOR: %u is a line number, the first %s is a filename of a file with the extension "second %s" and the third %s is a unique identifier for bugreports
  115. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI");
  116. for (std::vector<std::string>::const_iterator U = list_uris.begin();
  117. U != list_uris.end(); ++U)
  118. {
  119. std::string URI = *U;
  120. if (U->empty() || FixupURI(URI) == false)
  121. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "URI parse");
  122. if (list_suite.empty())
  123. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Suite");
  124. for (std::vector<std::string>::const_iterator S = list_suite.begin();
  125. S != list_suite.end(); ++S)
  126. {
  127. if (S->empty() == false && (*S)[S->size() - 1] == '/')
  128. {
  129. if (list_comp.empty() == false)
  130. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "absolute Suite Component");
  131. if (CreateItem(List, URI, *S, "", Options) == false)
  132. return false;
  133. }
  134. else
  135. {
  136. if (list_comp.empty())
  137. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), i, "sources", Fd.Name().c_str(), "Component");
  138. for (std::vector<std::string>::const_iterator C = list_comp.begin();
  139. C != list_comp.end(); ++C)
  140. {
  141. if (CreateItem(List, URI, *S, *C, Options) == false)
  142. {
  143. return false;
  144. }
  145. }
  146. }
  147. }
  148. }
  149. return true;
  150. }
  151. /*}}}*/
  152. // Type::ParseLine - Parse a single line /*{{{*/
  153. // ---------------------------------------------------------------------
  154. /* This is a generic one that is the 'usual' format for sources.list
  155. Weird types may override this. */
  156. bool pkgSourceList::Type::ParseLine(vector<metaIndex *> &List,
  157. const char *Buffer,
  158. unsigned int const CurLine,
  159. string const &File) const
  160. {
  161. for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
  162. // Parse option field if it exists
  163. // e.g.: [ option1=value1 option2=value2 ]
  164. map<string, string> Options;
  165. if (Buffer != 0 && Buffer[0] == '[')
  166. {
  167. ++Buffer; // ignore the [
  168. for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
  169. while (*Buffer != ']')
  170. {
  171. // get one option, e.g. option1=value1
  172. string option;
  173. if (ParseQuoteWord(Buffer,option) == false)
  174. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] unparseable");
  175. if (option.length() < 3)
  176. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] too short");
  177. // accept options even if the last has no space before the ]-end marker
  178. if (option.at(option.length()-1) == ']')
  179. {
  180. for (; *Buffer != ']'; --Buffer);
  181. option.resize(option.length()-1);
  182. }
  183. size_t const needle = option.find('=');
  184. if (needle == string::npos)
  185. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] not assignment");
  186. string const key = string(option, 0, needle);
  187. string const value = string(option, needle + 1, option.length());
  188. if (key.empty() == true)
  189. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no key");
  190. if (value.empty() == true)
  191. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "[option] no value");
  192. Options[key] = value;
  193. }
  194. ++Buffer; // ignore the ]
  195. for (;Buffer != 0 && isspace(*Buffer); ++Buffer); // Skip whitespaces
  196. }
  197. string URI;
  198. string Dist;
  199. string Section;
  200. if (ParseQuoteWord(Buffer,URI) == false)
  201. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI");
  202. if (ParseQuoteWord(Buffer,Dist) == false)
  203. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Suite");
  204. if (FixupURI(URI) == false)
  205. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "URI parse");
  206. // Check for an absolute dists specification.
  207. if (Dist.empty() == false && Dist[Dist.size() - 1] == '/')
  208. {
  209. if (ParseQuoteWord(Buffer,Section) == true)
  210. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "absolute Suite Component");
  211. Dist = SubstVar(Dist,"$(ARCH)",_config->Find("APT::Architecture"));
  212. return CreateItem(List, URI, Dist, Section, Options);
  213. }
  214. // Grab the rest of the dists
  215. if (ParseQuoteWord(Buffer,Section) == false)
  216. return _error->Error(_("Malformed entry %u in %s file %s (%s)"), CurLine, "list", File.c_str(), "Component");
  217. do
  218. {
  219. if (CreateItem(List, URI, Dist, Section, Options) == false)
  220. return false;
  221. }
  222. while (ParseQuoteWord(Buffer,Section) == true);
  223. return true;
  224. }
  225. /*}}}*/
  226. // SourceList::pkgSourceList - Constructors /*{{{*/
  227. // ---------------------------------------------------------------------
  228. /* */
  229. pkgSourceList::pkgSourceList() : d(NULL)
  230. {
  231. }
  232. /*}}}*/
  233. // SourceList::~pkgSourceList - Destructor /*{{{*/
  234. // ---------------------------------------------------------------------
  235. /* */
  236. pkgSourceList::~pkgSourceList()
  237. {
  238. for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
  239. delete *I;
  240. SrcList.clear();
  241. for (pkgIndexFile * const File : VolatileFiles)
  242. delete File;
  243. VolatileFiles.clear();
  244. }
  245. /*}}}*/
  246. // SourceList::ReadMainList - Read the main source list from etc /*{{{*/
  247. // ---------------------------------------------------------------------
  248. /* */
  249. bool pkgSourceList::ReadMainList()
  250. {
  251. // CNC:2003-03-03 - Multiple sources list support.
  252. bool Res = true;
  253. #if 0
  254. Res = ReadVendors();
  255. if (Res == false)
  256. return false;
  257. #endif
  258. Reset();
  259. // CNC:2003-11-28 - Entries in sources.list have priority over
  260. // entries in sources.list.d.
  261. string Main = _config->FindFile("Dir::Etc::sourcelist");
  262. string Parts = _config->FindDir("Dir::Etc::sourceparts");
  263. if (RealFileExists(Main) == true)
  264. Res &= ReadAppend(Main);
  265. else if (DirectoryExists(Parts) == false)
  266. // Only warn if there are no sources.list.d.
  267. _error->WarningE("DirectoryExists", _("Unable to read %s"), Parts.c_str());
  268. if (DirectoryExists(Parts) == true)
  269. Res &= ReadSourceDir(Parts);
  270. else if (RealFileExists(Main) == false)
  271. // Only warn if there is no sources.list file.
  272. _error->WarningE("RealFileExists", _("Unable to read %s"), Main.c_str());
  273. return Res;
  274. }
  275. /*}}}*/
  276. // SourceList::Reset - Clear the sourcelist contents /*{{{*/
  277. // ---------------------------------------------------------------------
  278. /* */
  279. void pkgSourceList::Reset()
  280. {
  281. for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
  282. delete *I;
  283. SrcList.erase(SrcList.begin(),SrcList.end());
  284. }
  285. /*}}}*/
  286. // SourceList::Read - Parse the sourcelist file /*{{{*/
  287. // ---------------------------------------------------------------------
  288. /* */
  289. bool pkgSourceList::Read(string const &File)
  290. {
  291. Reset();
  292. return ReadAppend(File);
  293. }
  294. /*}}}*/
  295. // SourceList::ReadAppend - Parse a sourcelist file /*{{{*/
  296. // ---------------------------------------------------------------------
  297. /* */
  298. bool pkgSourceList::ReadAppend(string const &File)
  299. {
  300. if (flExtension(File) == "sources")
  301. return ParseFileDeb822(File);
  302. else
  303. return ParseFileOldStyle(File);
  304. }
  305. /*}}}*/
  306. // SourceList::ReadFileOldStyle - Read Traditional style sources.list /*{{{*/
  307. // ---------------------------------------------------------------------
  308. /* */
  309. bool pkgSourceList::ParseFileOldStyle(std::string const &File)
  310. {
  311. // Open the stream for reading
  312. ifstream F(File.c_str(),ios::in /*| ios::nocreate*/);
  313. if (F.fail() == true)
  314. return _error->Errno("ifstream::ifstream",_("Opening %s"),File.c_str());
  315. std::string Buffer;
  316. for (unsigned int CurLine = 1; std::getline(F, Buffer); ++CurLine)
  317. {
  318. // remove comments
  319. size_t curpos = 0;
  320. while ((curpos = Buffer.find('#', curpos)) != std::string::npos)
  321. {
  322. size_t const openbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, '[');
  323. size_t const closedbrackets = std::count(Buffer.begin(), Buffer.begin() + curpos, ']');
  324. if (openbrackets > closedbrackets)
  325. {
  326. // a # in an option, unlikely, but oh well, it was supported so stick to it
  327. ++curpos;
  328. continue;
  329. }
  330. Buffer.erase(curpos);
  331. break;
  332. }
  333. // remove spaces before/after
  334. curpos = Buffer.find_first_not_of(" \t\r");
  335. if (curpos != 0)
  336. Buffer.erase(0, curpos);
  337. curpos = Buffer.find_last_not_of(" \t\r");
  338. if (curpos != std::string::npos)
  339. Buffer.erase(curpos + 1);
  340. if (Buffer.empty())
  341. continue;
  342. // Grok it
  343. std::string const LineType = Buffer.substr(0, Buffer.find(' '));
  344. if (LineType.empty() || LineType == Buffer)
  345. return _error->Error(_("Malformed line %u in source list %s (type)"),CurLine,File.c_str());
  346. Type *Parse = Type::GetType(LineType.c_str());
  347. if (Parse == 0)
  348. return _error->Error(_("Type '%s' is not known on line %u in source list %s"),LineType.c_str(),CurLine,File.c_str());
  349. if (Parse->ParseLine(SrcList, Buffer.c_str() + LineType.length(), CurLine, File) == false)
  350. return false;
  351. }
  352. return true;
  353. }
  354. /*}}}*/
  355. // SourceList::ParseFileDeb822 - Parse deb822 style sources.list /*{{{*/
  356. // ---------------------------------------------------------------------
  357. /* Returns: the number of stanzas parsed*/
  358. bool pkgSourceList::ParseFileDeb822(string const &File)
  359. {
  360. pkgUserTagSection Tags;
  361. unsigned int i = 1;
  362. // see if we can read the file
  363. FileFd Fd(File, FileFd::ReadOnly);
  364. pkgTagFile Sources(&Fd);
  365. if (_error->PendingError() == true)
  366. return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
  367. // read step by step
  368. while (Sources.Step(Tags) == true)
  369. {
  370. if(Tags.Exists("Types") == false)
  371. return _error->Error(_("Malformed stanza %u in source list %s (type)"),i,File.c_str());
  372. string const types = Tags.FindS("Types");
  373. std::vector<std::string> const list_types = VectorizeString(types, ' ');
  374. for (std::vector<std::string>::const_iterator I = list_types.begin();
  375. I != list_types.end(); ++I)
  376. {
  377. Type *Parse = Type::GetType((*I).c_str());
  378. if (Parse == 0)
  379. {
  380. _error->Error(_("Type '%s' is not known on stanza %u in source list %s"), (*I).c_str(),i,Fd.Name().c_str());
  381. return false;
  382. }
  383. if (!Parse->ParseStanza(SrcList, Tags, i, Fd))
  384. return false;
  385. ++i;
  386. }
  387. }
  388. return true;
  389. }
  390. /*}}}*/
  391. // SourceList::FindIndex - Get the index associated with a file /*{{{*/
  392. // ---------------------------------------------------------------------
  393. /* */
  394. bool pkgSourceList::FindIndex(pkgCache::PkgFileIterator File,
  395. pkgIndexFile *&Found) const
  396. {
  397. for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
  398. {
  399. vector<pkgIndexFile *> *Indexes = (*I)->GetIndexFiles();
  400. for (vector<pkgIndexFile *>::const_iterator J = Indexes->begin();
  401. J != Indexes->end(); ++J)
  402. {
  403. if ((*J)->FindInCache(*File.Cache()) == File)
  404. {
  405. Found = (*J);
  406. return true;
  407. }
  408. }
  409. }
  410. for (vector<pkgIndexFile *>::const_iterator J = VolatileFiles.begin();
  411. J != VolatileFiles.end(); ++J)
  412. {
  413. if ((*J)->FindInCache(*File.Cache()) == File)
  414. {
  415. Found = (*J);
  416. return true;
  417. }
  418. }
  419. return false;
  420. }
  421. /*}}}*/
  422. // SourceList::GetIndexes - Load the index files into the downloader /*{{{*/
  423. // ---------------------------------------------------------------------
  424. /* */
  425. bool pkgSourceList::GetIndexes(pkgAcquire *Owner, bool GetAll) const
  426. {
  427. for (const_iterator I = SrcList.begin(); I != SrcList.end(); ++I)
  428. if ((*I)->GetIndexes(Owner,GetAll) == false)
  429. return false;
  430. return true;
  431. }
  432. /*}}}*/
  433. // CNC:2003-03-03 - By Anton V. Denisov <avd@altlinux.org>.
  434. // SourceList::ReadSourceDir - Read a directory with sources files
  435. // Based on ReadConfigDir() /*{{{*/
  436. // ---------------------------------------------------------------------
  437. /* */
  438. bool pkgSourceList::ReadSourceDir(string const &Dir)
  439. {
  440. std::vector<std::string> ext;
  441. ext.push_back("list");
  442. ext.push_back("sources");
  443. std::vector<std::string> const List = GetListOfFilesInDir(Dir, ext, true);
  444. // Read the files
  445. for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
  446. if (ReadAppend(*I) == false)
  447. return false;
  448. return true;
  449. }
  450. /*}}}*/
  451. // GetLastModified() /*{{{*/
  452. // ---------------------------------------------------------------------
  453. /* */
  454. time_t pkgSourceList::GetLastModifiedTime()
  455. {
  456. vector<string> List;
  457. string Main = _config->FindFile("Dir::Etc::sourcelist");
  458. string Parts = _config->FindDir("Dir::Etc::sourceparts");
  459. // go over the parts
  460. if (DirectoryExists(Parts) == true)
  461. List = GetListOfFilesInDir(Parts, "list", true);
  462. // calculate the time
  463. time_t mtime_sources = GetModificationTime(Main);
  464. for (vector<string>::const_iterator I = List.begin(); I != List.end(); ++I)
  465. mtime_sources = std::max(mtime_sources, GetModificationTime(*I));
  466. return mtime_sources;
  467. }
  468. /*}}}*/
  469. std::vector<pkgIndexFile*> pkgSourceList::GetVolatileFiles() const /*{{{*/
  470. {
  471. return VolatileFiles;
  472. }
  473. /*}}}*/
  474. void pkgSourceList::AddVolatileFile(pkgIndexFile * const File) /*{{{*/
  475. {
  476. if (File != NULL)
  477. VolatileFiles.push_back(File);
  478. }
  479. /*}}}*/