Devuan deployment of britney2
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

utils.py 34 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874
  1. # -*- coding: utf-8 -*-
  2. # Refactored parts from britney.py, which is/was:
  3. # Copyright (C) 2001-2008 Anthony Towns <ajt@debian.org>
  4. # Andreas Barth <aba@debian.org>
  5. # Fabio Tranchitella <kobold@debian.org>
  6. # Copyright (C) 2010-2012 Adam D. Barratt <adsb@debian.org>
  7. # Copyright (C) 2012 Niels Thykier <niels@thykier.net>
  8. #
  9. # New portions
  10. # Copyright (C) 2013 Adam D. Barratt <adsb@debian.org>
  11. # This program is free software; you can redistribute it and/or modify
  12. # it under the terms of the GNU General Public License as published by
  13. # the Free Software Foundation; either version 2 of the License, or
  14. # (at your option) any later version.
  15. # This program is distributed in the hope that it will be useful,
  16. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18. # GNU General Public License for more details.
  19. import apt_pkg
  20. import errno
  21. import logging
  22. import os
  23. import sys
  24. import time
  25. from collections import defaultdict
  26. from datetime import datetime
  27. from functools import partial
  28. from itertools import filterfalse
  29. import yaml
  30. from britney2 import SourcePackage
  31. from britney2.consts import (VERSION, PROVIDES, DEPENDS, CONFLICTS,
  32. ARCHITECTURE, SECTION,
  33. SOURCE, MAINTAINER, MULTIARCH,
  34. ESSENTIAL)
  35. from britney2.policies import PolicyVerdict
  36. class MigrationConstraintException(Exception):
  37. pass
  38. def ifilter_except(container, iterable=None):
  39. """Filter out elements in container
  40. If given an iterable it returns a filtered iterator, otherwise it
  41. returns a function to generate filtered iterators. The latter is
  42. useful if the same filter has to be (re-)used on multiple
  43. iterators that are not known on beforehand.
  44. """
  45. if iterable is not None:
  46. return filterfalse(container.__contains__, iterable)
  47. return partial(filterfalse, container.__contains__)
  48. def ifilter_only(container, iterable=None):
  49. """Filter out elements in which are not in container
  50. If given an iterable it returns a filtered iterator, otherwise it
  51. returns a function to generate filtered iterators. The latter is
  52. useful if the same filter has to be (re-)used on multiple
  53. iterators that are not known on beforehand.
  54. """
  55. if iterable is not None:
  56. return filter(container.__contains__, iterable)
  57. return partial(filter, container.__contains__)
  58. # iter_except is from the "itertools" recipe
  59. def iter_except(func, exception, first=None): # pragma: no cover - itertools recipe function
  60. """ Call a function repeatedly until an exception is raised.
  61. Converts a call-until-exception interface to an iterator interface.
  62. Like __builtin__.iter(func, sentinel) but uses an exception instead
  63. of a sentinel to end the loop.
  64. Examples:
  65. bsddbiter = iter_except(db.next, bsddb.error, db.first)
  66. heapiter = iter_except(functools.partial(heappop, h), IndexError)
  67. dictiter = iter_except(d.popitem, KeyError)
  68. dequeiter = iter_except(d.popleft, IndexError)
  69. queueiter = iter_except(q.get_nowait, Queue.Empty)
  70. setiter = iter_except(s.pop, KeyError)
  71. """
  72. try:
  73. if first is not None:
  74. yield first()
  75. while 1:
  76. yield func()
  77. except exception:
  78. pass
  79. def log_and_format_old_libraries(logger, libs):
  80. """Format and log old libraries in a table (no header)"""
  81. libraries = {}
  82. for i in libs:
  83. pkg = i.package
  84. if pkg in libraries:
  85. libraries[pkg].append(i.architecture)
  86. else:
  87. libraries[pkg] = [i.architecture]
  88. for lib in sorted(libraries):
  89. logger.info(" %s: %s", lib, " ".join(libraries[lib]))
  90. def compute_reverse_tree(pkg_universe, affected):
  91. """Calculate the full dependency tree for a set of packages
  92. This method returns the full dependency tree for a given set of
  93. packages. The first argument is an instance of the BinaryPackageUniverse
  94. and the second argument are a set of BinaryPackageId.
  95. The set of affected packages will be updated in place and must
  96. therefore be mutable.
  97. """
  98. remain = list(affected)
  99. while remain:
  100. pkg_id = remain.pop()
  101. new_pkg_ids = pkg_universe.reverse_dependencies_of(pkg_id) - affected
  102. affected.update(new_pkg_ids)
  103. remain.extend(new_pkg_ids)
  104. return None
  105. def write_nuninst(filename, nuninst):
  106. """Write the non-installable report
  107. Write the non-installable report derived from "nuninst" to the
  108. file denoted by "filename".
  109. """
  110. with open(filename, 'w', encoding='utf-8') as f:
  111. # Having two fields with (almost) identical dates seems a bit
  112. # redundant.
  113. f.write("Built on: " + time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())) + "\n")
  114. f.write("Last update: " + time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())) + "\n\n")
  115. for k in nuninst:
  116. f.write("%s: %s\n" % (k, " ".join(nuninst[k])))
  117. def read_nuninst(filename, architectures):
  118. """Read the non-installable report
  119. Read the non-installable report from the file denoted by
  120. "filename" and return it. Only architectures in "architectures"
  121. will be included in the report.
  122. """
  123. nuninst = {}
  124. with open(filename, encoding='ascii') as f:
  125. for r in f:
  126. if ":" not in r:
  127. continue
  128. arch, packages = r.strip().split(":", 1)
  129. if arch.split("+", 1)[0] in architectures:
  130. nuninst[arch] = set(packages.split())
  131. return nuninst
  132. def newly_uninst(nuold, nunew):
  133. """Return a nuninst statstic with only new uninstallable packages
  134. This method subtracts the uninstallable packages of the statistic
  135. "nunew" from the statistic "nuold".
  136. It returns a dictionary with the architectures as keys and the list
  137. of uninstallable packages as values. If there are no regressions
  138. on a given architecture, then the architecture will be omitted in
  139. the result. Accordingly, if none of the architectures have
  140. regressions an empty directory is returned.
  141. """
  142. res = {}
  143. for arch in ifilter_only(nunew, nuold):
  144. arch_nuninst = [x for x in nunew[arch] if x not in nuold[arch]]
  145. # Leave res empty if there are no newly uninst packages
  146. if arch_nuninst:
  147. res[arch] = arch_nuninst
  148. return res
  149. def format_and_log_uninst(logger, architectures, nuninst, *, loglevel=logging.INFO):
  150. """Emits the uninstallable packages to the log
  151. An example of the output string is:
  152. * i386: broken-pkg1, broken-pkg2
  153. Note that if there is no uninstallable packages, then nothing is emitted.
  154. """
  155. for arch in architectures:
  156. if arch in nuninst and nuninst[arch]:
  157. msg = " * %s: %s" % (arch, ", ".join(sorted(nuninst[arch])))
  158. logger.log(loglevel, msg)
  159. def write_heidi(filename, target_suite, *, outofsync_arches=frozenset(), sorted=sorted):
  160. """Write the output HeidiResult
  161. This method write the output for Heidi, which contains all the
  162. binary packages and the source packages in the form:
  163. <pkg-name> <pkg-version> <pkg-architecture> <pkg-section>
  164. <src-name> <src-version> source <src-section>
  165. The file is written as "filename" using the sources and packages
  166. from the "target_suite" parameter.
  167. outofsync_arches: If given, it is a set of architectures marked
  168. as "out of sync". The output file may exclude some out of date
  169. arch:all packages for those architectures to reduce the noise.
  170. The "X=X" parameters are optimizations to avoid "load global" in
  171. the loops.
  172. """
  173. sources_t = target_suite.sources
  174. packages_t = target_suite.binaries
  175. with open(filename, 'w', encoding='ascii') as f:
  176. # write binary packages
  177. for arch in sorted(packages_t):
  178. binaries = packages_t[arch]
  179. for pkg_name in sorted(binaries):
  180. pkg = binaries[pkg_name]
  181. pkgv = pkg.version
  182. pkgarch = pkg.architecture or 'all'
  183. pkgsec = pkg.section or 'faux'
  184. if pkgsec == 'faux' or pkgsec.endswith('/faux'):
  185. # Faux package; not really a part of testing
  186. continue
  187. if pkg.source_version and pkgarch == 'all' and \
  188. pkg.source_version != sources_t[pkg.source].version and \
  189. arch in outofsync_arches:
  190. # when architectures are marked as "outofsync", their binary
  191. # versions may be lower than those of the associated
  192. # source package in testing. the binary package list for
  193. # such architectures will include arch:all packages
  194. # matching those older versions, but we only want the
  195. # newer arch:all in testing
  196. continue
  197. f.write('%s %s %s %s\n' % (pkg_name, pkgv, pkgarch, pkgsec))
  198. # write sources
  199. for src_name in sorted(sources_t):
  200. src = sources_t[src_name]
  201. srcv = src.version
  202. srcsec = src.section or 'unknown'
  203. if srcsec == 'faux' or srcsec.endswith('/faux'):
  204. # Faux package; not really a part of testing
  205. continue
  206. f.write('%s %s source %s\n' % (src_name, srcv, srcsec))
  207. def write_heidi_delta(filename, all_selected):
  208. """Write the output delta
  209. This method writes the packages to be upgraded, in the form:
  210. <src-name> <src-version>
  211. or (if the source is to be removed):
  212. -<src-name> <src-version>
  213. The order corresponds to that shown in update_output.
  214. """
  215. with open(filename, "w", encoding='ascii') as fd:
  216. fd.write("#HeidiDelta\n")
  217. for item in all_selected:
  218. prefix = ""
  219. if item.is_removal:
  220. prefix = "-"
  221. if item.architecture == 'source':
  222. fd.write('%s%s %s\n' % (prefix, item.package, item.version))
  223. else:
  224. fd.write('%s%s %s %s\n' % (prefix, item.package,
  225. item.version, item.architecture))
  226. def write_excuses(excuselist, dest_file, output_format="yaml"):
  227. """Write the excuses to dest_file
  228. Writes a list of excuses in a specified output_format to the
  229. path denoted by dest_file. The output_format can either be "yaml"
  230. or "legacy-html".
  231. """
  232. if output_format == "yaml":
  233. with open(dest_file, 'w', encoding='utf-8') as f:
  234. edatalist = [e.excusedata() for e in excuselist]
  235. excusesdata = {
  236. 'sources': edatalist,
  237. 'generated-date': datetime.utcnow(),
  238. }
  239. f.write(yaml.dump(excusesdata, default_flow_style=False, allow_unicode=True))
  240. elif output_format == "legacy-html":
  241. with open(dest_file, 'w', encoding='utf-8') as f:
  242. f.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\" \"http://www.w3.org/TR/REC-html40/strict.dtd\">\n")
  243. f.write("<html><head><title>excuses...</title>")
  244. f.write("<meta http-equiv=\"Content-Type\" content=\"text/html;charset=utf-8\"></head><body>\n")
  245. f.write("<p>Generated: " + time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())) + "</p>\n")
  246. f.write("<ul>\n")
  247. for e in excuselist:
  248. f.write("<li>%s" % e.html())
  249. f.write("</ul></body></html>\n")
  250. else: # pragma: no cover
  251. raise ValueError('Output format must be either "yaml or "legacy-html"')
  252. def old_libraries(mi_factory, suite_info, outofsync_arches=frozenset()):
  253. """Detect old libraries left in the target suite for smooth transitions
  254. This method detects old libraries which are in the target suite but no
  255. longer built from the source package: they are still there because
  256. other packages still depend on them, but they should be removed as
  257. soon as possible.
  258. For "outofsync" architectures, outdated binaries are allowed to be in
  259. the target suite, so they are only added to the removal list if they
  260. are no longer in the (primary) source suite.
  261. """
  262. sources_t = suite_info.target_suite.sources
  263. binaries_t = suite_info.target_suite.binaries
  264. binaries_s = suite_info.primary_source_suite.binaries
  265. removals = []
  266. for arch in binaries_t:
  267. for pkg_name in binaries_t[arch]:
  268. pkg = binaries_t[arch][pkg_name]
  269. if sources_t[pkg.source].version != pkg.source_version and \
  270. (arch not in outofsync_arches or pkg_name not in binaries_s[arch]):
  271. removals.append(mi_factory.generate_removal_for_cruft_item(pkg.pkg_id))
  272. return removals
  273. def is_nuninst_asgood_generous(constraints, architectures, old, new, break_arches=frozenset()):
  274. """Compares the nuninst counters and constraints to see if they improved
  275. Given a list of architectures, the previous and the current nuninst
  276. counters, this function determines if the current nuninst counter
  277. is better than the previous one. Optionally it also accepts a set
  278. of "break_arches", the nuninst counter for any architecture listed
  279. in this set are completely ignored.
  280. If the nuninst counters are equal or better, then the constraints
  281. are checked for regressions (ignoring break_arches).
  282. Returns True if the new nuninst counter is better than the
  283. previous and there are no constraint regressions (ignoring Break-archs).
  284. Returns False otherwise.
  285. """
  286. diff = 0
  287. for arch in architectures:
  288. if arch in break_arches:
  289. continue
  290. diff = diff + (len(new[arch]) - len(old[arch]))
  291. if diff > 0:
  292. return False
  293. must_be_installable = constraints['keep-installable']
  294. for arch in architectures:
  295. if arch in break_arches:
  296. continue
  297. regression = new[arch] - old[arch]
  298. if not regression.isdisjoint(must_be_installable):
  299. return False
  300. return True
  301. def clone_nuninst(nuninst, *, packages_s=None, architectures=None):
  302. """Completely or Selectively deep clone nuninst
  303. Given nuninst table, the package table for a given suite and
  304. a list of architectures, this function will clone the nuninst
  305. table. Only the listed architectures will be deep cloned -
  306. the rest will only be shallow cloned. When packages_s is given,
  307. packages not listed in packages_s will be pruned from the clone
  308. (if packages_s is omitted, the per architecture nuninst is cloned
  309. as-is)
  310. """
  311. clone = nuninst.copy()
  312. if architectures is None:
  313. return clone
  314. if packages_s is not None:
  315. for arch in architectures:
  316. clone[arch] = set(x for x in nuninst[arch] if x in packages_s[arch])
  317. clone[arch + "+all"] = set(x for x in nuninst[arch + "+all"] if x in packages_s[arch])
  318. else:
  319. for arch in architectures:
  320. clone[arch] = set(nuninst[arch])
  321. clone[arch + "+all"] = set(nuninst[arch + "+all"])
  322. return clone
  323. def test_installability(target_suite, pkg_name, pkg_id, broken, nuninst_arch):
  324. """Test for installability of a package on an architecture
  325. (pkg_name, pkg_version, pkg_arch) is the package to check.
  326. broken is the set of broken packages. If p changes
  327. installability (e.g. goes from uninstallable to installable),
  328. broken will be updated accordingly.
  329. If nuninst_arch is not None then it also updated in the same
  330. way as broken is.
  331. """
  332. c = 0
  333. r = target_suite.is_installable(pkg_id)
  334. if not r:
  335. # not installable
  336. if pkg_name not in broken:
  337. # regression
  338. broken.add(pkg_name)
  339. c = -1
  340. if nuninst_arch is not None and pkg_name not in nuninst_arch:
  341. nuninst_arch.add(pkg_name)
  342. else:
  343. if pkg_name in broken:
  344. # Improvement
  345. broken.remove(pkg_name)
  346. c = 1
  347. if nuninst_arch is not None and pkg_name in nuninst_arch:
  348. nuninst_arch.remove(pkg_name)
  349. return c
  350. def check_installability(target_suite, binaries, arch, updates, affected, check_archall, nuninst):
  351. broken = nuninst[arch + "+all"]
  352. packages_t_a = binaries[arch]
  353. improvement = 0
  354. # broken packages (first round)
  355. for pkg_id in (x for x in updates if x.architecture == arch):
  356. name, version, parch = pkg_id
  357. if name not in packages_t_a:
  358. continue
  359. pkgdata = packages_t_a[name]
  360. if version != pkgdata.version:
  361. # Not the version in testing right now, ignore
  362. continue
  363. actual_arch = pkgdata.architecture
  364. nuninst_arch = None
  365. # only check arch:all packages if requested
  366. if check_archall or actual_arch != 'all':
  367. nuninst_arch = nuninst[parch]
  368. else:
  369. nuninst[parch].discard(name)
  370. result = test_installability(target_suite, name, pkg_id, broken, nuninst_arch)
  371. if improvement > 0 or not result:
  372. # Any improvement could in theory fix all of its rdeps, so
  373. # stop updating "improvement" after that.
  374. continue
  375. if result > 0:
  376. # Any improvement (even in arch:all packages) could fix any
  377. # number of rdeps
  378. improvement = 1
  379. continue
  380. if check_archall or actual_arch != 'all':
  381. # We cannot count arch:all breakage (except on no-break-arch-all arches)
  382. # because the nuninst check do not consider them regressions.
  383. improvement += result
  384. if improvement < 0:
  385. # The early round is sufficient to disprove the situation
  386. return
  387. for pkg_id in (x for x in affected if x.architecture == arch):
  388. name, version, parch = pkg_id
  389. if name not in packages_t_a:
  390. continue
  391. pkgdata = packages_t_a[name]
  392. if version != pkgdata.version:
  393. # Not the version in testing right now, ignore
  394. continue
  395. actual_arch = pkgdata.architecture
  396. nuninst_arch = None
  397. # only check arch:all packages if requested
  398. if check_archall or actual_arch != 'all':
  399. nuninst_arch = nuninst[parch]
  400. elif actual_arch == 'all':
  401. nuninst[parch].discard(name)
  402. test_installability(target_suite, name, pkg_id, broken, nuninst_arch)
  403. def possibly_compressed(path, *, permitted_compressions=None):
  404. """Find and select a (possibly compressed) variant of a path
  405. If the given path exists, it will be returned
  406. :param path The base path.
  407. :param permitted_compressions An optional list of alternative extensions to look for.
  408. Defaults to "gz" and "xz".
  409. :returns The path given possibly with one of the permitted extensions. Will raise a
  410. FileNotFoundError
  411. """
  412. if os.path.exists(path):
  413. return path
  414. if permitted_compressions is None:
  415. permitted_compressions = ['gz', 'xz']
  416. for ext in permitted_compressions:
  417. cpath = "%s.%s" % (path, ext)
  418. if os.path.exists(cpath):
  419. return cpath
  420. raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) # pragma: no cover
  421. def create_provides_map(packages):
  422. """Create a provides map from a map binary package names and thier BinaryPackage objects
  423. :param packages: A dict mapping binary package names to their BinaryPackage object
  424. :return: A provides map
  425. """
  426. # create provides
  427. provides = defaultdict(set)
  428. for pkg, dpkg in packages.items():
  429. # register virtual packages and real packages that provide
  430. # them
  431. for provided_pkg, provided_version, _ in dpkg.provides:
  432. provides[provided_pkg].add((pkg, provided_version))
  433. return provides
  434. def read_release_file(suite_dir):
  435. """Parses a given "Release" file
  436. :param suite_dir: The directory to the suite
  437. :return: A dict of the first (and only) paragraph in an Release file
  438. """
  439. release_file = os.path.join(suite_dir, 'Release')
  440. with open(release_file) as fd:
  441. tag_file = iter(apt_pkg.TagFile(fd))
  442. result = next(tag_file)
  443. if next(tag_file, None) is not None: # pragma: no cover
  444. raise TypeError("%s has more than one paragraph" % release_file)
  445. return result
  446. def read_sources_file(filename, sources=None, intern=sys.intern):
  447. """Parse a single Sources file into a hash
  448. Parse a single Sources file into a dict mapping a source package
  449. name to a SourcePackage object. If there are multiple source
  450. packages with the same version, then highest versioned source
  451. package (that is not marked as "Extra-Source-Only") is the
  452. version kept in the dict.
  453. :param filename: Path to the Sources file. Can be compressed by any algorithm supported by apt_pkg.TagFile
  454. :param sources: Optional dict to add the packages to. If given, this is also the value returned.
  455. :param intern: Internal optimisation / implementation detail to avoid python's "LOAD_GLOBAL" instruction in a loop
  456. :return a dict mapping a name to a source package
  457. """
  458. if sources is None:
  459. sources = {}
  460. tag_file = apt_pkg.TagFile(filename)
  461. get_field = tag_file.section.get
  462. step = tag_file.step
  463. while step():
  464. if get_field('Extra-Source-Only', 'no') == 'yes':
  465. # Ignore sources only referenced by Built-Using
  466. continue
  467. pkg = get_field('Package')
  468. ver = get_field('Version')
  469. # There may be multiple versions of the source package
  470. # (in unstable) if some architectures have out-of-date
  471. # binaries. We only ever consider the source with the
  472. # largest version for migration.
  473. if pkg in sources and apt_pkg.version_compare(sources[pkg][0], ver) > 0:
  474. continue
  475. maint = get_field('Maintainer')
  476. if maint:
  477. maint = intern(maint.strip())
  478. section = get_field('Section')
  479. if section:
  480. section = intern(section.strip())
  481. build_deps_arch = ", ".join(x for x in (get_field('Build-Depends'), get_field('Build-Depends-Arch'))
  482. if x is not None)
  483. if build_deps_arch != '':
  484. build_deps_arch = sys.intern(build_deps_arch)
  485. else:
  486. build_deps_arch = None
  487. build_deps_indep = get_field('Build-Depends-Indep')
  488. if build_deps_indep is not None:
  489. build_deps_indep = sys.intern(build_deps_indep)
  490. sources[intern(pkg)] = SourcePackage(intern(ver),
  491. section,
  492. set(),
  493. maint,
  494. False,
  495. build_deps_arch,
  496. build_deps_indep,
  497. get_field('Testsuite', '').split(),
  498. get_field('Testsuite-Triggers', '').replace(',', '').split(),
  499. )
  500. return sources
  501. def get_dependency_solvers(block, binaries_s_a, provides_s_a, *, build_depends=False, empty_set=frozenset()):
  502. """Find the packages which satisfy a dependency block
  503. This method returns the list of packages which satisfy a dependency
  504. block (as returned by apt_pkg.parse_depends) in a package table
  505. for a given suite and architecture (a la self.binaries[suite][arch])
  506. It can also handle build-dependency relations if the named parameter
  507. "build_depends" is set to True. In this case, block should be based
  508. on the return value from apt_pkg.parse_src_depends.
  509. :param block: The dependency block as parsed by apt_pkg.parse_depends (or apt_pkg.parse_src_depends
  510. if the "build_depends" is True)
  511. :param binaries_s_a: A dict mapping package names to the relevant BinaryPackage
  512. :param provides_s_a: A dict mapping package names to their providers (as generated by parse_provides)
  513. :param build_depends: If True, treat the "block" parameter as a build-dependency relation rather than
  514. a regular dependency relation.
  515. :param empty_set: Internal implementation detail / optimisation
  516. :return a list of package names solving the relation
  517. """
  518. packages = []
  519. # for every package, version and operation in the block
  520. for name, version, op in block:
  521. if ":" in name:
  522. name, archqual = name.split(":", 1)
  523. else:
  524. archqual = None
  525. # look for the package in unstable
  526. if name in binaries_s_a:
  527. package = binaries_s_a[name]
  528. # check the versioned dependency and architecture qualifier
  529. # (if present)
  530. if (op == '' and version == '') or apt_pkg.check_dep(package.version, op, version):
  531. if archqual is None:
  532. packages.append(package)
  533. elif build_depends and archqual == 'native':
  534. # Multi-arch handling for build-dependencies
  535. # - :native is ok always (since dpkg 1.19.1)
  536. packages.append(package)
  537. # Multi-arch handling for both build-dependencies and regular dependencies
  538. # - :any is ok iff the target has "M-A: allowed"
  539. if archqual == 'any' and package.multi_arch == 'allowed':
  540. packages.append(package)
  541. # look for the package in the virtual packages list and loop on them
  542. for prov, prov_version in provides_s_a.get(name, empty_set):
  543. assert prov in binaries_s_a
  544. # A provides only satisfies:
  545. # - an unversioned dependency (per Policy Manual ยง7.5)
  546. # - a dependency without an architecture qualifier
  547. # (per analysis of apt code)
  548. if archqual is not None:
  549. # Punt on this case - these days, APT and dpkg might actually agree on
  550. # this.
  551. continue
  552. if (op == '' and version == '') or \
  553. (prov_version != '' and apt_pkg.check_dep(prov_version, op, version)):
  554. packages.append(binaries_s_a[prov])
  555. return packages
  556. def invalidate_excuses(excuses, valid, invalid):
  557. """Invalidate impossible excuses
  558. This method invalidates the impossible excuses, which depend
  559. on invalid excuses. The two parameters contains the sets of
  560. `valid' and `invalid' excuses.
  561. """
  562. # build the reverse dependencies
  563. allrevdeps = defaultdict(dict)
  564. for exc in excuses.values():
  565. for d in exc.all_deps:
  566. if exc.name not in allrevdeps[d]:
  567. allrevdeps[d][exc.name] = set()
  568. for deptype in exc.all_deps[d]:
  569. allrevdeps[d][exc.name].add(deptype)
  570. # loop on the invalid excuses
  571. for ename in iter_except(invalid.pop, KeyError):
  572. # if there is no reverse dependency, skip the item
  573. if ename not in allrevdeps:
  574. continue
  575. # if the dependency can be satisfied by a testing-proposed-updates excuse, skip the item
  576. if (ename + "_tpu") in valid:
  577. continue
  578. rdep_verdict = PolicyVerdict.REJECTED_WAITING_FOR_ANOTHER_ITEM
  579. if excuses[ename].policy_verdict.is_blocked:
  580. rdep_verdict = PolicyVerdict.REJECTED_BLOCKED_BY_ANOTHER_ITEM
  581. # loop on the reverse dependencies
  582. if ename in allrevdeps:
  583. for x in allrevdeps[ename]:
  584. # if the item is valid and it is not marked as `forced', then we invalidate it
  585. if x in valid and not excuses[x].forced:
  586. # otherwise, invalidate the dependency and mark as invalidated and
  587. # remove the depending excuses
  588. excuses[x].invalidate_dependency(ename)
  589. valid.discard(x)
  590. invalid.add(x)
  591. for deptype in allrevdeps[ename][x]:
  592. excuses[x].addhtml("Invalidated by %s" % deptype.get_description())
  593. excuses[x].addreason(deptype.get_reason())
  594. if excuses[x].policy_verdict.value < rdep_verdict.value:
  595. excuses[x].policy_verdict = rdep_verdict
  596. def compile_nuninst(target_suite, architectures, nobreakall_arches):
  597. """Compile a nuninst dict from the current testing
  598. :param target_suite: The target suite
  599. :param architectures: List of architectures
  600. :param nobreakall_arches: List of architectures where arch:all packages must be installable
  601. """
  602. nuninst = {}
  603. binaries_t = target_suite.binaries
  604. # for all the architectures
  605. for arch in architectures:
  606. # if it is in the nobreakall ones, check arch-independent packages too
  607. check_archall = arch in nobreakall_arches
  608. # check all the packages for this architecture
  609. nuninst[arch] = set()
  610. packages_t_a = binaries_t[arch]
  611. for pkg_name, pkg_data in packages_t_a.items():
  612. r = target_suite.is_installable(pkg_data.pkg_id)
  613. if not r:
  614. nuninst[arch].add(pkg_name)
  615. # if they are not required, remove architecture-independent packages
  616. nuninst[arch + "+all"] = nuninst[arch].copy()
  617. if not check_archall:
  618. for pkg_name in nuninst[arch + "+all"]:
  619. pkg_data = packages_t_a[pkg_name]
  620. if pkg_data.architecture == 'all':
  621. nuninst[arch].remove(pkg_name)
  622. return nuninst
  623. def find_smooth_updateable_binaries(binaries_to_check,
  624. source_data,
  625. pkg_universe,
  626. target_suite,
  627. binaries_t,
  628. binaries_s,
  629. removals,
  630. smooth_updates):
  631. check = set()
  632. smoothbins = set()
  633. for pkg_id in binaries_to_check:
  634. binary, _, parch = pkg_id
  635. cruft = False
  636. cruftbins = set()
  637. # Not a candidate for smooth up date (newer non-cruft version in unstable)
  638. if binary in binaries_s[parch]:
  639. if binaries_s[parch][binary].source_version == source_data.version:
  640. continue
  641. cruftbins.add(binaries_s[parch][binary].pkg_id)
  642. cruft = True
  643. # Maybe a candidate (cruft or removed binary): check if config allows us to smooth update it.
  644. if cruft or 'ALL' in smooth_updates or binaries_t[parch][binary].section in smooth_updates:
  645. # if the package has reverse-dependencies which are
  646. # built from other sources, it's a valid candidate for
  647. # a smooth update. if not, it may still be a valid
  648. # candidate if one if its r-deps is itself a candidate,
  649. # so note it for checking later
  650. rdeps = set(pkg_universe.reverse_dependencies_of(pkg_id))
  651. # We ignore all binaries listed in "removals" as we
  652. # assume they will leave at the same time as the
  653. # given package.
  654. rdeps.difference_update(removals, binaries_to_check)
  655. smooth_update_it = False
  656. if target_suite.any_of_these_are_in_the_suite(rdeps):
  657. combined = set(smoothbins)
  658. combined.add(pkg_id)
  659. for rdep in rdeps:
  660. for dep_clause in pkg_universe.dependencies_of(rdep):
  661. # filter out cruft binaries from unstable, because
  662. # they will not be added to the set of packages that
  663. # will be migrated
  664. if dep_clause - cruftbins <= combined:
  665. smooth_update_it = True
  666. break
  667. if smooth_update_it:
  668. smoothbins = combined
  669. else:
  670. check.add(pkg_id)
  671. # check whether we should perform a smooth update for
  672. # packages which are candidates but do not have r-deps
  673. # outside of the current source
  674. while 1:
  675. found_any = False
  676. for pkg_id in check:
  677. rdeps = pkg_universe.reverse_dependencies_of(pkg_id)
  678. if not rdeps.isdisjoint(smoothbins):
  679. smoothbins.add(pkg_id)
  680. found_any = True
  681. if not found_any:
  682. break
  683. check = [x for x in check if x not in smoothbins]
  684. return smoothbins
  685. def parse_provides(provides_raw, pkg_id=None, logger=None):
  686. parts = apt_pkg.parse_depends(provides_raw, False)
  687. nprov = []
  688. for or_clause in parts:
  689. if len(or_clause) != 1: # pragma: no cover
  690. if logger is not None:
  691. msg = "Ignoring invalid provides in %s: Alternatives [%s]"
  692. logger.warning(msg, str(pkg_id), str(or_clause))
  693. continue
  694. for part in or_clause:
  695. provided, provided_version, op = part
  696. if op != '' and op != '=': # pragma: no cover
  697. if logger is not None:
  698. msg = "Ignoring invalid provides in %s: %s (%s %s)"
  699. logger.warning(msg, str(pkg_id), provided, op, provided_version)
  700. continue
  701. provided = sys.intern(provided)
  702. provided_version = sys.intern(provided_version)
  703. part = (provided, provided_version, sys.intern(op))
  704. nprov.append(part)
  705. return nprov
  706. def compute_item_name(sources_t, sources_s, source_name, parch):
  707. if source_name in sources_t and sources_t[source_name].version == sources_s[source_name].version:
  708. return "%s/%s" % (source_name, parch)
  709. return source_name
  710. def parse_builtusing(builtusing_raw, pkg_id=None, logger=None):
  711. parts = apt_pkg.parse_depends(builtusing_raw, False)
  712. nbu = []
  713. for or_clause in parts:
  714. if len(or_clause) != 1: # pragma: no cover
  715. if logger is not None:
  716. msg = "Ignoring invalid builtusing in %s: Alternatives [%s]"
  717. logger.warning(msg, str(pkg_id), str(or_clause))
  718. continue
  719. for part in or_clause:
  720. bu, bu_version, op = part
  721. if op != '=': # pragma: no cover
  722. if logger is not None:
  723. msg = "Ignoring invalid builtusing in %s: %s (%s %s)"
  724. logger.warning(msg, str(pkg_id), bu, op, bu_version)
  725. continue
  726. bu = sys.intern(bu)
  727. bu_version = sys.intern(bu_version)
  728. part = (bu, bu_version)
  729. nbu.append(part)
  730. return nbu