Devuan deployment of britney2
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

863 lines
34 KiB

  1. # -*- coding: utf-8 -*-
  2. # Refactored parts from britney.py, which is/was:
  3. # Copyright (C) 2001-2008 Anthony Towns <ajt@debian.org>
  4. # Andreas Barth <aba@debian.org>
  5. # Fabio Tranchitella <kobold@debian.org>
  6. # Copyright (C) 2010-2012 Adam D. Barratt <adsb@debian.org>
  7. # Copyright (C) 2012 Niels Thykier <niels@thykier.net>
  8. #
  9. # New portions
  10. # Copyright (C) 2013 Adam D. Barratt <adsb@debian.org>
  11. # This program is free software; you can redistribute it and/or modify
  12. # it under the terms of the GNU General Public License as published by
  13. # the Free Software Foundation; either version 2 of the License, or
  14. # (at your option) any later version.
  15. # This program is distributed in the hope that it will be useful,
  16. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  17. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18. # GNU General Public License for more details.
  19. import apt_pkg
  20. import errno
  21. import logging
  22. import os
  23. import sys
  24. import time
  25. from collections import defaultdict
  26. from datetime import datetime
  27. from functools import partial
  28. from itertools import filterfalse, chain
  29. import yaml
  30. from britney2 import SourcePackage
  31. from britney2.policies import PolicyVerdict
  32. class MigrationConstraintException(Exception):
  33. pass
  34. def ifilter_except(container, iterable=None):
  35. """Filter out elements in container
  36. If given an iterable it returns a filtered iterator, otherwise it
  37. returns a function to generate filtered iterators. The latter is
  38. useful if the same filter has to be (re-)used on multiple
  39. iterators that are not known on beforehand.
  40. """
  41. if iterable is not None:
  42. return filterfalse(container.__contains__, iterable)
  43. return partial(filterfalse, container.__contains__)
  44. def ifilter_only(container, iterable=None):
  45. """Filter out elements in which are not in container
  46. If given an iterable it returns a filtered iterator, otherwise it
  47. returns a function to generate filtered iterators. The latter is
  48. useful if the same filter has to be (re-)used on multiple
  49. iterators that are not known on beforehand.
  50. """
  51. if iterable is not None:
  52. return filter(container.__contains__, iterable)
  53. return partial(filter, container.__contains__)
  54. # iter_except is from the "itertools" recipe
  55. def iter_except(func, exception, first=None): # pragma: no cover - itertools recipe function
  56. """ Call a function repeatedly until an exception is raised.
  57. Converts a call-until-exception interface to an iterator interface.
  58. Like __builtin__.iter(func, sentinel) but uses an exception instead
  59. of a sentinel to end the loop.
  60. Examples:
  61. bsddbiter = iter_except(db.next, bsddb.error, db.first)
  62. heapiter = iter_except(functools.partial(heappop, h), IndexError)
  63. dictiter = iter_except(d.popitem, KeyError)
  64. dequeiter = iter_except(d.popleft, IndexError)
  65. queueiter = iter_except(q.get_nowait, Queue.Empty)
  66. setiter = iter_except(s.pop, KeyError)
  67. """
  68. try:
  69. if first is not None:
  70. yield first()
  71. while 1:
  72. yield func()
  73. except exception:
  74. pass
  75. def log_and_format_old_libraries(logger, libs):
  76. """Format and log old libraries in a table (no header)"""
  77. libraries = {}
  78. for i in libs:
  79. pkg = i.package
  80. if pkg in libraries:
  81. libraries[pkg].append(i.architecture)
  82. else:
  83. libraries[pkg] = [i.architecture]
  84. for lib in sorted(libraries):
  85. logger.info(" %s: %s", lib, " ".join(libraries[lib]))
  86. def compute_reverse_tree(pkg_universe, affected):
  87. """Calculate the full dependency tree for a set of packages
  88. This method returns the full dependency tree for a given set of
  89. packages. The first argument is an instance of the BinaryPackageUniverse
  90. and the second argument are a set of BinaryPackageId.
  91. The set of affected packages will be updated in place and must
  92. therefore be mutable.
  93. """
  94. remain = list(affected)
  95. while remain:
  96. pkg_id = remain.pop()
  97. new_pkg_ids = pkg_universe.reverse_dependencies_of(pkg_id) - affected
  98. affected.update(new_pkg_ids)
  99. remain.extend(new_pkg_ids)
  100. return None
  101. def add_transitive_dependencies_flatten(pkg_universe, initial_set):
  102. """Find and include all transitive dependencies
  103. This method updates the initial_set parameter to include all transitive
  104. dependencies. The first argument is an instance of the BinaryPackageUniverse
  105. and the second argument are a set of BinaryPackageId.
  106. The set of initial packages will be updated in place and must
  107. therefore be mutable.
  108. """
  109. remain = list(initial_set)
  110. while remain:
  111. pkg_id = remain.pop()
  112. new_pkg_ids = [x for x in chain.from_iterable(pkg_universe.dependencies_of(pkg_id)) if x not in initial_set]
  113. initial_set.update(new_pkg_ids)
  114. remain.extend(new_pkg_ids)
  115. return None
  116. def write_nuninst(filename, nuninst):
  117. """Write the non-installable report
  118. Write the non-installable report derived from "nuninst" to the
  119. file denoted by "filename".
  120. """
  121. with open(filename, 'w', encoding='utf-8') as f:
  122. # Having two fields with (almost) identical dates seems a bit
  123. # redundant.
  124. f.write("Built on: " + time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())) + "\n")
  125. f.write("Last update: " + time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())) + "\n\n")
  126. for k in nuninst:
  127. f.write("%s: %s\n" % (k, " ".join(nuninst[k])))
  128. def read_nuninst(filename, architectures):
  129. """Read the non-installable report
  130. Read the non-installable report from the file denoted by
  131. "filename" and return it. Only architectures in "architectures"
  132. will be included in the report.
  133. """
  134. nuninst = {}
  135. with open(filename, encoding='ascii') as f:
  136. for r in f:
  137. if ":" not in r:
  138. continue
  139. arch, packages = r.strip().split(":", 1)
  140. if arch.split("+", 1)[0] in architectures:
  141. nuninst[arch] = set(packages.split())
  142. return nuninst
  143. def newly_uninst(nuold, nunew):
  144. """Return a nuninst statstic with only new uninstallable packages
  145. This method subtracts the uninstallable packages of the statistic
  146. "nunew" from the statistic "nuold".
  147. It returns a dictionary with the architectures as keys and the list
  148. of uninstallable packages as values. If there are no regressions
  149. on a given architecture, then the architecture will be omitted in
  150. the result. Accordingly, if none of the architectures have
  151. regressions an empty directory is returned.
  152. """
  153. res = {}
  154. for arch in ifilter_only(nunew, nuold):
  155. arch_nuninst = [x for x in nunew[arch] if x not in nuold[arch]]
  156. # Leave res empty if there are no newly uninst packages
  157. if arch_nuninst:
  158. res[arch] = arch_nuninst
  159. return res
  160. def format_and_log_uninst(logger, architectures, nuninst, *, loglevel=logging.INFO):
  161. """Emits the uninstallable packages to the log
  162. An example of the output string is:
  163. * i386: broken-pkg1, broken-pkg2
  164. Note that if there is no uninstallable packages, then nothing is emitted.
  165. """
  166. for arch in architectures:
  167. if arch in nuninst and nuninst[arch]:
  168. msg = " * %s: %s" % (arch, ", ".join(sorted(nuninst[arch])))
  169. logger.log(loglevel, msg)
  170. def write_heidi(filename, target_suite, *, outofsync_arches=frozenset(), sorted=sorted):
  171. """Write the output HeidiResult
  172. This method write the output for Heidi, which contains all the
  173. binary packages and the source packages in the form:
  174. <pkg-name> <pkg-version> <pkg-architecture> <pkg-section>
  175. <src-name> <src-version> source <src-section>
  176. The file is written as "filename" using the sources and packages
  177. from the "target_suite" parameter.
  178. outofsync_arches: If given, it is a set of architectures marked
  179. as "out of sync". The output file may exclude some out of date
  180. arch:all packages for those architectures to reduce the noise.
  181. The "X=X" parameters are optimizations to avoid "load global" in
  182. the loops.
  183. """
  184. sources_t = target_suite.sources
  185. packages_t = target_suite.binaries
  186. with open(filename, 'w', encoding='ascii') as f:
  187. # write binary packages
  188. for arch in sorted(packages_t):
  189. binaries = packages_t[arch]
  190. for pkg_name in sorted(binaries):
  191. pkg = binaries[pkg_name]
  192. pkgv = pkg.version
  193. pkgarch = pkg.architecture or 'all'
  194. pkgsec = pkg.section or 'faux'
  195. if pkgsec == 'faux' or pkgsec.endswith('/faux'):
  196. # Faux package; not really a part of testing
  197. continue
  198. if pkg.source_version and pkgarch == 'all' and \
  199. pkg.source_version != sources_t[pkg.source].version and \
  200. arch in outofsync_arches:
  201. # when architectures are marked as "outofsync", their binary
  202. # versions may be lower than those of the associated
  203. # source package in testing. the binary package list for
  204. # such architectures will include arch:all packages
  205. # matching those older versions, but we only want the
  206. # newer arch:all in testing
  207. continue
  208. f.write('%s %s %s %s\n' % (pkg_name, pkgv, pkgarch, pkgsec))
  209. # write sources
  210. for src_name in sorted(sources_t):
  211. src = sources_t[src_name]
  212. srcv = src.version
  213. srcsec = src.section or 'unknown'
  214. if srcsec == 'faux' or srcsec.endswith('/faux'):
  215. # Faux package; not really a part of testing
  216. continue
  217. f.write('%s %s source %s\n' % (src_name, srcv, srcsec))
  218. def write_heidi_delta(filename, all_selected):
  219. """Write the output delta
  220. This method writes the packages to be upgraded, in the form:
  221. <src-name> <src-version>
  222. or (if the source is to be removed):
  223. -<src-name> <src-version>
  224. The order corresponds to that shown in update_output.
  225. """
  226. with open(filename, "w", encoding='ascii') as fd:
  227. fd.write("#HeidiDelta\n")
  228. for item in all_selected:
  229. prefix = ""
  230. if item.is_removal:
  231. prefix = "-"
  232. if item.architecture == 'source':
  233. fd.write('%s%s %s\n' % (prefix, item.package, item.version))
  234. else:
  235. fd.write('%s%s %s %s\n' % (prefix, item.package,
  236. item.version, item.architecture))
  237. def write_excuses(excuselist, dest_file, output_format="yaml"):
  238. """Write the excuses to dest_file
  239. Writes a list of excuses in a specified output_format to the
  240. path denoted by dest_file. The output_format can either be "yaml"
  241. or "legacy-html".
  242. """
  243. if output_format == "yaml":
  244. with open(dest_file, 'w', encoding='utf-8') as f:
  245. edatalist = [e.excusedata() for e in excuselist]
  246. excusesdata = {
  247. 'sources': edatalist,
  248. 'generated-date': datetime.utcnow(),
  249. }
  250. f.write(yaml.dump(excusesdata, default_flow_style=False, allow_unicode=True))
  251. elif output_format == "legacy-html":
  252. with open(dest_file, 'w', encoding='utf-8') as f:
  253. f.write("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01//EN\" \"http://www.w3.org/TR/REC-html40/strict.dtd\">\n")
  254. f.write("<html><head><title>excuses...</title>")
  255. f.write("<meta http-equiv=\"Content-Type\" content=\"text/html;charset=utf-8\"></head><body>\n")
  256. f.write("<p>Generated: " + time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())) + "</p>\n")
  257. f.write("<ul>\n")
  258. for e in excuselist:
  259. f.write("<li>%s" % e.html())
  260. f.write("</ul></body></html>\n")
  261. else: # pragma: no cover
  262. raise ValueError('Output format must be either "yaml or "legacy-html"')
  263. def old_libraries(mi_factory, suite_info, outofsync_arches=frozenset()):
  264. """Detect old libraries left in the target suite for smooth transitions
  265. This method detects old libraries which are in the target suite but no
  266. longer built from the source package: they are still there because
  267. other packages still depend on them, but they should be removed as
  268. soon as possible.
  269. For "outofsync" architectures, outdated binaries are allowed to be in
  270. the target suite, so they are only added to the removal list if they
  271. are no longer in the (primary) source suite.
  272. """
  273. sources_t = suite_info.target_suite.sources
  274. binaries_t = suite_info.target_suite.binaries
  275. binaries_s = suite_info.primary_source_suite.binaries
  276. removals = []
  277. for arch in binaries_t:
  278. for pkg_name in binaries_t[arch]:
  279. pkg = binaries_t[arch][pkg_name]
  280. if sources_t[pkg.source].version != pkg.source_version and \
  281. (arch not in outofsync_arches or pkg_name not in binaries_s[arch]):
  282. removals.append(mi_factory.generate_removal_for_cruft_item(pkg.pkg_id))
  283. return removals
  284. def is_nuninst_asgood_generous(constraints, allow_uninst, architectures, old, new, break_arches=frozenset()):
  285. """Compares the nuninst counters and constraints to see if they improved
  286. Given a list of architectures, the previous and the current nuninst
  287. counters, this function determines if the current nuninst counter
  288. is better than the previous one. Optionally it also accepts a set
  289. of "break_arches", the nuninst counter for any architecture listed
  290. in this set are completely ignored.
  291. If the nuninst counters are equal or better, then the constraints
  292. are checked for regressions (ignoring break_arches).
  293. Returns True if the new nuninst counter is better than the
  294. previous and there are no constraint regressions (ignoring Break-archs).
  295. Returns False otherwise.
  296. """
  297. diff = 0
  298. for arch in architectures:
  299. if arch in break_arches:
  300. continue
  301. diff = diff + \
  302. (len(new[arch] - allow_uninst[arch])
  303. - len(old[arch] - allow_uninst[arch]))
  304. if diff > 0:
  305. return False
  306. must_be_installable = constraints['keep-installable']
  307. for arch in architectures:
  308. if arch in break_arches:
  309. continue
  310. regression = new[arch] - old[arch]
  311. if not regression.isdisjoint(must_be_installable):
  312. return False
  313. return True
  314. def clone_nuninst(nuninst, *, packages_s=None, architectures=None):
  315. """Completely or Selectively deep clone nuninst
  316. Given nuninst table, the package table for a given suite and
  317. a list of architectures, this function will clone the nuninst
  318. table. Only the listed architectures will be deep cloned -
  319. the rest will only be shallow cloned. When packages_s is given,
  320. packages not listed in packages_s will be pruned from the clone
  321. (if packages_s is omitted, the per architecture nuninst is cloned
  322. as-is)
  323. """
  324. clone = nuninst.copy()
  325. if architectures is None:
  326. return clone
  327. if packages_s is not None:
  328. for arch in architectures:
  329. clone[arch] = set(x for x in nuninst[arch] if x in packages_s[arch])
  330. clone[arch + "+all"] = set(x for x in nuninst[arch + "+all"] if x in packages_s[arch])
  331. else:
  332. for arch in architectures:
  333. clone[arch] = set(nuninst[arch])
  334. clone[arch + "+all"] = set(nuninst[arch + "+all"])
  335. return clone
  336. def test_installability(target_suite, pkg_name, pkg_id, broken, nuninst_arch):
  337. """Test for installability of a package on an architecture
  338. (pkg_name, pkg_version, pkg_arch) is the package to check.
  339. broken is the set of broken packages. If p changes
  340. installability (e.g. goes from uninstallable to installable),
  341. broken will be updated accordingly.
  342. If nuninst_arch is not None then it also updated in the same
  343. way as broken is.
  344. """
  345. c = 0
  346. r = target_suite.is_installable(pkg_id)
  347. if not r:
  348. # not installable
  349. if pkg_name not in broken:
  350. # regression
  351. broken.add(pkg_name)
  352. c = -1
  353. if nuninst_arch is not None and pkg_name not in nuninst_arch:
  354. nuninst_arch.add(pkg_name)
  355. else:
  356. if pkg_name in broken:
  357. # Improvement
  358. broken.remove(pkg_name)
  359. c = 1
  360. if nuninst_arch is not None and pkg_name in nuninst_arch:
  361. nuninst_arch.remove(pkg_name)
  362. return c
  363. def check_installability(target_suite, binaries, arch, updates, check_archall, nuninst):
  364. broken = nuninst[arch + "+all"]
  365. packages_t_a = binaries[arch]
  366. for pkg_id in (x for x in updates if x.architecture == arch):
  367. name, version, parch = pkg_id
  368. if name not in packages_t_a:
  369. continue
  370. pkgdata = packages_t_a[name]
  371. if version != pkgdata.version:
  372. # Not the version in testing right now, ignore
  373. continue
  374. actual_arch = pkgdata.architecture
  375. nuninst_arch = None
  376. # only check arch:all packages if requested
  377. if check_archall or actual_arch != 'all':
  378. nuninst_arch = nuninst[parch]
  379. elif actual_arch == 'all':
  380. nuninst[parch].discard(name)
  381. test_installability(target_suite, name, pkg_id, broken, nuninst_arch)
  382. def possibly_compressed(path, *, permitted_compressions=None):
  383. """Find and select a (possibly compressed) variant of a path
  384. If the given path exists, it will be returned
  385. :param path The base path.
  386. :param permitted_compressions An optional list of alternative extensions to look for.
  387. Defaults to "gz" and "xz".
  388. :returns The path given possibly with one of the permitted extensions. Will raise a
  389. FileNotFoundError
  390. """
  391. if os.path.exists(path):
  392. return path
  393. if permitted_compressions is None:
  394. permitted_compressions = ['gz', 'xz']
  395. for ext in permitted_compressions:
  396. cpath = "%s.%s" % (path, ext)
  397. if os.path.exists(cpath):
  398. return cpath
  399. raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), path) # pragma: no cover
  400. def create_provides_map(packages):
  401. """Create a provides map from a map binary package names and thier BinaryPackage objects
  402. :param packages: A dict mapping binary package names to their BinaryPackage object
  403. :return: A provides map
  404. """
  405. # create provides
  406. provides = defaultdict(set)
  407. for pkg, dpkg in packages.items():
  408. # register virtual packages and real packages that provide
  409. # them
  410. for provided_pkg, provided_version, _ in dpkg.provides:
  411. provides[provided_pkg].add((pkg, provided_version))
  412. return provides
  413. def read_release_file(suite_dir):
  414. """Parses a given "Release" file
  415. :param suite_dir: The directory to the suite
  416. :return: A dict of the first (and only) paragraph in an Release file
  417. """
  418. release_file = os.path.join(suite_dir, 'Release')
  419. with open(release_file) as fd:
  420. tag_file = iter(apt_pkg.TagFile(fd))
  421. result = next(tag_file)
  422. if next(tag_file, None) is not None: # pragma: no cover
  423. raise TypeError("%s has more than one paragraph" % release_file)
  424. return result
  425. def read_sources_file(filename, sources=None, intern=sys.intern):
  426. """Parse a single Sources file into a hash
  427. Parse a single Sources file into a dict mapping a source package
  428. name to a SourcePackage object. If there are multiple source
  429. packages with the same version, then highest versioned source
  430. package (that is not marked as "Extra-Source-Only") is the
  431. version kept in the dict.
  432. :param filename: Path to the Sources file. Can be compressed by any algorithm supported by apt_pkg.TagFile
  433. :param sources: Optional dict to add the packages to. If given, this is also the value returned.
  434. :param intern: Internal optimisation / implementation detail to avoid python's "LOAD_GLOBAL" instruction in a loop
  435. :return a dict mapping a name to a source package
  436. """
  437. if sources is None:
  438. sources = {}
  439. tag_file = apt_pkg.TagFile(filename)
  440. get_field = tag_file.section.get
  441. step = tag_file.step
  442. while step():
  443. if get_field('Extra-Source-Only', 'no') == 'yes':
  444. # Ignore sources only referenced by Built-Using
  445. continue
  446. pkg = get_field('Package')
  447. ver = get_field('Version')
  448. # There may be multiple versions of the source package
  449. # (in unstable) if some architectures have out-of-date
  450. # binaries. We only ever consider the source with the
  451. # largest version for migration.
  452. if pkg in sources and apt_pkg.version_compare(sources[pkg][0], ver) > 0:
  453. continue
  454. maint = get_field('Maintainer')
  455. if maint:
  456. maint = intern(maint.strip())
  457. section = get_field('Section')
  458. if section:
  459. section = intern(section.strip())
  460. build_deps_arch = ", ".join(x for x in (get_field('Build-Depends'), get_field('Build-Depends-Arch'))
  461. if x is not None)
  462. if build_deps_arch != '':
  463. build_deps_arch = sys.intern(build_deps_arch)
  464. else:
  465. build_deps_arch = None
  466. build_deps_indep = get_field('Build-Depends-Indep')
  467. if build_deps_indep is not None:
  468. build_deps_indep = sys.intern(build_deps_indep)
  469. sources[intern(pkg)] = SourcePackage(intern(ver),
  470. section,
  471. set(),
  472. maint,
  473. False,
  474. build_deps_arch,
  475. build_deps_indep,
  476. get_field('Testsuite', '').split(),
  477. get_field('Testsuite-Triggers', '').replace(',', '').split(),
  478. )
  479. return sources
  480. def get_dependency_solvers(block, binaries_s_a, provides_s_a, *, build_depends=False, empty_set=frozenset()):
  481. """Find the packages which satisfy a dependency block
  482. This method returns the list of packages which satisfy a dependency
  483. block (as returned by apt_pkg.parse_depends) in a package table
  484. for a given suite and architecture (a la self.binaries[suite][arch])
  485. It can also handle build-dependency relations if the named parameter
  486. "build_depends" is set to True. In this case, block should be based
  487. on the return value from apt_pkg.parse_src_depends.
  488. :param block: The dependency block as parsed by apt_pkg.parse_depends (or apt_pkg.parse_src_depends
  489. if the "build_depends" is True)
  490. :param binaries_s_a: A dict mapping package names to the relevant BinaryPackage
  491. :param provides_s_a: A dict mapping package names to their providers (as generated by parse_provides)
  492. :param build_depends: If True, treat the "block" parameter as a build-dependency relation rather than
  493. a regular dependency relation.
  494. :param empty_set: Internal implementation detail / optimisation
  495. :return a list of package names solving the relation
  496. """
  497. packages = []
  498. # for every package, version and operation in the block
  499. for name, version, op in block:
  500. if ":" in name:
  501. name, archqual = name.split(":", 1)
  502. else:
  503. archqual = None
  504. # look for the package in unstable
  505. if name in binaries_s_a:
  506. package = binaries_s_a[name]
  507. # check the versioned dependency and architecture qualifier
  508. # (if present)
  509. if (op == '' and version == '') or apt_pkg.check_dep(package.version, op, version):
  510. if archqual is None:
  511. packages.append(package)
  512. elif build_depends and archqual == 'native':
  513. # Multi-arch handling for build-dependencies
  514. # - :native is ok always (since dpkg 1.19.1)
  515. packages.append(package)
  516. # Multi-arch handling for both build-dependencies and regular dependencies
  517. # - :any is ok iff the target has "M-A: allowed"
  518. if archqual == 'any' and package.multi_arch == 'allowed':
  519. packages.append(package)
  520. # look for the package in the virtual packages list and loop on them
  521. for prov, prov_version in provides_s_a.get(name, empty_set):
  522. assert prov in binaries_s_a
  523. # A provides only satisfies:
  524. # - an unversioned dependency (per Policy Manual ยง7.5)
  525. # - a dependency without an architecture qualifier
  526. # (per analysis of apt code)
  527. if archqual is not None:
  528. # Punt on this case - these days, APT and dpkg might actually agree on
  529. # this.
  530. continue
  531. if (op == '' and version == '') or \
  532. (prov_version != '' and apt_pkg.check_dep(prov_version, op, version)):
  533. packages.append(binaries_s_a[prov])
  534. return packages
  535. def invalidate_excuses(excuses, valid, invalid):
  536. """Invalidate impossible excuses
  537. This method invalidates the impossible excuses, which depend
  538. on invalid excuses. The two parameters contains the sets of
  539. `valid' and `invalid' excuses.
  540. """
  541. # build the reverse dependencies
  542. allrevdeps = defaultdict(dict)
  543. for exc in excuses.values():
  544. for d in exc.all_deps:
  545. if exc.name not in allrevdeps[d]:
  546. allrevdeps[d][exc.name] = set()
  547. for deptype in exc.all_deps[d]:
  548. allrevdeps[d][exc.name].add(deptype)
  549. # loop on the invalid excuses
  550. for ename in iter_except(invalid.pop, KeyError):
  551. # if there is no reverse dependency, skip the item
  552. if ename not in allrevdeps:
  553. continue
  554. # if the dependency can be satisfied by a testing-proposed-updates excuse, skip the item
  555. if (ename + "_tpu") in valid:
  556. continue
  557. rdep_verdict = PolicyVerdict.REJECTED_WAITING_FOR_ANOTHER_ITEM
  558. if excuses[ename].policy_verdict.is_blocked:
  559. rdep_verdict = PolicyVerdict.REJECTED_BLOCKED_BY_ANOTHER_ITEM
  560. # loop on the reverse dependencies
  561. if ename in allrevdeps:
  562. for x in allrevdeps[ename]:
  563. # if the item is valid and it is not marked as `forced', then we invalidate it
  564. if x in valid and not excuses[x].forced:
  565. if excuses[x].policy_verdict < rdep_verdict:
  566. excuses[x].policy_verdict = rdep_verdict
  567. # otherwise, invalidate the dependency and mark as invalidated and
  568. # remove the depending excuses
  569. excuses[x].invalidate_dependency(ename, rdep_verdict)
  570. valid.discard(x)
  571. invalid.add(x)
  572. for deptype in allrevdeps[ename][x]:
  573. excuses[x].add_verdict_info(rdep_verdict, "Invalidated by %s" % deptype.get_description())
  574. excuses[x].addreason(deptype.get_reason())
  575. def compile_nuninst(target_suite, architectures, nobreakall_arches):
  576. """Compile a nuninst dict from the current testing
  577. :param target_suite: The target suite
  578. :param architectures: List of architectures
  579. :param nobreakall_arches: List of architectures where arch:all packages must be installable
  580. """
  581. nuninst = {}
  582. binaries_t = target_suite.binaries
  583. # for all the architectures
  584. for arch in architectures:
  585. # if it is in the nobreakall ones, check arch-independent packages too
  586. check_archall = arch in nobreakall_arches
  587. # check all the packages for this architecture
  588. nuninst[arch] = set()
  589. packages_t_a = binaries_t[arch]
  590. for pkg_name, pkg_data in packages_t_a.items():
  591. r = target_suite.is_installable(pkg_data.pkg_id)
  592. if not r:
  593. nuninst[arch].add(pkg_name)
  594. # if they are not required, remove architecture-independent packages
  595. nuninst[arch + "+all"] = nuninst[arch].copy()
  596. if not check_archall:
  597. for pkg_name in nuninst[arch + "+all"]:
  598. pkg_data = packages_t_a[pkg_name]
  599. if pkg_data.architecture == 'all':
  600. nuninst[arch].remove(pkg_name)
  601. return nuninst
  602. def find_smooth_updateable_binaries(binaries_to_check,
  603. source_data,
  604. pkg_universe,
  605. target_suite,
  606. binaries_t,
  607. binaries_s,
  608. removals,
  609. smooth_updates):
  610. check = set()
  611. smoothbins = set()
  612. for pkg_id in binaries_to_check:
  613. binary, _, parch = pkg_id
  614. cruftbins = set()
  615. # Not a candidate for smooth up date (newer non-cruft version in unstable)
  616. if binary in binaries_s[parch]:
  617. if binaries_s[parch][binary].source_version == source_data.version:
  618. continue
  619. cruftbins.add(binaries_s[parch][binary].pkg_id)
  620. # Maybe a candidate (cruft or removed binary): check if config allows us to smooth update it.
  621. if 'ALL' in smooth_updates or binaries_t[parch][binary].section in smooth_updates:
  622. # if the package has reverse-dependencies which are
  623. # built from other sources, it's a valid candidate for
  624. # a smooth update. if not, it may still be a valid
  625. # candidate if one if its r-deps is itself a candidate,
  626. # so note it for checking later
  627. rdeps = set(pkg_universe.reverse_dependencies_of(pkg_id))
  628. # We ignore all binaries listed in "removals" as we
  629. # assume they will leave at the same time as the
  630. # given package.
  631. rdeps.difference_update(removals, binaries_to_check)
  632. smooth_update_it = False
  633. if target_suite.any_of_these_are_in_the_suite(rdeps):
  634. combined = set(smoothbins)
  635. combined.add(pkg_id)
  636. for rdep in rdeps:
  637. # each dependency clause has a set of possible
  638. # alternatives that can satisfy that dependency.
  639. # if any of them is outside the set of smoothbins, the
  640. # dependency can be satisfied even if this binary was
  641. # removed, so there is no need to keep it around for a
  642. # smooth update
  643. # if not, only this binary can satisfy the dependency, so
  644. # we should keep it around until the rdep is no longer in
  645. # testing
  646. for dep_clause in pkg_universe.dependencies_of(rdep):
  647. # filter out cruft binaries from unstable, because
  648. # they will not be added to the set of packages that
  649. # will be migrated
  650. if dep_clause - cruftbins <= combined:
  651. smooth_update_it = True
  652. break
  653. if smooth_update_it:
  654. smoothbins = combined
  655. else:
  656. check.add(pkg_id)
  657. # check whether we should perform a smooth update for
  658. # packages which are candidates but do not have r-deps
  659. # outside of the current source
  660. while 1:
  661. found_any = False
  662. for pkg_id in check:
  663. rdeps = pkg_universe.reverse_dependencies_of(pkg_id)
  664. if not rdeps.isdisjoint(smoothbins):
  665. smoothbins.add(pkg_id)
  666. found_any = True
  667. if not found_any:
  668. break
  669. check = [x for x in check if x not in smoothbins]
  670. return smoothbins
  671. def parse_provides(provides_raw, pkg_id=None, logger=None):
  672. parts = apt_pkg.parse_depends(provides_raw, False)
  673. nprov = []
  674. for or_clause in parts:
  675. if len(or_clause) != 1: # pragma: no cover
  676. if logger is not None:
  677. msg = "Ignoring invalid provides in %s: Alternatives [%s]"
  678. logger.warning(msg, str(pkg_id), str(or_clause))
  679. continue
  680. for part in or_clause:
  681. provided, provided_version, op = part
  682. if op != '' and op != '=': # pragma: no cover
  683. if logger is not None:
  684. msg = "Ignoring invalid provides in %s: %s (%s %s)"
  685. logger.warning(msg, str(pkg_id), provided, op, provided_version)
  686. continue
  687. provided = sys.intern(provided)
  688. provided_version = sys.intern(provided_version)
  689. part = (provided, provided_version, sys.intern(op))
  690. nprov.append(part)
  691. return nprov
  692. def compute_item_name(sources_t, sources_s, source_name, parch):
  693. if source_name in sources_t and sources_t[source_name].version == sources_s[source_name].version:
  694. return "%s/%s" % (source_name, parch)
  695. return source_name
  696. def parse_builtusing(builtusing_raw, pkg_id=None, logger=None):
  697. parts = apt_pkg.parse_depends(builtusing_raw, False)
  698. nbu = []
  699. for or_clause in parts:
  700. if len(or_clause) != 1: # pragma: no cover
  701. if logger is not None:
  702. msg = "Ignoring invalid builtusing in %s: Alternatives [%s]"
  703. logger.warning(msg, str(pkg_id), str(or_clause))
  704. continue
  705. for part in or_clause:
  706. bu, bu_version, op = part
  707. if op != '=': # pragma: no cover
  708. if logger is not None:
  709. msg = "Ignoring invalid builtusing in %s: %s (%s %s)"
  710. logger.warning(msg, str(pkg_id), bu, op, bu_version)
  711. continue
  712. bu = sys.intern(bu)
  713. bu_version = sys.intern(bu_version)
  714. part = (bu, bu_version)
  715. nbu.append(part)
  716. return nbu