Devuan deployment of britney2
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

britney.py 72 KiB

14 years ago
14 years ago
9 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548
  1. #!/usr/bin/python3 -u
  2. # -*- coding: utf-8 -*-
  3. # Copyright (C) 2001-2008 Anthony Towns <ajt@debian.org>
  4. # Andreas Barth <aba@debian.org>
  5. # Fabio Tranchitella <kobold@debian.org>
  6. # Copyright (C) 2010-2013 Adam D. Barratt <adsb@debian.org>
  7. # This program is free software; you can redistribute it and/or modify
  8. # it under the terms of the GNU General Public License as published by
  9. # the Free Software Foundation; either version 2 of the License, or
  10. # (at your option) any later version.
  11. # This program is distributed in the hope that it will be useful,
  12. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. # GNU General Public License for more details.
  15. """
  16. = Introduction =
  17. This is the Debian testing updater script, also known as "Britney".
  18. Packages are usually installed into the `testing' distribution after
  19. they have undergone some degree of testing in unstable. The goal of
  20. this software is to do this task in a smart way, allowing testing
  21. to always be fully installable and close to being a release candidate.
  22. Britney's source code is split between two different but related tasks:
  23. the first one is the generation of the update excuses, while the
  24. second tries to update testing with the valid candidates; first
  25. each package alone, then larger and even larger sets of packages
  26. together. Each try is accepted if testing is not more uninstallable
  27. after the update than before.
  28. = Data Loading =
  29. In order to analyze the entire Debian distribution, Britney needs to
  30. load in memory the whole archive: this means more than 10.000 packages
  31. for twelve architectures, as well as the dependency interconnections
  32. between them. For this reason, the memory requirements for running this
  33. software are quite high and at least 1 gigabyte of RAM should be available.
  34. Britney loads the source packages from the `Sources' file and the binary
  35. packages from the `Packages_${arch}' files, where ${arch} is substituted
  36. with the supported architectures. While loading the data, the software
  37. analyzes the dependencies and builds a directed weighted graph in memory
  38. with all the interconnections between the packages (see Britney.read_sources
  39. and Britney.read_binaries).
  40. Other than source and binary packages, Britney loads the following data:
  41. * BugsV, which contains the list of release-critical bugs for a given
  42. version of a source or binary package (see RCBugPolicy.read_bugs).
  43. * Dates, which contains the date of the upload of a given version
  44. of a source package (see Britney.read_dates).
  45. * Urgencies, which contains the urgency of the upload of a given
  46. version of a source package (see AgePolicy._read_urgencies).
  47. * Hints, which contains lists of commands which modify the standard behaviour
  48. of Britney (see Britney.read_hints).
  49. For a more detailed explanation about the format of these files, please read
  50. the documentation of the related methods. The exact meaning of them will be
  51. instead explained in the chapter "Excuses Generation".
  52. = Excuses =
  53. An excuse is a detailed explanation of why a package can or cannot
  54. be updated in the testing distribution from a newer package in
  55. another distribution (like for example unstable). The main purpose
  56. of the excuses is to be written in an HTML file which will be
  57. published over HTTP. The maintainers will be able to parse it manually
  58. or automatically to find the explanation of why their packages have
  59. been updated or not.
  60. == Excuses generation ==
  61. These are the steps (with references to method names) that Britney
  62. does for the generation of the update excuses.
  63. * If a source package is available in testing but it is not
  64. present in unstable and no binary packages in unstable are
  65. built from it, then it is marked for removal.
  66. * Every source package in unstable and testing-proposed-updates,
  67. if already present in testing, is checked for binary-NMUs, new
  68. or dropped binary packages in all the supported architectures
  69. (see Britney.should_upgrade_srcarch). The steps to detect if an
  70. upgrade is needed are:
  71. 1. If there is a `remove' hint for the source package, the package
  72. is ignored: it will be removed and not updated.
  73. 2. For every binary package built from the new source, it checks
  74. for unsatisfied dependencies, new binary packages and updated
  75. binary packages (binNMU), excluding the architecture-independent
  76. ones, and packages not built from the same source.
  77. 3. For every binary package built from the old source, it checks
  78. if it is still built from the new source; if this is not true
  79. and the package is not architecture-independent, the script
  80. removes it from testing.
  81. 4. Finally, if there is something worth doing (eg. a new or updated
  82. binary package) and nothing wrong it marks the source package
  83. as "Valid candidate", or "Not considered" if there is something
  84. wrong which prevented the update.
  85. * Every source package in unstable and testing-proposed-updates is
  86. checked for upgrade (see Britney.should_upgrade_src). The steps
  87. to detect if an upgrade is needed are:
  88. 1. If the source package in testing is more recent the new one
  89. is ignored.
  90. 2. If the source package doesn't exist (is fake), which means that
  91. a binary package refers to it but it is not present in the
  92. `Sources' file, the new one is ignored.
  93. 3. If the package doesn't exist in testing, the urgency of the
  94. upload is ignored and set to the default (actually `low').
  95. 4. If there is a `remove' hint for the source package, the package
  96. is ignored: it will be removed and not updated.
  97. 5. If there is a `block' hint for the source package without an
  98. `unblock` hint or a `block-all source`, the package is ignored.
  99. 6. If there is a `block-udeb' hint for the source package, it will
  100. have the same effect as `block', but may only be cancelled by
  101. a subsequent `unblock-udeb' hint.
  102. 7. If the suite is unstable, the update can go ahead only if the
  103. upload happened more than the minimum days specified by the
  104. urgency of the upload; if this is not true, the package is
  105. ignored as `too-young'. Note that the urgency is sticky, meaning
  106. that the highest urgency uploaded since the previous testing
  107. transition is taken into account.
  108. 8. If the suite is unstable, all the architecture-dependent binary
  109. packages and the architecture-independent ones for the `nobreakall'
  110. architectures have to be built from the source we are considering.
  111. If this is not true, then these are called `out-of-date'
  112. architectures and the package is ignored.
  113. 9. The source package must have at least one binary package, otherwise
  114. it is ignored.
  115. 10. If the suite is unstable, the new source package must have no
  116. release critical bugs which do not also apply to the testing
  117. one. If this is not true, the package is ignored as `buggy'.
  118. 11. If there is a `force' hint for the source package, then it is
  119. updated even if it is marked as ignored from the previous steps.
  120. 12. If the suite is {testing-,}proposed-updates, the source package can
  121. be updated only if there is an explicit approval for it. Unless
  122. a `force' hint exists, the new package must also be available
  123. on all of the architectures for which it has binary packages in
  124. testing.
  125. 13. If the package will be ignored, mark it as "Valid candidate",
  126. otherwise mark it as "Not considered".
  127. * The list of `remove' hints is processed: if the requested source
  128. package is not already being updated or removed and the version
  129. actually in testing is the same specified with the `remove' hint,
  130. it is marked for removal.
  131. * The excuses are sorted by the number of days from the last upload
  132. (days-old) and by name.
  133. * A list of unconsidered excuses (for which the package is not upgraded)
  134. is built. Using this list, all of the excuses depending on them are
  135. marked as invalid "impossible dependencies".
  136. * The excuses are written in an HTML file.
  137. """
  138. import contextlib
  139. import logging
  140. import optparse
  141. import os
  142. import sys
  143. import time
  144. from collections import defaultdict
  145. from functools import reduce
  146. from itertools import chain
  147. from operator import attrgetter
  148. import apt_pkg
  149. from britney2 import SourcePackage, BinaryPackageId, BinaryPackage
  150. from britney2.excusefinder import ExcuseFinder
  151. from britney2.hints import HintParser
  152. from britney2.inputs.suiteloader import DebMirrorLikeSuiteContentLoader, MissingRequiredConfigurationError
  153. from britney2.installability.builder import build_installability_tester
  154. from britney2.installability.solver import InstallabilitySolver
  155. from britney2.migration import MigrationManager
  156. from britney2.migrationitem import MigrationItemFactory
  157. from britney2.policies.policy import (AgePolicy, RCBugPolicy, PiupartsPolicy, BuildDependsPolicy, PolicyEngine,
  158. BlockPolicy, BuiltUsingPolicy, BuiltOnBuilddPolicy)
  159. from britney2.policies.autopkgtest import AutopkgtestPolicy
  160. from britney2.utils import (log_and_format_old_libraries,
  161. read_nuninst, write_nuninst, write_heidi,
  162. format_and_log_uninst, newly_uninst,
  163. write_excuses, write_heidi_delta,
  164. old_libraries, is_nuninst_asgood_generous,
  165. clone_nuninst, compile_nuninst, parse_provides,
  166. MigrationConstraintException,
  167. )
  168. __author__ = 'Fabio Tranchitella and the Debian Release Team'
  169. __version__ = '2.0'
  170. class Britney(object):
  171. """Britney, the Debian testing updater script
  172. This is the script that updates the testing distribution. It is executed
  173. each day after the installation of the updated packages. It generates the
  174. `Packages' files for the testing distribution, but it does so in an
  175. intelligent manner; it tries to avoid any inconsistency and to use only
  176. non-buggy packages.
  177. For more documentation on this script, please read the Developers Reference.
  178. """
  179. HINTS_HELPERS = ("easy", "hint", "remove", "block", "block-udeb", "unblock", "unblock-udeb", "approve",
  180. "remark", "ignore-piuparts", "ignore-rc-bugs", "force-skiptest", "force-badtest")
  181. HINTS_STANDARD = ("urgent", "age-days") + HINTS_HELPERS
  182. # ALL = {"force", "force-hint", "block-all"} | HINTS_STANDARD | registered policy hints (not covered above)
  183. HINTS_ALL = ('ALL')
  184. def __init__(self):
  185. """Class constructor
  186. This method initializes and populates the data lists, which contain all
  187. the information needed by the other methods of the class.
  188. """
  189. # setup logging - provide the "short level name" (i.e. INFO -> I) that
  190. # we used to use prior to using the logging module.
  191. old_factory = logging.getLogRecordFactory()
  192. short_level_mapping = {
  193. 'CRITICAL': 'F',
  194. 'INFO': 'I',
  195. 'WARNING': 'W',
  196. 'ERROR': 'E',
  197. 'DEBUG': 'N',
  198. }
  199. def record_factory(*args, **kwargs): # pragma: no cover
  200. record = old_factory(*args, **kwargs)
  201. try:
  202. record.shortlevelname = short_level_mapping[record.levelname]
  203. except KeyError:
  204. record.shortlevelname = record.levelname
  205. return record
  206. logging.setLogRecordFactory(record_factory)
  207. logging.basicConfig(format='{shortlevelname}: [{asctime}] - {message}',
  208. style='{',
  209. datefmt="%Y-%m-%dT%H:%M:%S%z",
  210. stream=sys.stdout,
  211. )
  212. self.logger = logging.getLogger()
  213. # Logger for "upgrade_output"; the file handler will be attached later when
  214. # we are ready to open the file.
  215. self.output_logger = logging.getLogger('britney2.output.upgrade_output')
  216. self.output_logger.setLevel(logging.INFO)
  217. # initialize the apt_pkg back-end
  218. apt_pkg.init()
  219. # parse the command line arguments
  220. self._policy_engine = PolicyEngine()
  221. self.suite_info = None # Initialized during __parse_arguments
  222. self.__parse_arguments()
  223. self.all_selected = []
  224. self.excuses = {}
  225. self.upgrade_me = []
  226. if self.options.nuninst_cache:
  227. self.logger.info("Not building the list of non-installable packages, as requested")
  228. if self.options.print_uninst:
  229. nuninst = read_nuninst(self.options.noninst_status,
  230. self.options.architectures)
  231. print('* summary')
  232. print('\n'.join('%4d %s' % (len(nuninst[x]), x) for x in self.options.architectures))
  233. return
  234. try:
  235. constraints_file = os.path.join(self.options.static_input_dir, 'constraints')
  236. faux_packages = os.path.join(self.options.static_input_dir, 'faux-packages')
  237. except AttributeError:
  238. self.logger.info("The static_input_dir option is not set")
  239. constraints_file = None
  240. faux_packages = None
  241. if faux_packages is not None and os.path.exists(faux_packages):
  242. self.logger.info("Loading faux packages from %s", faux_packages)
  243. self._load_faux_packages(faux_packages)
  244. elif faux_packages is not None:
  245. self.logger.info("No Faux packages as %s does not exist", faux_packages)
  246. if constraints_file is not None and os.path.exists(constraints_file):
  247. self.logger.info("Loading constraints from %s", constraints_file)
  248. self.constraints = self._load_constraints(constraints_file)
  249. else:
  250. if constraints_file is not None:
  251. self.logger.info("No constraints as %s does not exist", constraints_file)
  252. self.constraints = {
  253. 'keep-installable': [],
  254. }
  255. self.logger.info("Compiling Installability tester")
  256. self.pkg_universe, self._inst_tester = build_installability_tester(self.suite_info, self.options.architectures)
  257. target_suite = self.suite_info.target_suite
  258. target_suite.inst_tester = self._inst_tester
  259. self.allow_uninst = {}
  260. for arch in self.options.architectures:
  261. self.allow_uninst[arch] = set()
  262. self._migration_item_factory = MigrationItemFactory(self.suite_info)
  263. self._hint_parser = HintParser(self._migration_item_factory)
  264. self._migration_manager = MigrationManager(self.options, self.suite_info, self.all_binaries, self.pkg_universe,
  265. self.constraints, self.allow_uninst, self._migration_item_factory)
  266. if not self.options.nuninst_cache:
  267. self.logger.info("Building the list of non-installable packages for the full archive")
  268. self._inst_tester.compute_installability()
  269. nuninst = compile_nuninst(target_suite,
  270. self.options.architectures,
  271. self.options.nobreakall_arches)
  272. self.nuninst_orig = nuninst
  273. for arch in self.options.architectures:
  274. self.logger.info("> Found %d non-installable packages", len(nuninst[arch]))
  275. if self.options.print_uninst:
  276. self.nuninst_arch_report(nuninst, arch)
  277. if self.options.print_uninst:
  278. print('* summary')
  279. print('\n'.join(map(lambda x: '%4d %s' % (len(nuninst[x]), x), self.options.architectures)))
  280. return
  281. else:
  282. write_nuninst(self.options.noninst_status, nuninst)
  283. stats = self._inst_tester.compute_stats()
  284. self.logger.info("> Installability tester statistics (per architecture)")
  285. for arch in self.options.architectures:
  286. arch_stat = stats[arch]
  287. self.logger.info("> %s", arch)
  288. for stat in arch_stat.stat_summary():
  289. self.logger.info("> - %s", stat)
  290. else:
  291. self.logger.info("Loading uninstallability counters from cache")
  292. self.nuninst_orig = read_nuninst(self.options.noninst_status,
  293. self.options.architectures)
  294. # nuninst_orig may get updated during the upgrade process
  295. self.nuninst_orig_save = clone_nuninst(self.nuninst_orig, architectures=self.options.architectures)
  296. self._policy_engine.register_policy_hints(self._hint_parser)
  297. try:
  298. self.read_hints(self.options.hintsdir)
  299. except AttributeError:
  300. self.read_hints(os.path.join(self.suite_info['unstable'].path, 'Hints'))
  301. self._policy_engine.initialise(self, self.hints)
  302. def __parse_arguments(self):
  303. """Parse the command line arguments
  304. This method parses and initializes the command line arguments.
  305. While doing so, it preprocesses some of the options to be converted
  306. in a suitable form for the other methods of the class.
  307. """
  308. # initialize the parser
  309. parser = optparse.OptionParser(version="%prog")
  310. parser.add_option("-v", "", action="count", dest="verbose", help="enable verbose output")
  311. parser.add_option("-c", "--config", action="store", dest="config", default="/etc/britney.conf",
  312. help="path for the configuration file")
  313. parser.add_option("", "--architectures", action="store", dest="architectures", default=None,
  314. help="override architectures from configuration file")
  315. parser.add_option("", "--actions", action="store", dest="actions", default=None,
  316. help="override the list of actions to be performed")
  317. parser.add_option("", "--hints", action="store", dest="hints", default=None,
  318. help="additional hints, separated by semicolons")
  319. parser.add_option("", "--hint-tester", action="store_true", dest="hint_tester", default=None,
  320. help="provide a command line interface to test hints")
  321. parser.add_option("", "--dry-run", action="store_true", dest="dry_run", default=False,
  322. help="disable all outputs to the testing directory")
  323. parser.add_option("", "--nuninst-cache", action="store_true", dest="nuninst_cache", default=False,
  324. help="do not build the non-installability status, use the cache from file")
  325. parser.add_option("", "--print-uninst", action="store_true", dest="print_uninst", default=False,
  326. help="just print a summary of uninstallable packages")
  327. parser.add_option("", "--compute-migrations", action="store_true", dest="compute_migrations", default=True,
  328. help="Compute which packages can migrate (the default)")
  329. parser.add_option("", "--no-compute-migrations", action="store_false", dest="compute_migrations",
  330. help="Do not compute which packages can migrate.")
  331. parser.add_option("", "--series", action="store", dest="series", default='',
  332. help="set distribution series name")
  333. (self.options, self.args) = parser.parse_args()
  334. if self.options.verbose:
  335. self.logger.setLevel(logging.INFO)
  336. else:
  337. self.logger.setLevel(logging.WARNING)
  338. # TODO: Define a more obvious toggle for debug information
  339. try: # pragma: no cover
  340. if int(os.environ.get('BRITNEY_DEBUG', '0')):
  341. self.logger.setLevel(logging.DEBUG)
  342. except ValueError: # pragma: no cover
  343. pass
  344. # integrity checks
  345. if self.options.nuninst_cache and self.options.print_uninst: # pragma: no cover
  346. self.logger.error("nuninst_cache and print_uninst are mutually exclusive!")
  347. sys.exit(1)
  348. # if the configuration file exists, then read it and set the additional options
  349. elif not os.path.isfile(self.options.config): # pragma: no cover
  350. self.logger.error("Unable to read the configuration file (%s), exiting!", self.options.config)
  351. sys.exit(1)
  352. # minimum days for unstable-testing transition and the list of hints
  353. # are handled as an ad-hoc case
  354. MINDAYS = {}
  355. self.HINTS = {'command-line': self.HINTS_ALL}
  356. with open(self.options.config, encoding='utf-8') as config:
  357. for line in config:
  358. if '=' in line and not line.strip().startswith('#'):
  359. k, v = line.split('=', 1)
  360. k = k.strip()
  361. v = v.strip()
  362. if k.startswith("MINDAYS_"):
  363. MINDAYS[k.split("_")[1].lower()] = int(v)
  364. elif k.startswith("HINTS_"):
  365. self.HINTS[k.split("_")[1].lower()] = \
  366. reduce(lambda x, y: x+y, [
  367. hasattr(self, "HINTS_" + i) and
  368. getattr(self, "HINTS_" + i) or
  369. (i,) for i in v.split()])
  370. elif not hasattr(self.options, k.lower()) or \
  371. not getattr(self.options, k.lower()):
  372. setattr(self.options, k.lower(), v)
  373. if hasattr(self.options, 'components'): # pragma: no cover
  374. self.logger.error("The COMPONENTS configuration has been removed.")
  375. self.logger.error("Britney will read the value from the Release file automatically")
  376. sys.exit(1)
  377. suite_loader = DebMirrorLikeSuiteContentLoader(self.options)
  378. try:
  379. self.suite_info = suite_loader.load_suites()
  380. except MissingRequiredConfigurationError as e: # pragma: no cover
  381. self.logger.error("Could not load the suite content due to missing configuration: %s", str(e))
  382. sys.exit(1)
  383. self.all_binaries = suite_loader.all_binaries()
  384. self.options.components = suite_loader.components
  385. self.options.architectures = suite_loader.architectures
  386. self.options.nobreakall_arches = suite_loader.nobreakall_arches
  387. self.options.outofsync_arches = suite_loader.outofsync_arches
  388. self.options.break_arches = suite_loader.break_arches
  389. self.options.new_arches = suite_loader.new_arches
  390. if self.options.series == '':
  391. self.options.series = self.suite_info.target_suite.name
  392. if not hasattr(self.options, "heidi_delta_output"):
  393. self.options.heidi_delta_output = self.options.heidi_output + "Delta"
  394. self.options.smooth_updates = self.options.smooth_updates.split()
  395. if not hasattr(self.options, 'ignore_cruft') or \
  396. self.options.ignore_cruft == "0":
  397. self.options.ignore_cruft = False
  398. if not hasattr(self.options, 'check_consistency_level'):
  399. self.options.check_consistency_level = 2
  400. else:
  401. self.options.check_consistency_level = int(self.options.check_consistency_level)
  402. if not hasattr(self.options, 'adt_retry_url_mech'):
  403. self.options.adt_retry_url_mech = ''
  404. self._policy_engine.add_policy(RCBugPolicy(self.options, self.suite_info))
  405. self._policy_engine.add_policy(PiupartsPolicy(self.options, self.suite_info))
  406. if getattr(self.options, 'adt_enable') == 'yes':
  407. self._policy_engine.add_policy(AutopkgtestPolicy(self.options, self.suite_info))
  408. self._policy_engine.add_policy(AgePolicy(self.options, self.suite_info, MINDAYS))
  409. self._policy_engine.add_policy(BuildDependsPolicy(self.options, self.suite_info))
  410. self._policy_engine.add_policy(BlockPolicy(self.options, self.suite_info))
  411. self._policy_engine.add_policy(BuiltUsingPolicy(self.options, self.suite_info))
  412. if getattr(self.options, 'check_buildd', 'no') == 'yes':
  413. self._policy_engine.add_policy(BuiltOnBuilddPolicy(self.options, self.suite_info))
  414. @property
  415. def hints(self):
  416. return self._hint_parser.hints
  417. def _load_faux_packages(self, faux_packages_file):
  418. """Loads fake packages
  419. In rare cases, it is useful to create a "fake" package that can be used to satisfy
  420. dependencies. This is usually needed for packages that are not shipped directly
  421. on this mirror but is a prerequisite for using this mirror (e.g. some vendors provide
  422. non-distributable "setup" packages and contrib/non-free packages depend on these).
  423. :param faux_packages_file: Path to the file containing the fake package definitions
  424. """
  425. tag_file = apt_pkg.TagFile(faux_packages_file)
  426. get_field = tag_file.section.get
  427. step = tag_file.step
  428. no = 0
  429. pri_source_suite = self.suite_info.primary_source_suite
  430. target_suite = self.suite_info.target_suite
  431. while step():
  432. no += 1
  433. pkg_name = get_field('Package', None)
  434. if pkg_name is None: # pragma: no cover
  435. raise ValueError("Missing Package field in paragraph %d (file %s)" % (no, faux_packages_file))
  436. pkg_name = sys.intern(pkg_name)
  437. version = sys.intern(get_field('Version', '1.0-1'))
  438. provides_raw = get_field('Provides')
  439. archs_raw = get_field('Architecture', None)
  440. component = get_field('Component', 'non-free')
  441. if archs_raw:
  442. archs = archs_raw.split()
  443. else:
  444. archs = self.options.architectures
  445. faux_section = 'faux'
  446. if component != 'main':
  447. faux_section = "%s/faux" % component
  448. src_data = SourcePackage(version,
  449. sys.intern(faux_section),
  450. set(),
  451. None,
  452. True,
  453. None,
  454. None,
  455. [],
  456. [],
  457. )
  458. target_suite.sources[pkg_name] = src_data
  459. pri_source_suite.sources[pkg_name] = src_data
  460. for arch in archs:
  461. pkg_id = BinaryPackageId(pkg_name, version, arch)
  462. if provides_raw:
  463. provides = parse_provides(provides_raw, pkg_id=pkg_id, logger=self.logger)
  464. else:
  465. provides = []
  466. bin_data = BinaryPackage(version,
  467. faux_section,
  468. pkg_name,
  469. version,
  470. arch,
  471. get_field('Multi-Arch'),
  472. None,
  473. None,
  474. provides,
  475. False,
  476. pkg_id,
  477. [],
  478. )
  479. src_data.binaries.add(pkg_id)
  480. target_suite.binaries[arch][pkg_name] = bin_data
  481. pri_source_suite.binaries[arch][pkg_name] = bin_data
  482. # register provided packages with the target suite provides table
  483. for provided_pkg, provided_version, _ in bin_data.provides:
  484. target_suite.provides_table[arch][provided_pkg].add((pkg_name, provided_version))
  485. self.all_binaries[pkg_id] = bin_data
  486. def _load_constraints(self, constraints_file):
  487. """Loads configurable constraints
  488. The constraints file can contain extra rules that Britney should attempt
  489. to satisfy. Examples can be "keep package X in testing and ensure it is
  490. installable".
  491. :param constraints_file: Path to the file containing the constraints
  492. """
  493. tag_file = apt_pkg.TagFile(constraints_file)
  494. get_field = tag_file.section.get
  495. step = tag_file.step
  496. no = 0
  497. faux_version = sys.intern('1')
  498. faux_section = sys.intern('faux')
  499. keep_installable = []
  500. constraints = {
  501. 'keep-installable': keep_installable
  502. }
  503. pri_source_suite = self.suite_info.primary_source_suite
  504. target_suite = self.suite_info.target_suite
  505. while step():
  506. no += 1
  507. pkg_name = get_field('Fake-Package-Name', None)
  508. if pkg_name is None: # pragma: no cover
  509. raise ValueError("Missing Fake-Package-Name field in paragraph %d (file %s)" % (no, constraints_file))
  510. pkg_name = sys.intern(pkg_name)
  511. def mandatory_field(x):
  512. v = get_field(x, None)
  513. if v is None: # pragma: no cover
  514. raise ValueError("Missing %s field for %s (file %s)" % (x, pkg_name, constraints_file))
  515. return v
  516. constraint = mandatory_field('Constraint')
  517. if constraint not in {'present-and-installable'}: # pragma: no cover
  518. raise ValueError("Unsupported constraint %s for %s (file %s)" % (constraint, pkg_name, constraints_file))
  519. self.logger.info(" - constraint %s", pkg_name)
  520. pkg_list = [x.strip() for x in mandatory_field('Package-List').split("\n")
  521. if x.strip() != '' and not x.strip().startswith("#")]
  522. src_data = SourcePackage(faux_version,
  523. faux_section,
  524. set(),
  525. None,
  526. True,
  527. None,
  528. None,
  529. [],
  530. [],
  531. )
  532. target_suite.sources[pkg_name] = src_data
  533. pri_source_suite.sources[pkg_name] = src_data
  534. keep_installable.append(pkg_name)
  535. for arch in self.options.architectures:
  536. deps = []
  537. for pkg_spec in pkg_list:
  538. s = pkg_spec.split(None, 1)
  539. if len(s) == 1:
  540. deps.append(s[0])
  541. else:
  542. pkg, arch_res = s
  543. if not (arch_res.startswith('[') and arch_res.endswith(']')): # pragma: no cover
  544. raise ValueError("Invalid arch-restriction on %s - should be [arch1 arch2] (for %s file %s)"
  545. % (pkg, pkg_name, constraints_file))
  546. arch_res = arch_res[1:-1].split()
  547. if not arch_res: # pragma: no cover
  548. msg = "Empty arch-restriction for %s: Uses comma or negation (for %s file %s)"
  549. raise ValueError(msg % (pkg, pkg_name, constraints_file))
  550. for a in arch_res:
  551. if a == arch:
  552. deps.append(pkg)
  553. elif ',' in a or '!' in a: # pragma: no cover
  554. msg = "Invalid arch-restriction for %s: Uses comma or negation (for %s file %s)"
  555. raise ValueError(msg % (pkg, pkg_name, constraints_file))
  556. pkg_id = BinaryPackageId(pkg_name, faux_version, arch)
  557. bin_data = BinaryPackage(faux_version,
  558. faux_section,
  559. pkg_name,
  560. faux_version,
  561. arch,
  562. 'no',
  563. ', '.join(deps),
  564. None,
  565. [],
  566. False,
  567. pkg_id,
  568. [],
  569. )
  570. src_data.binaries.add(pkg_id)
  571. target_suite.binaries[arch][pkg_name] = bin_data
  572. pri_source_suite.binaries[arch][pkg_name] = bin_data
  573. self.all_binaries[pkg_id] = bin_data
  574. return constraints
  575. # Data reading/writing methods
  576. # ----------------------------
  577. def read_hints(self, hintsdir):
  578. """Read the hint commands from the specified directory
  579. The hint commands are read from the files contained in the directory
  580. specified by the `hintsdir' parameter.
  581. The names of the files have to be the same as the authorized users
  582. for the hints.
  583. The file contains rows with the format:
  584. <command> <package-name>[/<version>]
  585. The method returns a dictionary where the key is the command, and
  586. the value is the list of affected packages.
  587. """
  588. for who in self.HINTS.keys():
  589. if who == 'command-line':
  590. lines = self.options.hints and self.options.hints.split(';') or ()
  591. filename = '<cmd-line>'
  592. self._hint_parser.parse_hints(who, self.HINTS[who], filename, lines)
  593. else:
  594. filename = os.path.join(hintsdir, who)
  595. if not os.path.isfile(filename):
  596. self.logger.error("Cannot read hints list from %s, no such file!", filename)
  597. continue
  598. self.logger.info("Loading hints list from %s", filename)
  599. with open(filename, encoding='utf-8') as f:
  600. self._hint_parser.parse_hints(who, self.HINTS[who], filename, f)
  601. hints = self._hint_parser.hints
  602. for x in ["block", "block-all", "block-udeb", "unblock", "unblock-udeb", "force", "urgent", "remove", "age-days"]:
  603. z = defaultdict(dict)
  604. for hint in hints[x]:
  605. package = hint.package
  606. architecture = hint.architecture
  607. key = (hint, hint.user)
  608. if package in z and architecture in z[package] and z[package][architecture] != key:
  609. hint2 = z[package][architecture][0]
  610. if x in ['unblock', 'unblock-udeb']:
  611. if apt_pkg.version_compare(hint2.version, hint.version) < 0:
  612. # This hint is for a newer version, so discard the old one
  613. self.logger.warning("Overriding %s[%s] = ('%s', '%s', '%s') with ('%s', '%s', '%s')",
  614. x, package, hint2.version, hint2.architecture,
  615. hint2.user, hint.version, hint.architecture, hint.user)
  616. hint2.set_active(False)
  617. else:
  618. # This hint is for an older version, so ignore it in favour of the new one
  619. self.logger.warning("Ignoring %s[%s] = ('%s', '%s', '%s'), ('%s', '%s', '%s') is higher or equal",
  620. x, package, hint.version, hint.architecture, hint.user,
  621. hint2.version, hint2.architecture, hint2.user)
  622. hint.set_active(False)
  623. else:
  624. self.logger.warning("Overriding %s[%s] = ('%s', '%s') with ('%s', '%s')",
  625. x, package, hint2.user, hint2, hint.user, hint)
  626. hint2.set_active(False)
  627. z[package][architecture] = key
  628. for hint in hints['allow-uninst']:
  629. if hint.architecture == 'source':
  630. for arch in self.options.architectures:
  631. self.allow_uninst[arch].add(hint.package)
  632. else:
  633. self.allow_uninst[hint.architecture].add(hint.package)
  634. # Sanity check the hints hash
  635. if len(hints["block"]) == 0 and len(hints["block-udeb"]) == 0:
  636. self.logger.warning("WARNING: No block hints at all, not even udeb ones!")
  637. def write_excuses(self):
  638. """Produce and write the update excuses
  639. This method handles the update excuses generation: the packages are
  640. looked at to determine whether they are valid candidates. For the details
  641. of this procedure, please refer to the module docstring.
  642. """
  643. self.logger.info("Update Excuses generation started")
  644. mi_factory = self._migration_item_factory
  645. excusefinder = ExcuseFinder(self.options, self.suite_info, self.all_binaries,
  646. self.pkg_universe, self._policy_engine, mi_factory, self.hints)
  647. excuses, upgrade_me = excusefinder.find_actionable_excuses()
  648. self.excuses = excuses
  649. # sort the list of candidates
  650. self.upgrade_me = sorted(upgrade_me)
  651. old_lib_removals = old_libraries(mi_factory, self.suite_info, self.options.outofsync_arches)
  652. self.upgrade_me.extend(old_lib_removals)
  653. self.output_logger.info("List of old libraries added to upgrade_me (%d):", len(old_lib_removals))
  654. log_and_format_old_libraries(self.output_logger, old_lib_removals)
  655. # write excuses to the output file
  656. if not self.options.dry_run:
  657. self.logger.info("> Writing Excuses to %s", self.options.excuses_output)
  658. sorted_excuses = sorted(excuses.values(), key=lambda x: x.sortkey())
  659. write_excuses(sorted_excuses, self.options.excuses_output,
  660. output_format="legacy-html")
  661. if hasattr(self.options, 'excuses_yaml_output'):
  662. self.logger.info("> Writing YAML Excuses to %s", self.options.excuses_yaml_output)
  663. write_excuses(sorted_excuses, self.options.excuses_yaml_output,
  664. output_format="yaml")
  665. self.logger.info("Update Excuses generation completed")
  666. # Upgrade run
  667. # -----------
  668. def eval_nuninst(self, nuninst, original=None):
  669. """Return a string which represents the uninstallability counters
  670. This method returns a string which represents the uninstallability
  671. counters reading the uninstallability statistics `nuninst` and, if
  672. present, merging the results with the `original` one.
  673. An example of the output string is:
  674. 1+2: i-0:a-0:a-0:h-0:i-1:m-0:m-0:p-0:a-0:m-0:s-2:s-0
  675. where the first part is the number of broken packages in non-break
  676. architectures + the total number of broken packages for all the
  677. architectures.
  678. """
  679. res = []
  680. total = 0
  681. totalbreak = 0
  682. for arch in self.options.architectures:
  683. if arch in nuninst:
  684. n = len(nuninst[arch])
  685. elif original and arch in original:
  686. n = len(original[arch])
  687. else:
  688. continue
  689. if arch in self.options.break_arches:
  690. totalbreak = totalbreak + n
  691. else:
  692. total = total + n
  693. res.append("%s-%d" % (arch[0], n))
  694. return "%d+%d: %s" % (total, totalbreak, ":".join(res))
  695. def iter_packages(self, packages, selected, nuninst=None):
  696. """Iter on the list of actions and apply them one-by-one
  697. This method applies the changes from `packages` to testing, checking the uninstallability
  698. counters for every action performed. If the action does not improve them, it is reverted.
  699. The method returns the new uninstallability counters and the remaining actions if the
  700. final result is successful, otherwise (None, []).
  701. """
  702. group_info = {}
  703. rescheduled_packages = packages
  704. maybe_rescheduled_packages = []
  705. output_logger = self.output_logger
  706. solver = InstallabilitySolver(self.pkg_universe, self._inst_tester)
  707. mm = self._migration_manager
  708. target_suite = self.suite_info.target_suite
  709. for y in sorted((y for y in packages), key=attrgetter('uvname')):
  710. try:
  711. _, updates, rms, _ = mm.compute_groups(y)
  712. result = (y, frozenset(updates), frozenset(rms))
  713. group_info[y] = result
  714. except MigrationConstraintException as e:
  715. rescheduled_packages.remove(y)
  716. output_logger.info("not adding package to list: %s", (y.package))
  717. output_logger.info(" got exception: %s" % (repr(e)))
  718. if nuninst:
  719. nuninst_orig = nuninst
  720. else:
  721. nuninst_orig = self.nuninst_orig
  722. nuninst_last_accepted = nuninst_orig
  723. output_logger.info("recur: [] %s %d/0", ",".join(x.uvname for x in selected), len(packages))
  724. while rescheduled_packages:
  725. groups = {group_info[x] for x in rescheduled_packages}
  726. worklist = solver.solve_groups(groups)
  727. rescheduled_packages = []
  728. worklist.reverse()
  729. while worklist:
  730. comp = worklist.pop()
  731. comp_name = ' '.join(item.uvname for item in comp)
  732. output_logger.info("trying: %s" % comp_name)
  733. with mm.start_transaction() as transaction:
  734. accepted = False
  735. try:
  736. accepted, nuninst_after, failed_arch, new_cruft = mm.migrate_items_to_target_suite(
  737. comp,
  738. nuninst_last_accepted
  739. )
  740. if accepted:
  741. selected.extend(comp)
  742. transaction.commit()
  743. output_logger.info("accepted: %s", comp_name)
  744. output_logger.info(" ori: %s", self.eval_nuninst(nuninst_orig))
  745. output_logger.info(" pre: %s", self.eval_nuninst(nuninst_last_accepted))
  746. output_logger.info(" now: %s", self.eval_nuninst(nuninst_after))
  747. if new_cruft:
  748. output_logger.info(
  749. " added new cruft items to list: %s",
  750. " ".join(x.uvname for x in new_cruft))
  751. if len(selected) <= 20:
  752. output_logger.info(" all: %s", " ".join(x.uvname for x in selected))
  753. else:
  754. output_logger.info(" most: (%d) .. %s",
  755. len(selected),
  756. " ".join(x.uvname for x in selected[-20:]))
  757. if self.options.check_consistency_level >= 3:
  758. target_suite.check_suite_source_pkg_consistency('iter_packages after commit')
  759. nuninst_last_accepted = nuninst_after
  760. for cruft_item in new_cruft:
  761. try:
  762. _, updates, rms, _ = mm.compute_groups(cruft_item)
  763. result = (cruft_item, frozenset(updates), frozenset(rms))
  764. group_info[cruft_item] = result
  765. worklist.append([cruft_item])
  766. except MigrationConstraintException as e:
  767. output_logger.info(
  768. " got exception adding cruft item %s to list: %s" %
  769. (cruft_item.uvname, repr(e)))
  770. rescheduled_packages.extend(maybe_rescheduled_packages)
  771. maybe_rescheduled_packages.clear()
  772. else:
  773. transaction.rollback()
  774. broken = sorted(b for b in nuninst_after[failed_arch]
  775. if b not in nuninst_last_accepted[failed_arch])
  776. compare_nuninst = None
  777. if any(item for item in comp if item.architecture != 'source'):
  778. compare_nuninst = nuninst_last_accepted
  779. # NB: try_migration already reverted this for us, so just print the results and move on
  780. output_logger.info("skipped: %s (%d, %d, %d)",
  781. comp_name,
  782. len(rescheduled_packages),
  783. len(maybe_rescheduled_packages),
  784. len(worklist)
  785. )
  786. output_logger.info(" got: %s", self.eval_nuninst(nuninst_after, compare_nuninst))
  787. output_logger.info(" * %s: %s", failed_arch, ", ".join(broken))
  788. if self.options.check_consistency_level >= 3:
  789. target_suite.check_suite_source_pkg_consistency('iter_package after rollback (not accepted)')
  790. except MigrationConstraintException as e:
  791. transaction.rollback()
  792. output_logger.info("skipped: %s (%d, %d, %d)",
  793. comp_name,
  794. len(rescheduled_packages),
  795. len(maybe_rescheduled_packages),
  796. len(worklist)
  797. )
  798. output_logger.info(" got exception: %s" % (repr(e)))
  799. if self.options.check_consistency_level >= 3:
  800. target_suite.check_suite_source_pkg_consistency(
  801. 'iter_package after rollback (MigrationConstraintException)')
  802. if not accepted:
  803. if len(comp) > 1:
  804. output_logger.info(" - splitting the component into single items and retrying them")
  805. worklist.extend([item] for item in comp)
  806. else:
  807. maybe_rescheduled_packages.append(comp[0])
  808. output_logger.info(" finish: [%s]", ",".join(x.uvname for x in selected))
  809. output_logger.info("endloop: %s", self.eval_nuninst(self.nuninst_orig))
  810. output_logger.info(" now: %s", self.eval_nuninst(nuninst_last_accepted))
  811. format_and_log_uninst(output_logger,
  812. self.options.architectures,
  813. newly_uninst(self.nuninst_orig, nuninst_last_accepted)
  814. )
  815. output_logger.info("")
  816. return (nuninst_last_accepted, maybe_rescheduled_packages)
  817. def do_all(self, hinttype=None, init=None, actions=None):
  818. """Testing update runner
  819. This method tries to update testing checking the uninstallability
  820. counters before and after the actions to decide if the update was
  821. successful or not.
  822. """
  823. selected = []
  824. if actions:
  825. upgrade_me = actions[:]
  826. else:
  827. upgrade_me = self.upgrade_me[:]
  828. nuninst_start = self.nuninst_orig
  829. output_logger = self.output_logger
  830. target_suite = self.suite_info.target_suite
  831. # these are special parameters for hints processing
  832. force = False
  833. recurse = True
  834. nuninst_end = None
  835. extra = []
  836. mm = self._migration_manager
  837. if hinttype == "easy" or hinttype == "force-hint":
  838. force = hinttype == "force-hint"
  839. recurse = False
  840. # if we have a list of initial packages, check them
  841. if init:
  842. for x in init:
  843. if x not in upgrade_me:
  844. output_logger.warning("failed: %s is not a valid candidate (or it already migrated)", x.uvname)
  845. return None
  846. selected.append(x)
  847. upgrade_me.remove(x)
  848. output_logger.info("start: %s", self.eval_nuninst(nuninst_start))
  849. output_logger.info("orig: %s", self.eval_nuninst(nuninst_start))
  850. if init and not force:
  851. # We will need to be able to roll back (e.g. easy or a "hint"-hint)
  852. _start_transaction = mm.start_transaction
  853. else:
  854. # No "outer" transaction needed as we will never need to rollback
  855. # (e.g. "force-hint" or a regular "main run"). Emulate the start_transaction
  856. # call from the MigrationManager, so the rest of the code follows the
  857. # same flow regardless of whether we need the transaction or not.
  858. @contextlib.contextmanager
  859. def _start_transaction():
  860. yield None
  861. with _start_transaction() as transaction:
  862. if init:
  863. # init => a hint (e.g. "easy") - so do the hint run
  864. (_, nuninst_end, _, new_cruft) = mm.migrate_items_to_target_suite(selected,
  865. self.nuninst_orig,
  866. stop_on_first_regression=False)
  867. if recurse:
  868. # Ensure upgrade_me and selected do not overlap, if we
  869. # follow-up with a recurse ("hint"-hint).
  870. upgrade_me = [x for x in upgrade_me if x not in set(selected)]
  871. else:
  872. # On non-recursive hints check for cruft and purge it proactively in case it "fixes" the hint.
  873. cruft = [x for x in upgrade_me if x.is_cruft_removal]
  874. if new_cruft:
  875. output_logger.info(
  876. "Change added new cruft items to list: %s",
  877. " ".join(x.uvname for x in new_cruft))
  878. cruft.extend(new_cruft)
  879. if cruft:
  880. output_logger.info("Checking if changes enables cruft removal")
  881. (nuninst_end, remaining_cruft) = self.iter_packages(cruft,
  882. selected,
  883. nuninst=nuninst_end)
  884. output_logger.info("Removed %d of %d cruft item(s) after the changes",
  885. len(cruft) - len(remaining_cruft), len(cruft))
  886. new_cruft.difference_update(remaining_cruft)
  887. # Add new cruft items regardless of whether we recurse. A future run might clean
  888. # them for us.
  889. upgrade_me.extend(new_cruft)
  890. if recurse:
  891. # Either the main run or the recursive run of a "hint"-hint.
  892. (nuninst_end, extra) = self.iter_packages(upgrade_me,
  893. selected,
  894. nuninst=nuninst_end)
  895. nuninst_end_str = self.eval_nuninst(nuninst_end)
  896. if not recurse:
  897. # easy or force-hint
  898. output_logger.info("easy: %s", nuninst_end_str)
  899. if not force:
  900. format_and_log_uninst(self.output_logger,
  901. self.options.architectures,
  902. newly_uninst(nuninst_start, nuninst_end)
  903. )
  904. if force:
  905. # Force implies "unconditionally better"
  906. better = True
  907. else:
  908. break_arches = set(self.options.break_arches)
  909. if all(x.architecture in break_arches for x in selected):
  910. # If we only migrated items from break-arches, then we
  911. # do not allow any regressions on these architectures.
  912. # This usually only happens with hints
  913. break_arches = set()
  914. better = is_nuninst_asgood_generous(self.constraints,
  915. self.allow_uninst,
  916. self.options.architectures,
  917. self.nuninst_orig,
  918. nuninst_end,
  919. break_arches)
  920. if better:
  921. # Result accepted either by force or by being better than the original result.
  922. output_logger.info("final: %s", ",".join(sorted(x.uvname for x in selected)))
  923. output_logger.info("start: %s", self.eval_nuninst(nuninst_start))
  924. output_logger.info(" orig: %s", self.eval_nuninst(self.nuninst_orig))
  925. output_logger.info(" end: %s", nuninst_end_str)
  926. if force:
  927. broken = newly_uninst(nuninst_start, nuninst_end)
  928. if broken:
  929. output_logger.warning("force breaks:")
  930. format_and_log_uninst(self.output_logger,
  931. self.options.architectures,
  932. broken,
  933. loglevel=logging.WARNING,
  934. )
  935. else:
  936. output_logger.info("force did not break any packages")
  937. output_logger.info("SUCCESS (%d/%d)", len(actions or self.upgrade_me), len(extra))
  938. self.nuninst_orig = nuninst_end
  939. self.all_selected += selected
  940. if transaction:
  941. transaction.commit()
  942. if self.options.check_consistency_level >= 2:
  943. target_suite.check_suite_source_pkg_consistency('do_all after commit')
  944. if not actions:
  945. if recurse:
  946. self.upgrade_me = extra
  947. else:
  948. self.upgrade_me = [x for x in self.upgrade_me if x not in set(selected)]
  949. else:
  950. output_logger.info("FAILED\n")
  951. if not transaction:
  952. # if we 'FAILED', but we cannot rollback, we will probably
  953. # leave a broken state behind
  954. # this should not happen
  955. raise AssertionError("do_all FAILED but no transaction to rollback")
  956. transaction.rollback()
  957. if self.options.check_consistency_level >= 2:
  958. target_suite.check_suite_source_pkg_consistency('do_all after rollback')
  959. output_logger.info("")
  960. def assert_nuninst_is_correct(self):
  961. self.logger.info("> Update complete - Verifying non-installability counters")
  962. cached_nuninst = self.nuninst_orig
  963. self._inst_tester.compute_installability()
  964. computed_nuninst = compile_nuninst(self.suite_info.target_suite,
  965. self.options.architectures,
  966. self.options.nobreakall_arches)
  967. if cached_nuninst != computed_nuninst: # pragma: no cover
  968. only_on_break_archs = True
  969. self.logger.error("==================== NUNINST OUT OF SYNC =========================")
  970. for arch in self.options.architectures:
  971. expected_nuninst = set(cached_nuninst[arch])
  972. actual_nuninst = set(computed_nuninst[arch])
  973. false_negatives = actual_nuninst - expected_nuninst
  974. false_positives = expected_nuninst - actual_nuninst
  975. # Britney does not quite work correctly with
  976. # break/fucked arches, so ignore issues there for now.
  977. if (false_negatives or false_positives) and arch not in self.options.break_arches:
  978. only_on_break_archs = False
  979. if false_negatives:
  980. self.logger.error(" %s - unnoticed nuninst: %s", arch, str(false_negatives))
  981. if false_positives:
  982. self.logger.error(" %s - invalid nuninst: %s", arch, str(false_positives))
  983. self.logger.info(" %s - actual nuninst: %s", arch, str(actual_nuninst))
  984. self.logger.error("==================== NUNINST OUT OF SYNC =========================")
  985. if not only_on_break_archs:
  986. raise AssertionError("NUNINST OUT OF SYNC")
  987. else:
  988. self.logger.warning("Nuninst is out of sync on some break arches")
  989. self.logger.info("> All non-installability counters are ok")
  990. def upgrade_testing(self):
  991. """Upgrade testing using the packages from the source suites
  992. This method tries to upgrade testing using the packages from the
  993. source suites.
  994. Before running the do_all method, it tries the easy and force-hint
  995. commands.
  996. """
  997. output_logger = self.output_logger
  998. self.logger.info("Starting the upgrade test")
  999. output_logger.info("Generated on: %s", time.strftime("%Y.%m.%d %H:%M:%S %z", time.gmtime(time.time())))
  1000. output_logger.info("Arch order is: %s", ", ".join(self.options.architectures))
  1001. if not self.options.actions:
  1002. # process `easy' hints
  1003. for x in self.hints['easy']:
  1004. self.do_hint("easy", x.user, x.packages)
  1005. # process `force-hint' hints
  1006. for x in self.hints["force-hint"]:
  1007. self.do_hint("force-hint", x.user, x.packages)
  1008. # run the first round of the upgrade
  1009. # - do separate runs for break arches
  1010. allpackages = []
  1011. normpackages = self.upgrade_me[:]
  1012. archpackages = {}
  1013. for a in self.options.break_arches:
  1014. archpackages[a] = [p for p in normpackages if p.architecture == a]
  1015. normpackages = [p for p in normpackages if p not in archpackages[a]]
  1016. self.upgrade_me = normpackages
  1017. output_logger.info("info: main run")
  1018. self.do_all()
  1019. allpackages += self.upgrade_me
  1020. for a in self.options.break_arches:
  1021. backup = self.options.break_arches
  1022. self.options.break_arches = " ".join(x for x in self.options.break_arches if x != a)
  1023. self.upgrade_me = archpackages[a]
  1024. output_logger.info("info: broken arch run for %s", a)
  1025. self.do_all()
  1026. allpackages += self.upgrade_me
  1027. self.options.break_arches = backup
  1028. self.upgrade_me = allpackages
  1029. if self.options.actions:
  1030. self.printuninstchange()
  1031. return
  1032. # process `hint' hints
  1033. hintcnt = 0
  1034. for x in self.hints["hint"][:50]:
  1035. if hintcnt > 50:
  1036. output_logger.info("Skipping remaining hints...")
  1037. break
  1038. if self.do_hint("hint", x.user, x.packages):
  1039. hintcnt += 1
  1040. # run the auto hinter
  1041. self.run_auto_hinter()
  1042. if getattr(self.options, "remove_obsolete", "yes") == "yes":
  1043. # obsolete source packages
  1044. # a package is obsolete if none of the binary packages in testing
  1045. # are built by it
  1046. self.logger.info("> Removing obsolete source packages from the target suite")
  1047. # local copies for performance
  1048. target_suite = self.suite_info.target_suite
  1049. sources_t = target_suite.sources
  1050. binaries_t = target_suite.binaries
  1051. mi_factory = self._migration_item_factory
  1052. used = set(binaries_t[arch][binary].source
  1053. for arch in binaries_t
  1054. for binary in binaries_t[arch]
  1055. )
  1056. removals = [mi_factory.parse_item("-%s/%s" % (source, sources_t[source].version), auto_correct=False)
  1057. for source in sources_t if source not in used
  1058. ]
  1059. if removals:
  1060. output_logger.info("Removing obsolete source packages from the target suite (%d):", len(removals))
  1061. self.do_all(actions=removals)
  1062. # smooth updates
  1063. removals = old_libraries(self._migration_item_factory, self.suite_info, self.options.outofsync_arches)
  1064. if removals:
  1065. output_logger.info("Removing packages left in the target suite (e.g. smooth updates or cruft)")
  1066. log_and_format_old_libraries(self.output_logger, removals)
  1067. self.do_all(actions=removals)
  1068. removals = old_libraries(self._migration_item_factory, self.suite_info, self.options.outofsync_arches)
  1069. output_logger.info("List of old libraries in the target suite (%d):", len(removals))
  1070. log_and_format_old_libraries(self.output_logger, removals)
  1071. self.printuninstchange()
  1072. if self.options.check_consistency_level >= 1:
  1073. target_suite = self.suite_info.target_suite
  1074. self.assert_nuninst_is_correct()
  1075. target_suite.check_suite_source_pkg_consistency('end')
  1076. # output files
  1077. if not self.options.dry_run:
  1078. target_suite = self.suite_info.target_suite
  1079. # write HeidiResult
  1080. self.logger.info("Writing Heidi results to %s", self.options.heidi_output)
  1081. write_heidi(self.options.heidi_output,
  1082. target_suite,
  1083. outofsync_arches=self.options.outofsync_arches)
  1084. self.logger.info("Writing delta to %s", self.options.heidi_delta_output)
  1085. write_heidi_delta(self.options.heidi_delta_output,
  1086. self.all_selected)
  1087. self.logger.info("Test completed!")
  1088. def printuninstchange(self):
  1089. self.logger.info("Checking for newly uninstallable packages")
  1090. uninst = newly_uninst(self.nuninst_orig_save, self.nuninst_orig)
  1091. if uninst:
  1092. self.output_logger.warning("")
  1093. self.output_logger.warning("Newly uninstallable packages in the target suite:")
  1094. format_and_log_uninst(self.output_logger,
  1095. self.options.architectures,
  1096. uninst,
  1097. loglevel=logging.WARNING,
  1098. )
  1099. def hint_tester(self):
  1100. """Run a command line interface to test hints
  1101. This method provides a command line interface for the release team to
  1102. try hints and evaluate the results.
  1103. """
  1104. import readline
  1105. from britney2.completer import Completer
  1106. histfile = os.path.expanduser('~/.britney2_history')
  1107. if os.path.exists(histfile):
  1108. readline.read_history_file(histfile)
  1109. readline.parse_and_bind('tab: complete')
  1110. readline.set_completer(Completer(self).completer)
  1111. # Package names can contain "-" and we use "/" in our presentation of them as well,
  1112. # so ensure readline does not split on these characters.
  1113. readline.set_completer_delims(readline.get_completer_delims().replace('-', '').replace('/', ''))
  1114. known_hints = self._hint_parser.registered_hints
  1115. print("Britney hint tester")
  1116. print()
  1117. print("Besides inputting known britney hints, the follow commands are also available")
  1118. print(" * quit/exit - terminates the shell")
  1119. print(" * python-console - jump into an interactive python shell (with the current loaded dataset)")
  1120. print()
  1121. while True:
  1122. # read the command from the command line
  1123. try:
  1124. user_input = input('britney> ').split()
  1125. except EOFError:
  1126. print("")
  1127. break
  1128. except KeyboardInterrupt:
  1129. print("")
  1130. continue
  1131. # quit the hint tester
  1132. if user_input and user_input[0] in ('quit', 'exit'):
  1133. break
  1134. elif user_input and user_input[0] == 'python-console':
  1135. try:
  1136. import britney2.console
  1137. except ImportError as e:
  1138. print("Failed to import britney.console module: %s" % repr(e))
  1139. continue
  1140. britney2.console.run_python_console(self)
  1141. print("Returning to the britney hint-tester console")
  1142. # run a hint
  1143. elif user_input and user_input[0] in ('easy', 'hint', 'force-hint'):
  1144. mi_factory = self._migration_item_factory
  1145. try:
  1146. self.do_hint(user_input[0], 'hint-tester', mi_factory.parse_items(user_input[1:]))
  1147. self.printuninstchange()
  1148. except KeyboardInterrupt:
  1149. continue
  1150. elif user_input and user_input[0] in known_hints:
  1151. self._hint_parser.parse_hints('hint-tester', self.HINTS_ALL, '<stdin>', [' '.join(user_input)])
  1152. self.write_excuses()
  1153. try:
  1154. readline.write_history_file(histfile)
  1155. except IOError as e:
  1156. self.logger.warning("Could not write %s: %s", histfile, e)
  1157. def do_hint(self, hinttype, who, pkgvers):
  1158. """Process hints
  1159. This method process `easy`, `hint` and `force-hint` hints. If the
  1160. requested version is not in the relevant source suite, then the hint
  1161. is skipped.
  1162. """
  1163. output_logger = self.output_logger
  1164. self.logger.info("> Processing '%s' hint from %s", hinttype, who)
  1165. output_logger.info("Trying %s from %s: %s", hinttype, who,
  1166. " ".join("%s/%s" % (x.uvname, x.version) for x in pkgvers)
  1167. )
  1168. issues = []
  1169. # loop on the requested packages and versions
  1170. for idx in range(len(pkgvers)):
  1171. pkg = pkgvers[idx]
  1172. # skip removal requests
  1173. if pkg.is_removal:
  1174. continue
  1175. suite = pkg.suite
  1176. if pkg.package not in suite.sources:
  1177. issues.append("Source %s has no version in %s" % (pkg.package, suite.name))
  1178. elif apt_pkg.version_compare(suite.sources[pkg.package].version, pkg.version) != 0:
  1179. issues.append("Version mismatch, %s %s != %s" % (pkg.package, pkg.version,
  1180. suite.sources[pkg.package].version))
  1181. if issues:
  1182. output_logger.warning("%s: Not using hint", ", ".join(issues))
  1183. return False
  1184. self.do_all(hinttype, pkgvers)
  1185. return True
  1186. def get_auto_hinter_hints(self, upgrade_me):
  1187. """Auto-generate "easy" hints.
  1188. This method attempts to generate "easy" hints for sets of packages which
  1189. must migrate together. Beginning with a package which does not depend on
  1190. any other package (in terms of excuses), a list of dependencies and
  1191. reverse dependencies is recursively created.
  1192. Once all such lists have been generated, any which are subsets of other
  1193. lists are ignored in favour of the larger lists. The remaining lists are
  1194. then attempted in turn as "easy" hints.
  1195. We also try to auto hint circular dependencies analyzing the update
  1196. excuses relationships. If they build a circular dependency, which we already
  1197. know as not-working with the standard do_all algorithm, try to `easy` them.
  1198. """
  1199. self.logger.info("> Processing hints from the auto hinter")
  1200. sources_t = self.suite_info.target_suite.sources
  1201. excuses = self.excuses
  1202. # consider only excuses which are valid candidates and still relevant.
  1203. valid_excuses = frozenset(y.uvname for y in upgrade_me
  1204. if not y.is_cruft_removal and
  1205. (y not in sources_t or sources_t[y].version != excuses[y].ver[1]))
  1206. excuses_deps = {name: valid_excuses.intersection(excuse.get_deps())
  1207. for name, excuse in excuses.items() if name in valid_excuses}
  1208. excuses_rdeps = defaultdict(set)
  1209. for name, deps in excuses_deps.items():
  1210. for dep in deps:
  1211. excuses_rdeps[dep].add(name)
  1212. def find_related(e, hint, circular_first=False):
  1213. excuse = excuses[e]
  1214. if not circular_first:
  1215. hint[e] = excuse.ver[1]
  1216. if not excuse.get_deps():
  1217. return hint
  1218. for p in excuses_deps[e]:
  1219. if p in hint or p not in valid_excuses:
  1220. continue
  1221. if not find_related(p, hint):
  1222. return False
  1223. return hint
  1224. # loop on them
  1225. candidates = []
  1226. mincands = []
  1227. seen_hints = set()
  1228. for e in valid_excuses:
  1229. excuse = excuses[e]
  1230. if excuse.get_deps():
  1231. hint = find_related(e, {}, True)
  1232. if isinstance(hint, dict) and e in hint:
  1233. h = frozenset(hint.items())
  1234. if h not in seen_hints:
  1235. candidates.append(h)
  1236. seen_hints.add(h)
  1237. else:
  1238. items = [(e, excuse.ver[1])]
  1239. orig_size = 1
  1240. looped = False
  1241. seen_items = set()
  1242. seen_items.update(items)
  1243. for item, ver in items:
  1244. # excuses which depend on "item" or are depended on by it
  1245. new_items = {(x, excuses[x].ver[1]) for x in chain(excuses_deps[item], excuses_rdeps[item])}
  1246. new_items -= seen_items
  1247. items.extend(new_items)
  1248. seen_items.update(new_items)
  1249. if not looped and len(items) > 1:
  1250. orig_size = len(items)
  1251. h = frozenset(seen_items)
  1252. if h not in seen_hints:
  1253. mincands.append(h)
  1254. seen_hints.add(h)
  1255. looped = True
  1256. if len(items) != orig_size:
  1257. h = frozenset(seen_items)
  1258. if h != mincands[-1] and h not in seen_hints:
  1259. candidates.append(h)
  1260. seen_hints.add(h)
  1261. return [candidates, mincands]
  1262. def run_auto_hinter(self):
  1263. mi_factory = self._migration_item_factory
  1264. for l in self.get_auto_hinter_hints(self.upgrade_me):
  1265. for hint in l:
  1266. self.do_hint("easy", "autohinter", [mi_factory.parse_item("%s/%s" % (x[0], x[1]), auto_correct=False)
  1267. for x in sorted(hint)])
  1268. def nuninst_arch_report(self, nuninst, arch):
  1269. """Print a report of uninstallable packages for one architecture."""
  1270. all = defaultdict(set)
  1271. binaries_t = self.suite_info.target_suite.binaries
  1272. for p in nuninst[arch]:
  1273. pkg = binaries_t[arch][p]
  1274. all[(pkg.source, pkg.source_version)].add(p)
  1275. print('* %s' % arch)
  1276. for (src, ver), pkgs in sorted(all.items()):
  1277. print(' %s (%s): %s' % (src, ver, ' '.join(sorted(pkgs))))
  1278. print()
  1279. def main(self):
  1280. """Main method
  1281. This is the entry point for the class: it includes the list of calls
  1282. for the member methods which will produce the output files.
  1283. """
  1284. # if running in --print-uninst mode, quit
  1285. if self.options.print_uninst:
  1286. return
  1287. # if no actions are provided, build the excuses and sort them
  1288. elif not self.options.actions:
  1289. self.write_excuses()
  1290. # otherwise, use the actions provided by the command line
  1291. else:
  1292. self.upgrade_me = self.options.actions.split()
  1293. if self.options.compute_migrations or self.options.hint_tester:
  1294. if self.options.dry_run:
  1295. self.logger.info("Upgrade output not (also) written to a separate file"
  1296. " as this is a dry-run.")
  1297. elif hasattr(self.options, 'upgrade_output'):
  1298. upgrade_output = getattr(self.options, 'upgrade_output')
  1299. file_handler = logging.FileHandler(upgrade_output, mode='w', encoding='utf-8')
  1300. output_formatter = logging.Formatter('%(message)s')
  1301. file_handler.setFormatter(output_formatter)
  1302. self.output_logger.addHandler(file_handler)
  1303. self.logger.info("Logging upgrade output to %s", upgrade_output)
  1304. else:
  1305. self.logger.info("Upgrade output not (also) written to a separate file"
  1306. " as the UPGRADE_OUTPUT configuration is not provided.")
  1307. # run the hint tester
  1308. if self.options.hint_tester:
  1309. self.hint_tester()
  1310. # run the upgrade test
  1311. else:
  1312. self.upgrade_testing()
  1313. self.logger.info('> Stats from the installability tester')
  1314. for stat in self._inst_tester.stats.stats():
  1315. self.logger.info('> %s', stat)
  1316. else:
  1317. self.logger.info('Migration computation skipped as requested.')
  1318. if not self.options.dry_run:
  1319. self._policy_engine.save_state(self)
  1320. logging.shutdown()
  1321. if __name__ == '__main__':
  1322. Britney().main()