Devuan deployment of britney2
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

445 lines
16 KiB

  1. # This file is merged from Debian's tests and Ubuntu's autopktest implementation
  2. # For Ubuntu's part Canonical is the original copyright holder.
  3. #
  4. # (C) 2015 Canonical Ltd.
  5. #
  6. # This program is free software; you can redistribute it and/or modify
  7. # it under the terms of the GNU General Public License as published by
  8. # the Free Software Foundation; either version 2 of the License, or
  9. # (at your option) any later version.
  10. ## Debian's part
  11. from britney2 import BinaryPackageId
  12. from britney2.installability.builder import InstallabilityTesterBuilder
  13. TEST_HINTER = 'test-hinter'
  14. HINTS_ALL = ('ALL')
  15. DEFAULT_URGENCY = 'medium'
  16. ## autopkgtest part
  17. import os
  18. import shutil
  19. import subprocess
  20. import tempfile
  21. import unittest
  22. PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  23. architectures = ['amd64', 'arm64', 'armhf', 'i386', 'powerpc', 'ppc64el']
  24. ##
  25. def new_pkg_universe_builder():
  26. return UniverseBuilder()
  27. class MockObject(object):
  28. def __init__(self, **kwargs):
  29. for key, value in kwargs.items():
  30. setattr(self, key, value)
  31. class PkgUniversePackageBuilder(object):
  32. def __init__(self, uni_builder, pkg_id):
  33. self._uni_builder = uni_builder
  34. self._pkg_id = pkg_id
  35. self._dependencies = set()
  36. self._conflicts = set()
  37. self._in_testing = True
  38. self._is_essential = False
  39. def is_essential(self):
  40. self._is_essential = True
  41. return self
  42. def in_testing(self):
  43. self._in_testing = True
  44. return self
  45. def not_in_testing(self):
  46. self._in_testing = False
  47. return self
  48. def depends_on(self, pkg):
  49. return self.depends_on_any_of(pkg)
  50. def depends_on_any_of(self, *pkgs):
  51. self._dependencies.add(frozenset(self._uni_builder._fetch_pkg_id(x) for x in pkgs))
  52. return self
  53. def conflicts_with(self, *pkgs):
  54. self._conflicts.update(self._uni_builder._fetch_pkg_id(x) for x in pkgs)
  55. return self
  56. def new_package(self, *args, **kwargs):
  57. return self._uni_builder.new_package(*args, **kwargs)
  58. @property
  59. def pkg_id(self):
  60. return self._pkg_id
  61. def universe_builder(self):
  62. return self._uni_builder
  63. def build(self, *args, **kwargs):
  64. return self._uni_builder.build(*args, **kwargs)
  65. class UniverseBuilder(object):
  66. def __init__(self):
  67. self._cache = {}
  68. self._packages = {}
  69. self._default_version = '1.0-1'
  70. self._default_architecture = 'amd64'
  71. def _fetch_pkg_id(self, pkgish, version=None, architecture=None):
  72. if pkgish in self._cache:
  73. return self._cache[pkgish]
  74. if version is None:
  75. version = self._default_version
  76. if architecture is None:
  77. architecture = self._default_architecture
  78. if type(pkgish) == str:
  79. pkg_id = BinaryPackageId(pkgish, version, architecture)
  80. elif isinstance(pkgish, BinaryPackageId):
  81. pkg_id = pkgish
  82. elif type(pkgish) == tuple:
  83. if len(pkgish) == 2:
  84. pkg_id = BinaryPackageId(pkgish[0], pkgish[1], architecture)
  85. else:
  86. pkg_id = BinaryPackageId(*pkgish)
  87. elif isinstance(pkgish, PkgUniversePackageBuilder):
  88. pkg_id = pkgish._pkg_id
  89. else:
  90. raise ValueError("No clue on how to convert %s into a package id" % pkgish)
  91. self._cache[pkg_id] = pkg_id
  92. return pkg_id
  93. def new_package(self, raw_pkg_id_or_pkg, *, version=None, architecture=None):
  94. pkg_id = self._fetch_pkg_id(raw_pkg_id_or_pkg, version=version, architecture=architecture)
  95. pkg_builder = PkgUniversePackageBuilder(self, pkg_id)
  96. if pkg_id in self._packages:
  97. raise ValueError("Package %s already added previously" % pkg_id)
  98. self._packages[pkg_id] = pkg_builder
  99. return pkg_builder
  100. def build(self):
  101. builder = InstallabilityTesterBuilder()
  102. for pkg_id, pkg_builder in self._packages.items():
  103. builder.add_binary(pkg_id,
  104. essential=pkg_builder._is_essential,
  105. in_testing=pkg_builder._in_testing,
  106. )
  107. builder.set_relations(pkg_id, pkg_builder._dependencies, pkg_builder._conflicts)
  108. return builder.build()
  109. def pkg_id(self, pkgish):
  110. return self._fetch_pkg_id(pkgish)
  111. def update_package(self, pkgish):
  112. pkg_id = self._fetch_pkg_id(pkgish)
  113. if pkg_id not in self._packages:
  114. raise ValueError("Package %s has not been added yet" % pkg_id)
  115. return self._packages[pkg_id]
  116. # autopkgtest classes
  117. class TestData:
  118. def __init__(self):
  119. '''Construct local test package indexes.
  120. The archive is initially empty. You can create new packages with
  121. create_deb(). self.path contains the path of the archive, and
  122. self.apt_source provides an apt source "deb" line.
  123. It is kept in a temporary directory which gets removed when the Archive
  124. object gets deleted.
  125. '''
  126. self.path = tempfile.mkdtemp(prefix='testarchive.')
  127. self.apt_source = 'deb file://%s /' % self.path
  128. self.suite_testing = 'testing'
  129. self.suite_unstable = 'unstable'
  130. self.compute_migrations = ''
  131. self.dirs = {False: os.path.join(self.path, 'data', self.suite_testing),
  132. True: os.path.join(self.path, 'data', self.suite_unstable)}
  133. os.makedirs(self.dirs[False])
  134. os.mkdir(self.dirs[True])
  135. self.added_sources = {False: set(), True: set()}
  136. self.added_binaries = {False: set(), True: set()}
  137. # pre-create all files for all architectures
  138. for arch in architectures:
  139. for dir in self.dirs.values():
  140. with open(os.path.join(dir, 'Packages_' + arch), 'w'):
  141. pass
  142. for dir in self.dirs.values():
  143. for fname in ['Dates', 'Blocks', 'Urgency', 'BugsV']:
  144. with open(os.path.join(dir, fname), 'w'):
  145. pass
  146. os.mkdir(os.path.join(self.path, 'data', 'hints'))
  147. shutil.copytree(os.path.join(PROJECT_DIR, 'tests', 'policy-test-data', 'piuparts', 'basic'), os.path.join(self.dirs[False], 'state'))
  148. os.mkdir(os.path.join(self.path, 'output'))
  149. # create temporary home dir for proposed-migration autopktest status
  150. self.home = os.path.join(self.path, 'home')
  151. os.environ['HOME'] = self.home
  152. os.makedirs(os.path.join(self.home, 'proposed-migration',
  153. 'autopkgtest', 'work'))
  154. def __del__(self):
  155. shutil.rmtree(self.path)
  156. def add(self, name, unstable, fields={}, add_src=True, testsuite=None, srcfields=None):
  157. '''Add a binary package to the index file.
  158. You need to specify at least the package name and in which list to put
  159. it (unstable==True for unstable/proposed, or False for
  160. testing/release). fields specifies all additional entries, e. g.
  161. {'Depends': 'foo, bar', 'Conflicts: baz'}. There are defaults for most
  162. fields.
  163. Unless add_src is set to False, this will also automatically create a
  164. source record, based on fields['Source'] and name. In that case, the
  165. "Testsuite:" field is set to the testsuite argument.
  166. '''
  167. assert (name not in self.added_binaries[unstable])
  168. self.added_binaries[unstable].add(name)
  169. fields.setdefault('Architecture', 'any')
  170. fields.setdefault('Version', '1')
  171. fields.setdefault('Priority', 'optional')
  172. fields.setdefault('Section', 'devel')
  173. fields.setdefault('Description', 'test pkg')
  174. if fields['Architecture'] == 'any':
  175. fields_local_copy = fields.copy()
  176. for a in architectures:
  177. fields_local_copy['Architecture'] = a
  178. self._append(name, unstable, 'Packages_' + a, fields_local_copy)
  179. elif fields['Architecture'] == 'all':
  180. for a in architectures:
  181. self._append(name, unstable, 'Packages_' + a, fields)
  182. else:
  183. self._append(name, unstable, 'Packages_' + fields['Architecture'],
  184. fields)
  185. if add_src:
  186. src = fields.get('Source', name)
  187. if src not in self.added_sources[unstable]:
  188. if srcfields is None:
  189. srcfields = {}
  190. srcfields['Version'] = fields['Version']
  191. srcfields['Section'] = fields['Section']
  192. if testsuite:
  193. srcfields['Testsuite'] = testsuite
  194. self.add_src(src, unstable, srcfields)
  195. def add_src(self, name, unstable, fields={}):
  196. '''Add a source package to the index file.
  197. You need to specify at least the package name and in which list to put
  198. it (unstable==True for unstable/proposed, or False for
  199. testing/release). fields specifies all additional entries, which can be
  200. Version (default: 1), Section (default: devel), Testsuite (default:
  201. none), and Extra-Source-Only.
  202. '''
  203. assert (name not in self.added_sources[unstable])
  204. self.added_sources[unstable].add(name)
  205. fields.setdefault('Version', '1')
  206. fields.setdefault('Section', 'devel')
  207. self._append(name, unstable, 'Sources', fields)
  208. def _append(self, name, unstable, file_name, fields):
  209. with open(os.path.join(self.dirs[unstable], file_name), 'a') as f:
  210. f.write('''Package: %s
  211. Maintainer: Joe <joe@example.com>
  212. ''' % name)
  213. for k, v in fields.items():
  214. f.write('%s: %s\n' % (k, v))
  215. f.write('\n')
  216. def remove_all(self, unstable):
  217. '''Remove all added packages'''
  218. self.added_binaries[unstable] = set()
  219. self.added_sources[unstable] = set()
  220. for a in architectures:
  221. open(os.path.join(self.dirs[unstable], 'Packages_' + a), 'w').close()
  222. open(os.path.join(self.dirs[unstable], 'Sources'), 'w').close()
  223. def add_default_packages(self, libc6=True, green=True, lightgreen=True, darkgreen=True, blue=True, black=True, grey=True):
  224. '''To avoid duplication, add packages we need all the time'''
  225. # libc6 (always)
  226. self.add('libc6', False)
  227. if (libc6 is True):
  228. self.add('libc6', True)
  229. # src:green
  230. self.add('libgreen1', False, {'Source': 'green',
  231. 'Depends': 'libc6 (>= 0.9)'},
  232. testsuite='autopkgtest')
  233. if (green is True):
  234. self.add('libgreen1', True, {'Source': 'green',
  235. 'Depends': 'libc6 (>= 0.9)'},
  236. testsuite='autopkgtest')
  237. self.add('green', False, {'Depends': 'libc6 (>= 0.9), libgreen1',
  238. 'Conflicts': 'blue'},
  239. testsuite='autopkgtest')
  240. if (green is True):
  241. self.add('green', True, {'Depends': 'libc6 (>= 0.9), libgreen1',
  242. 'Conflicts': 'blue'},
  243. testsuite='autopkgtest')
  244. # lightgreen
  245. self.add('lightgreen', False, {'Depends': 'libgreen1'},
  246. testsuite='autopkgtest')
  247. if (lightgreen is True):
  248. self.add('lightgreen', True, {'Depends': 'libgreen1'},
  249. testsuite='autopkgtest')
  250. ## autodep8 or similar test
  251. # darkgreen
  252. self.add('darkgreen', False, {'Depends': 'libgreen1'},
  253. testsuite='autopkgtest-pkg-foo')
  254. if (darkgreen is True):
  255. self.add('darkgreen', True, {'Depends': 'libgreen1'},
  256. testsuite='autopkgtest-pkg-foo')
  257. # blue
  258. self.add('blue', False, {'Depends': 'libc6 (>= 0.9)',
  259. 'Conflicts': 'green'},
  260. testsuite='specialtest')
  261. if blue is True:
  262. self.add('blue', True, {'Depends': 'libc6 (>= 0.9)',
  263. 'Conflicts': 'green'},
  264. testsuite='specialtest')
  265. # black
  266. self.add('black', False, {},
  267. testsuite='autopkgtest')
  268. if black is True:
  269. self.add('black', True, {},
  270. testsuite='autopkgtest')
  271. # grey
  272. self.add('grey', False, {},
  273. testsuite='autopkgtest')
  274. if grey is True:
  275. self.add('grey', True, {},
  276. testsuite='autopkgtest')
  277. class TestBase(unittest.TestCase):
  278. def setUp(self):
  279. super(TestBase, self).setUp()
  280. self.maxDiff = None
  281. self.data = TestData()
  282. self.britney = os.path.join(PROJECT_DIR, 'britney.py')
  283. # create temporary config so that tests can hack it
  284. self.britney_conf = os.path.join(self.data.path, 'britney.conf')
  285. with open(self.britney_conf, 'w') as f:
  286. f.write('''
  287. TESTING = data/testing
  288. UNSTABLE = data/unstable
  289. NONINST_STATUS = data/testing/non-installable-status
  290. EXCUSES_OUTPUT = output/excuses.html
  291. EXCUSES_YAML_OUTPUT = output/excuses.yaml
  292. UPGRADE_OUTPUT = output/output.txt
  293. HEIDI_OUTPUT = output/HeidiResult
  294. STATIC_INPUT_DIR = data/testing/input
  295. STATE_DIR = data/testing/state
  296. ARCHITECTURES = amd64 arm64 armhf i386 powerpc ppc64el
  297. NOBREAKALL_ARCHES = amd64 arm64 armhf i386 powerpc ppc64el
  298. OUTOFSYNC_ARCHES =
  299. BREAK_ARCHES =
  300. NEW_ARCHES =
  301. MINDAYS_LOW = 0
  302. MINDAYS_MEDIUM = 0
  303. MINDAYS_HIGH = 0
  304. MINDAYS_CRITICAL = 0
  305. MINDAYS_EMERGENCY = 0
  306. DEFAULT_URGENCY = medium
  307. NO_PENALTIES = high critical emergency
  308. BOUNTY_MIN_AGE = 8
  309. HINTSDIR = data/hints
  310. HINTS_AUTOPKGTEST = ALL
  311. HINTS_FREEZE = block block-all block-udeb
  312. HINTS_FREEZE-EXCEPTION = unblock unblock-udeb
  313. HINTS_SATBRITNEY = easy
  314. HINTS_AUTO-REMOVALS = remove
  315. SMOOTH_UPDATES = badgers
  316. IGNORE_CRUFT = 0
  317. REMOVE_OBSOLETE = no
  318. ADT_ENABLE = yes
  319. ADT_ARCHES = amd64 i386
  320. ADT_AMQP = file://output/debci.input
  321. ADT_PPAS =
  322. ADT_SHARED_RESULTS_CACHE =
  323. ADT_SWIFT_URL = http://localhost:18085
  324. ADT_CI_URL = https://autopkgtest.ubuntu.com/
  325. ADT_HUGE = 20
  326. ADT_SUCCESS_BOUNTY =
  327. ADT_REGRESSION_PENALTY =
  328. ADT_BASELINE =
  329. ADT_RETRY_OLDER_THAN =
  330. ''')
  331. assert os.path.exists(self.britney)
  332. def tearDown(self):
  333. del self.data
  334. def run_britney(self, args=[]):
  335. '''Run britney.
  336. Assert that it succeeds and does not produce anything on stderr.
  337. Return (excuses.yaml, excuses.html, britney_out).
  338. '''
  339. britney = subprocess.Popen([self.britney, '-v', '-c', self.britney_conf,
  340. '%s' % self.data.compute_migrations],
  341. stdout=subprocess.PIPE,
  342. stderr=subprocess.PIPE,
  343. cwd=self.data.path,
  344. universal_newlines=True)
  345. (out, err) = britney.communicate()
  346. self.assertEqual(britney.returncode, 0, out + err)
  347. self.assertEqual(err, '')
  348. with open(os.path.join(self.data.path, 'output',
  349. 'excuses.yaml'), encoding='utf-8') as f:
  350. yaml = f.read()
  351. with open(os.path.join(self.data.path, 'output',
  352. 'excuses.html'), encoding='utf-8') as f:
  353. html = f.read()
  354. return (yaml, html, out)
  355. def create_hint(self, username, content):
  356. '''Create a hint file for the given username and content'''
  357. hints_path = os.path.join(
  358. self.data.path, 'data', 'hints', username)
  359. with open(hints_path, 'a') as fd:
  360. fd.write(content)
  361. fd.write('\n')