a tool to implement the autorelease cycle for devuan connecting gitlab repositories and issues with dak and jenkins
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

531 lines
18 KiB

  1. #!/usr/bin/env python3
  2. ########################################################################
  3. # This file is part of devuan-releasebot.
  4. #
  5. # Copyright: Franco (nextime) Lanza <nextime@nexlab.it> (c) 2015
  6. # Mark (LeePen) Hindley <mark@hindley.org.uk> (c) 2020
  7. #
  8. # devuan-releasebot is free software: you can redistribute it and/or modify
  9. # it under the terms of the GNU General Public License as published by
  10. # the Free Software Foundation, either version 3 of the License, or
  11. # (at your option) any later version.
  12. #
  13. # devuan-releasebot is distributed in the hope that it will be useful,
  14. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. # GNU General Public License for more details.
  17. #
  18. # You should have received a copy of the GNU General Public License
  19. # along with Foobar. If not, see <http://www.gnu.org/licenses/>.
  20. ##########################################################################
  21. import abc
  22. import configparser
  23. import functools
  24. import typing
  25. import attr
  26. # We use type annotations, and attrs implements the helper type keyword
  27. # for compatibility with python 3.5 in version 17.3.0.
  28. # We need to monkey patch things.
  29. if tuple(int(i) for i in attr.__version__.split('.')) < (17, 3, 0):
  30. if getattr(attr, 'original_ib', None) is None:
  31. attr.original_ib = attr.ib
  32. def our_attrib(type=None, *args, **kwargs):
  33. # factory is also new in 18.1.0
  34. factory = kwargs.pop('factory', None)
  35. if factory is not None:
  36. kwargs['default']=attr.Factory(factory)
  37. return attr.original_ib(*args, **kwargs)
  38. attr.ib = our_attrib
  39. def _list_helper(
  40. config, section: str, item: str, separator: str = ","
  41. ) -> typing.Iterable[str]:
  42. """
  43. Helper for comma-separated lists in .ini configurations
  44. """
  45. return [s.strip() for s in config.get(section, item).split(separator)]
  46. @attr.s
  47. class ReleaseBotCommand(object):
  48. """
  49. Data container for a ReleaseBot command.
  50. Which commands are supported is determined by the cmd_COMMAND methods in the
  51. ReleaseBot class.
  52. """
  53. name = attr.ib(type=str)
  54. package = attr.ib(type=str)
  55. labels = attr.ib(type=typing.Iterable[str])
  56. @attr.s
  57. class ReleaseBotRequester(object):
  58. """
  59. Data container for the requester and their permissions.
  60. is_admin provides access to building anything.
  61. permissions can be one of: 'member' or 'collaborator/PACKAGE'
  62. 'member' provides access to building anything but restricted suites and
  63. prefixes
  64. 'collaborator/PACKAGE' provides access to building PACKAGE
  65. This list of permissions is not guaranteed to be exhaustive, but should be
  66. enough to make decisions about the build.
  67. """
  68. name = attr.ib(type=str)
  69. is_admin = attr.ib(type=bool, default=False)
  70. permissions = attr.ib(factory=list, type=typing.Iterable[str])
  71. @attr.s
  72. class ReleaseBotTask(object):
  73. """
  74. Data container for all necessary data to carry out a ReleaseBot task.
  75. """
  76. id = attr.ib(type=str)
  77. repo_store = attr.ib(repr=False, type="GenericRepoStore")
  78. command = attr.ib(type=ReleaseBotCommand)
  79. requester = attr.ib(type=ReleaseBotRequester)
  80. def notify(self, message: str, resolve: bool = False) -> type(None):
  81. """
  82. Provide some progress notification regarding this task.
  83. Tasks will only be resolved if this is not a dryrun.
  84. """
  85. self.repo_store.notify_task(self, message, resolve)
  86. def resolve_notify(self, message: str) -> type(None):
  87. """
  88. Resolve a task with a notification.
  89. Tasks will only be resolved if this is not a dryrun.
  90. """
  91. self.repo_store.notify_task(self, message, resolve=True)
  92. class GenericRepoStore(abc.ABC):
  93. """
  94. Basic interface to abstract away the repository store (Gitea, GitLab, ...).
  95. If we ever need a different subsystem, this would hopefully stay the same.
  96. """
  97. def __init__(self, config, args):
  98. self.config = config
  99. self.args = args
  100. self._host = self.config.get("git", "host")
  101. self._token = self.config.get("git", "token")
  102. self.namespace = self.config.get("git", "namespace", fallback="devuan").lower()
  103. self.username = self.config.get(
  104. "git", "username", fallback="releasebot"
  105. ).lower()
  106. @abc.abstractmethod
  107. def get_pending_tasks(self) -> typing.Iterable[ReleaseBotTask]:
  108. """
  109. Get assigned issues/jobs/builds to the configured user in the namespace.
  110. Underlying implementations should provide all information needed to make
  111. a decision about whether or not and how such a request is to be processed.
  112. """
  113. pass
  114. def notify_task(self, task: ReleaseBotTask, message: str, resolve: bool):
  115. """
  116. Provide some progress notification regarding a task and optionally
  117. resolve it.
  118. Tasks will only be resolved if this is not a dryrun.
  119. """
  120. if self.args.debug:
  121. print("[DEBUG] ", message, task)
  122. if not self.args.dryrun:
  123. self.do_notify_task(task, message, resolve)
  124. @abc.abstractmethod
  125. def do_notify_task(self, task: ReleaseBotTask, message: str, resolve: bool):
  126. """
  127. Actually perform the notification. To be overridden in each implementation.
  128. This is never called if this is a dryrun.
  129. """
  130. pass
  131. class GiteaRepoStore(GenericRepoStore):
  132. """
  133. Gitea specific bits.
  134. """
  135. def __init__(self, config, args):
  136. super().__init__(config, args)
  137. # These could be overridden in the config, but we have sane defaults
  138. self.gitea_org = self.config.get("git", "gitea_org", fallback=self.namespace)
  139. self.gitea_team = self.config.get("git", "gitea_team", fallback="packages")
  140. import requests
  141. try:
  142. # python-requests 3
  143. self._http = requests.HTTPSession()
  144. except AttributeError:
  145. # python-requests 2
  146. self._http = requests.Session()
  147. # This takes care of authentication against gitea
  148. self._http.headers.update(
  149. {
  150. "Authorization": "token {}".format(self._token),
  151. "Accept": "application/json",
  152. "Content-Type": "application/json",
  153. }
  154. )
  155. def get_pending_tasks(self) -> typing.Iterable[ReleaseBotTask]:
  156. issues = self._get_build_issues()
  157. for issue in issues:
  158. # Basic data
  159. command = issue["title"].strip().lower()
  160. package = issue["repository"]["name"]
  161. labels = set([l["name"] for l in issue["labels"]])
  162. repo_owner = issue["repository"]["owner"].lower()
  163. issue_assignee = issue["assignee"]["username"].lower()
  164. task_id = "{}/{}/issues/{}".format(self.namespace, package, issue["number"])
  165. if self.args.debug:
  166. print("[DEBUG] Considering {}: {}.".format(task_id, command))
  167. if repo_owner != self.namespace or issue_assignee != self.username:
  168. # Silently skip if outside of the given namespace / user
  169. if self.args.debug:
  170. print(
  171. "[DEBUG] Skipping {}/{}/{}".format(
  172. repo_owner, package, issue["number"]
  173. )
  174. )
  175. continue
  176. # TODO: refuse reopened issues?
  177. # does gitea provide this information in the API?
  178. # Setup the ReleaseBotCommand
  179. releasebot_command = ReleaseBotCommand(
  180. name=command, package=package, labels=labels
  181. )
  182. # Setup the ReleaseBotRequester with their permissions
  183. requester = ReleaseBotRequester(
  184. name=issue["user"]["username"],
  185. is_admin=issue["user"].get("is_admin", False),
  186. )
  187. if self.is_packages_member(requester.name):
  188. requester.permissions.append("member")
  189. if self.is_package_collaborator(requester.name, package):
  190. requester.permissions.append("collaborator/{}".format(package))
  191. task = ReleaseBotTask(
  192. id=task_id,
  193. repo_store=self,
  194. command=releasebot_command,
  195. requester=requester,
  196. )
  197. yield task
  198. def do_notify_task(self, task: ReleaseBotTask, message: str, resolve: bool):
  199. url = "/repos/{}/comments".format(task.id)
  200. self._request("post", url, json=dict(body=message))
  201. if resolve:
  202. url = "/repos/{}".format(task.id)
  203. self._request("patch", url, json=dict(state="closed"))
  204. def _get_build_issues(self):
  205. """
  206. Overridable to facilitate testing.
  207. """
  208. return self._request(
  209. "get", "/repos/issues/search", json={"state": "open", "q": "build"}
  210. ).json()
  211. @property
  212. @functools.lru_cache()
  213. def packages_team_id(self) -> int:
  214. """
  215. We need this id internally for Gitea's API, it's cached per run.
  216. """
  217. teams = self._request(
  218. "get",
  219. "/orgs/{}/teams/search".format(self.gitea_org),
  220. params={"q": self.gitea_team, "include_desc": False},
  221. ).json()["data"]
  222. if len(teams) != 1:
  223. raise RuntimeError("Failed to identify packages team id")
  224. return teams[0]["id"]
  225. @functools.lru_cache()
  226. def is_packages_member(self, username: str) -> bool:
  227. """
  228. This is critical for permission management.
  229. It is cached for each user for this run for consistency.
  230. """
  231. response = self._request(
  232. "get",
  233. "/teams/{id}/members/{username}".format(
  234. id=self.packages_team_id, username=username
  235. ),
  236. )
  237. return response.ok
  238. @functools.lru_cache()
  239. def is_package_collaborator(self, username, package) -> bool:
  240. """
  241. This is critical for permission management.
  242. It is cached for each user/package combination for this run.
  243. """
  244. response = self._request(
  245. "get",
  246. "/repos/{owner}/{repo}/collaborators/{username}".format(
  247. owner=self.gitea_org, repo=package, username=username
  248. ),
  249. )
  250. return response.ok
  251. def _request(
  252. self, method: str, url: str, params: dict = dict(), json: dict = dict()
  253. ):
  254. response = self._http.request(
  255. method, "{}/api/v1{}".format(self._host, url), params=params, json=json
  256. )
  257. if response.status_code >= 400 and response.status_code != 404:
  258. raise RuntimeError(
  259. "Gitea API request {} failed with {}".format(url, response.status_code)
  260. )
  261. return response
  262. class GenericJobStore(abc.ABC):
  263. """
  264. Basic interface to abstract away the job store (Jenkins, buildbot, ...).
  265. If we ever need a different subsystem, this would hopefully stay the same.
  266. """
  267. def __init__(self, config, args):
  268. self.config = config
  269. self.args = args
  270. def queue_build(self, suite: str, package: str) -> str:
  271. """
  272. Queue a build for 'package' for 'suite' if not in dryrun mode.
  273. Return the URL for the build or, if in dryrun mode, an empty string.
  274. """
  275. url = ""
  276. if self.args.debug:
  277. print('[DEBUG] Queueing "{}" build for "{}"'.format(package, suite))
  278. if not self.args.dryrun:
  279. url = self.do_queue_build(suite, package)
  280. return url
  281. @abc.abstractmethod
  282. def do_queue_build(self, suite: str, package: str) -> str:
  283. """
  284. Actually queue the build. To be overridden in each implementation.
  285. This is never called if this is a dryrun.
  286. """
  287. pass
  288. class JenkinsJobStore(GenericJobStore):
  289. """
  290. Jenkins specific bits.
  291. """
  292. def __init__(self, config, args):
  293. super().__init__(config, args)
  294. self.jkjob_name = self.config.get("jenkins", "build_job")
  295. self._connect_jenkins()
  296. def _connect_jenkins(self):
  297. """
  298. Connect to jenkins and perform some sanity checks.
  299. This is a separate method to facilitate testing of the rest.
  300. """
  301. import jenkins
  302. self.jk = jenkins.Jenkins(
  303. self.config.get("jenkins", "host"),
  304. self.config.get("jenkins", "username"),
  305. self.config.get("jenkins", "password"),
  306. )
  307. # Check job (sanity)
  308. if not self.jk.job_exists(self.jkjob_name):
  309. raise RuntimeError("Build job not found on jenkins")
  310. if self.args.debug:
  311. print(
  312. "[DEBUG] Connected to Jenkins and checked build_job ", self.jkjob_name
  313. )
  314. def do_queue_build(self, suite: str, package: str) -> str:
  315. """
  316. Queue build if not in dryrun mode.
  317. Return the URL for the build or, if in dryrun mode, an empty string.
  318. """
  319. job_info = self.jk.get_job_info(self.jkjob_name)
  320. self.jk.build_job(self.jkjob_name, dict(codename=suite, srcpackage=package))
  321. return "{}{}".format(job_info["url"], job_info["nextBuildNumber"])
  322. class ReleaseBot(object):
  323. def __init__(
  324. self, config, args, job_store: GenericJobStore, repo_store: GenericRepoStore
  325. ):
  326. self.config = config
  327. self.args = args
  328. self.job_store = job_store
  329. self.repo_store = repo_store
  330. self.suites = _list_helper(config, "filters", "suites")
  331. def process(self):
  332. for task in self.repo_store.get_pending_tasks():
  333. yield self._process_task(task)
  334. def _process_task(self, task: ReleaseBotTask) -> ReleaseBotTask:
  335. # 0. Check command is valid
  336. cmd = getattr(self, "cmd_{}".format(task.command.name), None)
  337. if cmd is not None:
  338. # 1. check permissions
  339. if self.may_run_task(task):
  340. # 2. Process command
  341. if cmd(task):
  342. # 3. notify as necessary
  343. task.resolve_notify("Finished queueing")
  344. else:
  345. task.resolve_notify("You don't have permissions to run this task")
  346. else:
  347. task.resolve_notify(
  348. 'Command not supported: "{}". '
  349. "Available commands are: {}.".format(
  350. task.command.name, ", ".join(self.available_commands)
  351. )
  352. )
  353. return task
  354. @property
  355. @functools.lru_cache()
  356. def available_commands(self) -> typing.Iterable[str]:
  357. """
  358. Return an exhaustive list of all supported commands in this class.
  359. A command X is supported if a method is defined as follows:
  360. cmd_X(self, task: ReleaseBotTask):
  361. """
  362. return [x[4:] for x in dir(self) if x.startswith("cmd_")]
  363. @property
  364. @functools.lru_cache()
  365. def restricted_suites(self) -> typing.Iterable[str]:
  366. """
  367. Generate an exhaustive list of restricted suites.
  368. Use the cartesian product of restricted_suites and restricted_suffixes.
  369. """
  370. import itertools
  371. filters = _list_helper(self.config, "filters", "restricted_suites")
  372. suffixes = _list_helper(self.config, "filters", "restricted_suffixes")
  373. return ["-".join(i) for i in itertools.product(filters, suffixes)]
  374. def cmd_build(self, task: ReleaseBotTask):
  375. if not task.command.labels:
  376. task.resolve_notify("You must assign some labels")
  377. return False
  378. for suite in sorted(task.command.labels):
  379. if suite not in self.suites + self.restricted_suites:
  380. task.notify(
  381. 'Ignoring unrecognised tag or disallowed suite "{}"'.format(suite)
  382. )
  383. continue
  384. url = self.job_store.queue_build(suite, task.command.package)
  385. task.notify(
  386. 'Triggered "{}" build for "{}": {}'.format(
  387. task.command.package, suite, url
  388. )
  389. )
  390. return True
  391. def may_run_task(self, task: ReleaseBotTask) -> bool:
  392. """
  393. Decide whether or not a task may be run.
  394. This is returns false if any suite does not match the requester's permissions.
  395. See ReleaseBotRequester for the verbose description of the permissions.
  396. """
  397. # Admins may run tasks
  398. if task.requester.is_admin:
  399. return True
  400. # If there is at least one restricted suite, we refuse the whole task
  401. restricted_build = any(
  402. suite in self.restricted_suites for suite in task.command.labels
  403. )
  404. # Restricted tasks are only for admins
  405. if restricted_build:
  406. return False
  407. # Package members may run unrestricted tasks
  408. if "member" in task.requester.permissions:
  409. return True
  410. # Collaborators may run unrestricted tasks on this package
  411. if "collaborator/{}".format(task.command.package) in task.requester.permissions:
  412. return True
  413. # Default to deny
  414. return False
  415. def get_argparser():
  416. """
  417. Get ReleaseBot's argument parser.
  418. Any new parameters that are implemented should be added here.
  419. """
  420. import argparse
  421. parser = argparse.ArgumentParser()
  422. parser.add_argument("-c", "--config", type=argparse.FileType("r"), required=True)
  423. parser.add_argument("--debug", action="store_true")
  424. parser.add_argument("--dryrun", action="store_true")
  425. return parser
  426. def main(config, args):
  427. """
  428. Perform a ReleaseBot run
  429. """
  430. repo_store = GiteaRepoStore(config, args)
  431. job_store = JenkinsJobStore(config, args)
  432. releasebot = ReleaseBot(
  433. config=config, args=args, repo_store=repo_store, job_store=job_store
  434. )
  435. print(list(releasebot.process()))
  436. if __name__ == "__main__":
  437. args = get_argparser().parse_args()
  438. if args.config.name != "<stdin>" and not isinstance(args.config.name, int):
  439. import os
  440. # Ensure the config file with secrets is mode 0600 or 0400
  441. # This check is skipped if config is passed via stdin or file descriptor
  442. if os.stat(args.config.name).st_mode & 0o177 != 0:
  443. import sys
  444. sys.stderr.write(
  445. "Permissions are too broad ({})\n".format(args.config.name)
  446. )
  447. sys.exit(1)
  448. config = configparser.ConfigParser()
  449. config.read_file(args.config)
  450. main(config, args)