modified version of jenkins debian glue (https://github.com/mika/jenkins-debian-glue) for devuan
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

514 lines
18 KiB

  1. #!/bin/bash
  2. set -x
  3. set -u
  4. # make sure cowbuilder/pbuilder/... are available
  5. PATH='/bin:/sbin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin'
  6. echo "*** Starting $0 at $(date) ***"
  7. start_seconds=$(cut -d . -f 1 /proc/uptime)
  8. JENKINS_DEBIAN_GLUE_VERSION=$(dpkg --list jenkins-debian-glue 2>/dev/null | awk '/^ii/ {print $3}')
  9. if [ -n "${JENKINS_DEBIAN_GLUE_VERSION:-}" ] ; then
  10. echo "*** Running jenkins-debian-glue version $JENKINS_DEBIAN_GLUE_VERSION ***"
  11. fi
  12. checks_and_defaults() {
  13. if [ -r /etc/jenkins/debian_glue ] ; then
  14. . /etc/jenkins/debian_glue
  15. fi
  16. if [ -z "${JOB_NAME:-}" ] ; then
  17. echo "Error: No JOB_NAME defined, please run it in jenkins." >&2
  18. exit 1
  19. fi
  20. if [ -z "${architecture:-}" ] ; then
  21. echo "*** No architecture defined. Consider running it with matrix configuration. ***"
  22. architecture="$(dpkg-architecture -qDEB_HOST_ARCH)"
  23. echo "*** Falling back to default, using host architecture ${architecture}. ***"
  24. fi
  25. if [ -z "${REPOSITORY:-}" ] ; then
  26. REPOSITORY='/srv/repository'
  27. fi
  28. if [ -z "${PBUILDER_HOOKDIR:-}" ] ; then
  29. PBUILDER_HOOKDIR='/usr/share/jenkins-debian-glue/pbuilder-hookdir/'
  30. fi
  31. }
  32. clean_workspace() {
  33. echo "*** The following files have been noticed in the workspace [$(pwd)]: ***"
  34. ls -la ./
  35. # echo "*** Cleaning workspace in $(pwd) to make sure we're building from scratch. ***"
  36. # rm -f ./* || true
  37. }
  38. # make sure we don't leave files for next run
  39. bailout() {
  40. [ -n "${1:-}" ] && EXIT="${1}" || EXIT=0
  41. [ -n "${2:-}" ] && echo "$2" >&2
  42. echo "*** Getting rid of files in $WORKSPACE/binaries/ to avoid problems in next run. ***"
  43. rm -f "$WORKSPACE"/binaries/*
  44. ${SUDO_CMD:-} rm -rf /tmp/adt-$$
  45. [ -n "$start_seconds" ] && SECONDS="$[$(cut -d . -f 1 /proc/uptime)-$start_seconds]" || SECONDS="unknown"
  46. echo "*** Finished execution of $0 at $(date) [running ${SECONDS} seconds] ***"
  47. exit $EXIT
  48. }
  49. identify_package_name() {
  50. # make sure we get rid of 'repos' and 'binaries' from Jenkins job name
  51. PACKAGE=${JOB_NAME%-repos*}
  52. PACKAGE=${PACKAGE%-binaries*}
  53. if [ -n "${PACKAGE:-}" ] ; then
  54. echo "*** Identified Debian package name $PACKAGE ***"
  55. else
  56. bailout 1 "Error: could not identify Debian package name based on job name ${JOB_NAME:-}."
  57. fi
  58. }
  59. set_base_path() {
  60. # when BASE_PATH is set in the build step then don't default to $WORKSPACE
  61. if [ -n "${BASE_PATH:-}" ] ; then
  62. echo "*** Using provided ${BASE_PATH} as BASE_PATH ***"
  63. else
  64. BASE_PATH="${WORKSPACE}"
  65. echo "*** Using \$WORKSPACE [$BASE_PATH] as default for BASE_PATH ***"
  66. fi
  67. }
  68. build_info() {
  69. if [ -n "${REPOS:-}" ] ; then
  70. echo "*** Using supplied repository name $REPOS ***"
  71. else
  72. REPOS="${JOB_NAME%-binaries*}"
  73. REPOS="${REPOS%-repos*}"
  74. if [ -z "${distribution:-}" ]; then
  75. echo "*** No repository supplied, using repository name $REPOS ***"
  76. else
  77. REPOS="${REPOS}-${distribution}"
  78. echo "*** No repository supplied but distribution has been set, using repository name $REPOS ***"
  79. fi
  80. fi
  81. }
  82. identify_sourcefile() {
  83. if [ -n "${sources:-}" ] ; then
  84. echo "*** WARNING: sources variable [$sources] is set, please use BASE_PATH variable instead ***"
  85. echo "*** If \$sources is unrelated to build-and-provide-package you can ignore this warning ***"
  86. fi
  87. echo "*** Identifying newest package version ***"
  88. newest_version="0"
  89. for file in "${BASE_PATH}/"*.dsc ; do
  90. SOURCE_PACKAGE="$(awk '/^Source: / {print $2}' $file)"
  91. p="$(basename $file .dsc)"
  92. if [ "$p" = '*' ] ; then
  93. bailout 1 "Error: No source package found (forgot to configure source files deployment?)"
  94. fi
  95. cur_version="${p#*_}"
  96. if dpkg --compare-versions "${cur_version}" gt "${newest_version}" ; then
  97. newest_version="${cur_version}"
  98. else
  99. base_version="${cur_version}"
  100. fi
  101. done
  102. echo "*** Found package version $newest_version ***"
  103. sourcefile="${BASE_PATH}/${SOURCE_PACKAGE}"_*"${newest_version}".dsc
  104. echo "*** Using $sourcefile (version: ${newest_version})"
  105. }
  106. dist_and_arch_settings() {
  107. if [ -z "${architecture:-}" ] || [ "${architecture:-}" = "all" ] ; then
  108. arch="$(dpkg-architecture -qDEB_HOST_ARCH)"
  109. echo "*** No architecture set or architecture set to 'all', using system arch ${arch} ***"
  110. else
  111. arch="${architecture}"
  112. echo "*** architecture is set to ${architecture} ***"
  113. fi
  114. # only set $arch for other functions in this script if PROVIDE_ONLY is set
  115. if [ -n "${PROVIDE_ONLY:-}" ] ; then
  116. echo "*** Config variable 'PROVIDE_ONLY' is set, not setting COWBUILDER_BASE, COWBUILDER_DIST and DIST ***"
  117. return 0
  118. fi
  119. if [ -n "${distribution:-}" ] ; then
  120. local DIST="${distribution}"
  121. else
  122. # default to the currently running distribution to avoid hardcoding
  123. # a distribution which might not be supported by the running system
  124. local distribution=$(lsb_release --short --codename 2>/dev/null)
  125. [ -n "${distribution}" ] || distribution="sid" # fallback to "sid" iff lsb_release fails
  126. local DIST="$distribution"
  127. fi
  128. # if COWBUILDER_DIST is set it overrides distribution then
  129. if [ -n "${COWBUILDER_DIST:-}" ]; then
  130. echo "*** COWBUILDER_DIST is set to $COWBUILDER_DIST - using it for base.cow if it does not exist yet. ***"
  131. else
  132. echo "*** Using cowbuilder base for distribution ${DIST} ***"
  133. COWBUILDER_DIST="${DIST}"
  134. fi
  135. if [ -n "${COWBUILDER_BASE:-}" ] ; then
  136. echo "*** COWBUILDER_BASE is set to $COWBUILDER_BASE - using as cowbuilder base.cow ***"
  137. else
  138. COWBUILDER_BASE="/var/cache/pbuilder/base-${COWBUILDER_DIST}-${arch}.cow"
  139. echo "*** No COWBUILDER_BASE set, using $COWBUILDER_BASE as cowbuilder base.cow ***"
  140. fi
  141. }
  142. cowbuilder_init() {
  143. if [ -n "${COMPONENTS:-}" ] ; then
  144. echo "*** COMPONENTS is set [$COMPONENTS], using for pbuilder configuration ***"
  145. local pbuilderrc=$(mktemp)
  146. echo "COMPONENTS=\"${COMPONENTS}\"" > $pbuilderrc
  147. local pbuildercfg="--configfile=$pbuilderrc"
  148. fi
  149. # workaround for Ubuntu problem, as cowdancer is available only in universe :(
  150. # https://bugs.launchpad.net/ubuntu/+source/cowdancer/+bug/237591
  151. # https://bugs.launchpad.net/ubuntu/+source/cowdancer/+bug/747053
  152. if lsb_release --id 2>/dev/null | grep -q Ubuntu ; then
  153. if [ -z "${COMPONENTS:-}" ] ; then
  154. echo "*** Ubuntu detected, enabling universe repository component to work around cowdancer issue ***"
  155. local pbuilderrc=$(mktemp)
  156. echo 'COMPONENTS="main universe"' > $pbuilderrc
  157. local pbuildercfg="--configfile=$pbuilderrc"
  158. fi
  159. fi
  160. if [ ! -d "${COWBUILDER_BASE}" ]; then
  161. echo "*** Creating cowbuilder base $COWBUILDER_BASE for arch $arch and distribution $COWBUILDER_DIST ***"
  162. sudo cowbuilder --create --basepath "${COWBUILDER_BASE}" --distribution "${COWBUILDER_DIST}" \
  163. --debootstrapopts --arch --debootstrapopts "$arch" \
  164. --debootstrapopts --variant=buildd ${pbuildercfg:-} \
  165. --hookdir "${PBUILDER_HOOKDIR}"
  166. [ $? -eq 0 ] || bailout 1 "Error: Failed to create cowbuilder base ${COWBUILDER_BASE}."
  167. else
  168. echo "*** Updating cowbuilder cow base ***"
  169. sudo cowbuilder --update --basepath "${COWBUILDER_BASE}"
  170. [ $? -eq 0 ] || bailout 1 "Error: Failed to update cowbuilder base ${COWBUILDER_BASE}."
  171. fi
  172. [ -n "${pbuilderrc:-}" ] && rm -f "$pbuilderrc"
  173. }
  174. identify_build_type() {
  175. # defaults
  176. DEBBUILDOPTS="-sa"
  177. SKIP_ARCH_BUILD=false
  178. if [ "${architecture:-}" = "all" ] ; then
  179. echo "*** \$architecture is set to 'all', skipping further identify_build_type checks. ***"
  180. echo "*** Consider setting \$architecture to amd64, i386,... instead. ***"
  181. return 0
  182. fi
  183. if [ -z "${MAIN_ARCHITECTURE:-}" ] ; then
  184. if [ "$(dpkg-architecture -qDEB_HOST_ARCH)" = "${architecture:-}" ] ; then
  185. echo "*** MAIN_ARCHITECTURE is unset. ***"
  186. echo "*** Host architecture [$(dpkg-architecture -qDEB_HOST_ARCH)] matches \$architecture [${architecture:-}], using default ${DEBBUILDOPTS:-} buildoption ***"
  187. return 0
  188. else
  189. echo "*** MAIN_ARCHITECTURE is unset. ***"
  190. echo "*** Host architecture [$(dpkg-architecture -qDEB_HOST_ARCH)] does not match \$architecture [${architecture:-}] ... ***"
  191. echo "*** ... setting binary only build and continuing with identify_build_type ***"
  192. DEBBUILDOPTS="-B"
  193. fi
  194. else
  195. if [ "${MAIN_ARCHITECTURE:-}" = "${architecture:-}" ] ;then
  196. echo "*** MAIN_ARCHITECTURE is set [${MAIN_ARCHITECTURE:-}]. ***"
  197. echo "*** MAIN_ARCHITECTURE matches \$architecture [${architecture:-}], using default ${DEBBUILDOPTS:-} buildoption ***"
  198. return 0
  199. else
  200. echo "*** MAIN_ARCHITECTURE [${MAIN_ARCHITECTURE:-}] does not match \$architecture [${architecture:-}], setting binary only build and continuing with identify_build_type ***"
  201. DEBBUILDOPTS="-B"
  202. fi
  203. fi
  204. local TMPDIR=$(mktemp -d)
  205. local old_dir=$(pwd)
  206. cd "$TMPDIR"
  207. for file in ${BASE_PATH}/${SOURCE_PACKAGE}_*.tar.* ; do
  208. if tar atf "$file" 2>/dev/null | grep -q debian/control ; then
  209. # might be source/debian/control - so let's identify the path to debian/control
  210. local control_file=$(tar atf "$file" 2>/dev/null | grep 'debian/control$')
  211. tar axf "$file" "$control_file" || bailout 1 "Error while looking at debian/control in source archive."
  212. if grep -q '^Architecture: all' "$control_file" ; then
  213. if grep -q '^Architecture: any' "$control_file" ; then
  214. echo "*** Package provides arch 'all' + 'any', enabling -B buildoption for this architecture. ***"
  215. # -B -> binary-only build, limited to architecture dependent packages
  216. DEBBUILDOPTS="-B"
  217. break
  218. else
  219. # only "Architecture: all", so no arch specific packages since
  220. # we aren't building for $MAIN_ARCHITECTURE
  221. SKIP_ARCH_BUILD=true
  222. break
  223. fi
  224. fi
  225. fi
  226. done
  227. cd "$old_dir"
  228. rm -rf "${TMPDIR}"
  229. }
  230. autopkgtest_results() {
  231. if [ -n "${SKIP_AUTOPKGTEST_RESULTS:-}" ] ; then
  232. echo "** Skipping autopkgtest_results as requested via SKIP_AUTOPKGTEST_RESULTS ***"
  233. return 0
  234. fi
  235. # copy autopkgtest results from /tmp
  236. rm -rf adt
  237. ${SUDO_CMD:-} chmod -R go+rX /tmp/adt-$$
  238. cp -a /tmp/adt-$$ adt
  239. }
  240. cowbuilder_run() {
  241. echo "*** cowbuilder build phase for arch $architecture ***"
  242. mkdir -p "$WORKSPACE"/binaries/ /tmp/adt-$$
  243. # make sure we build arch specific packages only when necessary
  244. identify_build_type
  245. if $SKIP_ARCH_BUILD ; then
  246. autopkgtest_results
  247. bailout 0 "Nothing to do, architecture all binary packages only for non-primary architecture."
  248. fi
  249. case "$architecture" in
  250. i386)
  251. linux32 sudo cowbuilder --buildresult "$WORKSPACE"/binaries/ \
  252. --build $sourcefile \
  253. --basepath "${COWBUILDER_BASE}" --debbuildopts "${DEBBUILDOPTS:-}" \
  254. --hookdir "${PBUILDER_HOOKDIR}" --bindmounts /tmp/adt-$$
  255. [ $? -eq 0 ] || bailout 1 "Error: Failed to build with cowbuilder."
  256. ;;
  257. amd64|all|*)
  258. sudo cowbuilder --buildresult "$WORKSPACE"/binaries/ \
  259. --build $sourcefile \
  260. --basepath "${COWBUILDER_BASE}" --debbuildopts "${DEBBUILDOPTS:-}" \
  261. --hookdir "${PBUILDER_HOOKDIR}" --bindmounts /tmp/adt-$$
  262. [ $? -eq 0 ] || bailout 1 "Error: Failed to build with cowbuilder."
  263. ;;
  264. *)
  265. bailout 1 "Error: Unsupported architecture: $architecture"
  266. ;;
  267. esac
  268. }
  269. remove_packages() {
  270. if [ -n "${SKIP_REMOVAL:-}" ] ; then
  271. echo "*** Skipping removal of existing packages as requested via SKIP_REMOVAL ***"
  272. return 0
  273. fi
  274. echo "*** Removing source package version from repository ***"
  275. ${SUDO_CMD:-} reprepro -v -A source -b "${REPOSITORY}" --waitforlock 1000 remove "${REPOS}" "${SOURCE_PACKAGE}"
  276. echo "*** Removing previous binary package versions from repository ***"
  277. for p in $(dcmd "${WORKSPACE}/binaries/"*"${newest_version}_${arch}.changes" | grep '\.deb$') ; do
  278. file="$(basename $p)"
  279. binpackage="${file%%_*}"
  280. binary_list="${binary_list:-} ${binpackage}"
  281. # note: "removesrc" would remove foreign arch files (of different builds)
  282. if echo "$file" | egrep -q '_all.deb$'; then
  283. echo "*** Removing existing package ${binpackage} from repository ${REPOS} (arch all) ***"
  284. ${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" --waitforlock 1000 remove "${REPOS}" "${binpackage}"
  285. else
  286. echo "*** Removing existing package ${binpackage} from repository ${REPOS} for arch ${arch} ***"
  287. ${SUDO_CMD:-} reprepro -v -A "${arch}" -b "${REPOSITORY}" --waitforlock 1000 remove "${REPOS}" "${binpackage}"
  288. fi
  289. done
  290. }
  291. remove_missing_binary_packages() {
  292. if [ -n "${SKIP_REMOVAL:-}" ] ; then
  293. echo "*** Skipping removal of existing packages as requested via SKIP_REMOVAL ***"
  294. return 0
  295. fi
  296. echo "*** Checking for missing binary packages to be considered for removal ***"
  297. # In a binary-only build we don't get any arch-all (*_all.deb) packages and
  298. # therefore they won't be listed in the changes file. As a result they would
  299. # be reported as missing from the build and to be considered for removal.
  300. # As we don't want to remove the arch-all package e.g. from the amd64 repos
  301. # in the i386 run we've to skip the removal procedure then.
  302. case "${DEBBUILDOPTS:-}" in
  303. *-B*)
  304. echo "*** Skipping removal of missing binaries as being a binary-only build ***"
  305. return 0
  306. ;;
  307. esac
  308. for p in $(${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" --waitforlock 1000 --list-format '${package}\n' listmatched "${REPOS}" '*' | sort -u); do
  309. echo " $binary_list " | grep -q " $p " || missing_packages="${missing_packages:-} $p"
  310. done
  311. if echo "${missing_packages:-}" | grep -q '.' ; then
  312. echo "*** Binary package(s) found, missing in build version: ${missing_packages:-} ***"
  313. for p in $missing_packages ; do
  314. echo "*** Removing $p from $REPOS to avoid out-of-date data ***"
  315. ${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" --waitforlock 1000 remove "${REPOS}" "${p}"
  316. done
  317. fi
  318. }
  319. reprepro_wrapper() {
  320. if ! [ -d "$REPOSITORY" ] ; then
  321. bailout 1 "Error: repository ${REPOSITORY} does not exist."
  322. fi
  323. ${SUDO_CMD:-} generate-reprepro-codename "${REPOS}"
  324. remove_packages
  325. remove_missing_binary_packages
  326. archall=false
  327. case $architecture in
  328. all) archall=true
  329. architecture='*' # support as file expansion in reprepro cmdline
  330. ;;
  331. esac
  332. echo "*** Including packages in repository $REPOS ***"
  333. ${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" --waitforlock 1000 --ignore=wrongdistribution \
  334. include "${REPOS}" "${WORKSPACE}/binaries/"*"${newest_version}"_${architecture}.changes
  335. [ $? -eq 0 ] || bailout 1 "Error: Failed to include binary package in $REPOS repository."
  336. }
  337. trunk_release() {
  338. # setting TRUNK_RELEASE=true enables release-trunk repository,
  339. # to always get a copy of the package(s) to a central place
  340. if [ -z "${TRUNK_RELEASE:-}" ] ; then
  341. echo "*** TRUNK_RELEASE is not enabled ***"
  342. elif [ "${IGNORE_RELEASE_TRUNK:-}" = "true" ] ; then
  343. echo "*** IGNORE_RELEASE_TRUNK is set, ignoring request to add package(s) to $TRUNK_RELEASE repos ***"
  344. else
  345. echo "*** TRUNK_RELEASE is enabled ($TRUNK_RELEASE) ***"
  346. ${SUDO_CMD:-} generate-reprepro-codename "$TRUNK_RELEASE"
  347. ${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" --waitforlock 1000 --ignore=wrongdistribution copymatched "$TRUNK_RELEASE" "$REPOS" '*'
  348. [ $? -eq 0 ] || bailout 1 "Error: Failed to copy packages from ${REPOS} to ${TRUNK_RELEASE}."
  349. fi
  350. }
  351. release_repos() {
  352. echo "*** Environment variable 'release' is set, running through release steps. ***"
  353. local REPOSITORY="${REPOSITORY}/release/${release}"
  354. mkdir -p "${REPOSITORY}/incoming/${release}"
  355. mkdir -p "${REPOSITORY}/conf"
  356. cp "${WORKSPACE}/binaries/"* "${REPOSITORY}/incoming/${release}/"
  357. [ $? -eq 0 ] || bailout 1 "Error: Failed to copy binary packages to release directory."
  358. REPOSITORY=$REPOSITORY generate-reprepro-codename "${release}"
  359. if ! grep -q "^Name: $release$" "${REPOSITORY}/conf/incoming" 2>/dev/null ; then
  360. cat >> "${REPOSITORY}/conf/incoming" << EOF
  361. Name: $release
  362. IncomingDir: incoming/$release
  363. TempDir: tmp
  364. LogDir: log
  365. MorgueDir: ${REPOSITORY}/morgue
  366. Default: $release
  367. Cleanup: unused_files on_deny on_error
  368. EOF
  369. fi
  370. local old_dir=$(pwd)
  371. cd "${REPOSITORY}/incoming/${release}"
  372. ${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" --waitforlock 1000 --ignore=wrongdistribution \
  373. processincoming "${release}" "$(basename ${WORKSPACE}/binaries/*.changes)"
  374. local RC=$?
  375. cd "$old_dir"
  376. if [ $RC -ne 0 ] ; then
  377. bailout 1 "Error: Failed to execute processincoming for release ${release}."
  378. fi
  379. }
  380. deploy_to_releases() {
  381. if [ -n "${release:-}" ] && [ "$release" != "none" ] && [ "$release" != "trunk" ] && \
  382. # '${release}' is a hidden bomb: when provided through predefined parameters
  383. # from an upstream jenkins job (like foo-binaries receiving the parameters
  384. # from foo-source) but the job (foo-binaries) gets triggered manually (without
  385. # setting the predefined parameters therefore) then ${release} is set to
  386. # '${release}' instead of being empty
  387. [ "${release}" != '${release}' ] ; then
  388. release_repos
  389. else
  390. reprepro_wrapper
  391. trunk_release
  392. fi
  393. }
  394. # make them available for the Jenkin's 'Archiving artifacts'
  395. binaries_to_workspace() {
  396. echo "*** Moving binaries files to workspace. ***"
  397. mv "${WORKSPACE}/binaries/"* "${WORKSPACE}/"
  398. rmdir "${WORKSPACE}/binaries/"
  399. }
  400. # main execution
  401. trap bailout SIGHUP SIGINT SIGQUIT SIGABRT SIGKILL SIGALRM SIGTERM
  402. checks_and_defaults
  403. clean_workspace
  404. identify_package_name
  405. set_base_path
  406. build_info
  407. identify_sourcefile
  408. dist_and_arch_settings
  409. # do not run in repos job?
  410. if [ -n "${PROVIDE_ONLY:-}" ] ; then
  411. echo "*** Config variable 'PROVIDE_ONLY' is set, ignoring request to run cowbuilder. ***"
  412. else
  413. cowbuilder_init
  414. cowbuilder_run
  415. fi
  416. # do not run in binaries job?
  417. if [ -n "${BUILD_ONLY:-}" ] ; then
  418. echo "*** Config variable 'BUILD_ONLY' is set, ignoring request to use local repository. ***"
  419. else
  420. deploy_to_releases
  421. fi
  422. binaries_to_workspace
  423. autopkgtest_results
  424. bailout 0
  425. # vim:foldmethod=marker ts=2 ft=sh ai expandtab sw=2