diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5d3e7b5 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +PREFIX := /usr + +PROGRAMS := build-and-provide-package generate-git-snapshot generate-local-repository generate-reprepro-codename + +build: + @echo nothing to do + +install: $(scripts) + mkdir -p $(DESTDIR)/$(PREFIX)/bin/ + for prog in $(PROGRAMS); do \ + install -m 0755 $$prog $(DESTDIR)/$(PREFIX)/bin; \ + done + +.PHONY: install diff --git a/README.org b/README.org new file mode 100644 index 0000000..82260e2 --- /dev/null +++ b/README.org @@ -0,0 +1,114 @@ +* Background + +This Debian package provides scripts for building Debian packages +inside Jenkins (being a Continuous integration system). Currently +being a test driven and user needs driven prototype it's meant to make +Q/A builds of Debian packages inside Jenkins as manageable and +homogeneous as possible. + +* System requirements + ++ Jenkins ++ Cowbuilder ++ Reprepro + +* Setup instructions + +Jenkins jobs for a given Debian package /foobar/ needs to be named +/foobar-binaries/ and /foobar-source/. /foobar-source/ is the job name +where the source package is being built and /foobar-binaries/ is the +job name where the Debian binary packages are being built. + +If a second job for an already existing job for the Debian package +/foobar/ should be available then the name /foobar-test/ with the +according Jenkins jobs /foobar-test-source/ and /foobar-test-binaries/ +is supported. + +** Source job configuration (foobar-source) + +Choose "/This build is parameterized/" to be able to build specific +branches. + +Enable and configure "/Source Code Management/", currently only Git is +supported by the /generate-git-snapshot/ script. +(NOTE: Scripts for other version control systems are highly welcome!) +*Important*: set the "/Local subdirectory for repo (optional)/" option +under /Advanced/ settings to "/source/". + +Enable "/Trigger builds remotely/" and set an user-defined authentication +token. + +Use /@daily/ inside "/Poll SCM/". + +In the "/Build/" section add a build step "/Execute shell/" using: + +#+BEGIN_EXAMPLE +/usr/bin/generate-git-snapshot auto +#+END_EXAMPLE + +Under "/Post-build Actions/" select "/Archive the artifacts/" using: + +#+BEGIN_EXAMPLE +*.gz,*.bz2,*.xz,*.deb,*.dsc,*.changes +#+END_EXAMPLE + +Select the binaries job (/foobar-binaries/) under "/Build other +projects/" with the "/Trigger only if build succeeds/" option enabled. + +For "/Files to deploy/" enable "/Deploy artifacts from workspace to +remote directories/" using: + +#+BEGIN_EXAMPLE +*.gz,*.bz2,*.xz,*.deb,*.dsc,*.changes +#+END_EXAMPLE + +and for the "/Remote directory/" use: + +#+BEGIN_EXAMPLE +${JENKINS_HOME}/userContent/${JOB_NAME}/ +#+END_EXAMPLE + +Enable "/Record fingerprints of files to track usage/" and +"/Fingerprint all archived artifacts/". + +** Binaries job configuration (foobar-binaries) + +Add a "/Configuration Matrix/" with user-defined Axis titled +"/architecture/" and values "/amd64 i386/", specifying the +architectures that should be built. Choose "/Run each configuration +sequentially/". + +In the "/Build/" section add a build step "/Execute shell/" using: + +#+BEGIN_EXAMPLE +/usr/bin/build-and-provide-package +#+END_EXAMPLE + +Under "/Post-build Actions/" select "/Archive the artifacts/" using: + +#+BEGIN_EXAMPLE +*.gz,*.bz2,*.xz,*.deb,*.dsc,*.changes +#+END_EXAMPLE + +* Scripts + ++ *build-and-provide-package*: searches for newest package version in /${HUDSON_HOME}/userContent/${PACKAGE}-source// and uses the dsc file for building a binary package for the specific /$architecture/ of the matrix build using cowbuilder. The resulting binary package will be installed in reprepro to provide it for usage via APT. i386 builds are binary-only builds (limited to architecture dependent packages) and amd64 builds are binary-only builds (no source files are to be built). With the amd64 build the original dsc file is being installed in reprepro as source package. + ++ *generate-git-snapshot*: generates snapshot version in debian/changelog Debian package using git-dch. Use 'auto' as command line option to use git-dch's magic to build changelog, without the 'auto' option the version string will be build based on last tag/merge. + ++ *generate-local-repository*: scans the current working directory for Debian packages and generates Packages.gz, Contents and Release files. Meant for usage if reprepro would be overkill (e.g. no signed repository is needed). + ++ *generate-reprepro-codename*: takes a repository/codename as command line option and adds an according repository/codename definition to /srv/repository/conf/distributions (iff the codename is not present yet). As second command line option the GnuPG key ID for signing the repository can be specified (defaults to a static value otherwise). + +* Known TODOs + ++ Make configuration more flexible (key id, repository path, + cowbuilder base dirs,...). ++ Separate cowbuilder and reprepro steps in build-and-provide-package + (e.g. to use generate-local-repository instead of reprepro). ++ Make sure scaling with slave node works (including tagging of hosts, deploying files,...). ++ Provide Jenkins plugin to deploy and configure cowbuilder. ++ Provide Jenkins plugin to deploy and configure reprepro (including gpg key). + +* Author +Michael Prokop diff --git a/TODO b/TODO deleted file mode 100644 index 5b4236f..0000000 --- a/TODO +++ /dev/null @@ -1,3 +0,0 @@ -* streamline generate-snapshot -* support generating new reprepro repositories -* support deploying cowbuilder setups diff --git a/build-and-provide-package b/build-and-provide-package new file mode 100755 index 0000000..8652f24 --- /dev/null +++ b/build-and-provide-package @@ -0,0 +1,93 @@ +#!/bin/sh + +set -x +set -e + +if [ -z "${JOB_NAME}" ] ; then + echo "No JOB_NAME defined, please run it in jenkins." >&2 + exit 1 +fi + +if [ -z "${architecture}" ] ; then + echo "No architecture defined, please run it with matrix configuration using architecture." >&2 + exit 1 +fi + +PACKAGE=${JOB_NAME%-binaries*} +BINARY_PACKAGE=${PACKAGE%-test*} +if [ -z "$PACKAGE" ] ; then + echo "Error: could not identify Debian package name based on job name ${JOB_NAME}." >&2 + exit 1 +fi +echo "===== building binary package $BINARY_PACKAGE =====" + +if [ -n "$1" ] ; then + REPOS="$1" + echo "Using supplied repository name $REPOS" +else + REPOS="${JOB_NAME%-binaries*}" + echo "No repository supplied, using package name $REPOS" +fi + +echo "*** looking for newest package version ***" +newest_version="0" +for file in "${HUDSON_HOME}/userContent/${PACKAGE}-source/"*.dsc ; do + p="$(basename $file .dsc)" + if [ "$p" = '*' ] ; then + echo "No source package found (forgot to deploy via source job?)" >&2 + exit 1 + fi + cur_version="${p#*_}" + if dpkg --compare-versions "${cur_version}" gt "${newest_version}" ; then + newest_version="${cur_version}" + else + base_version="${cur_version}" + fi +done +echo "${newest_version}" +echo "*** found package version $newest_version ***" + +echo "*** cowbuilder build phase for arch $architecture ***" +case "$architecture" in + # -B -> binary-only build, limited to architecture dependent packages + i386) + linux32 sudo cowbuilder --buildresult "$WORKSPACE" \ + --build "${HUDSON_HOME}/userContent/${PACKAGE}-source/${BINARY_PACKAGE}"_*"${newest_version}".dsc \ + --basepath /var/cache/pbuilder/base32.cow --debbuildopts -B + ;; + # -b -> binary-only build, no source files are to be built and/or distributed + amd64) + sudo cowbuilder --buildresult "$WORKSPACE" \ + --build "${HUDSON_HOME}/userContent/${PACKAGE}-source/${BINARY_PACKAGE}"_*"${newest_version}".dsc \ + --basepath /var/cache/pbuilder/base.cow --debbuildopts -b + ;; + *) + echo "Unsupported architecture: $architecture" >&2 + exit 1 + ;; +esac + + +echo "*** removing previous versions from repository ***" +for p in $(dcmd "${WORKSPACE}/${BINARY_PACKAGE}"_*"${newest_version}_${architecture}.changes") ; do + file="$(basename $p)" + binpackage="${file%%_*}" + sudo reprepro -v -b /srv/repository --waitforlock 1000 --architecture \ + "$architecture" remove "${REPOS}" "${binpackage}" || true + sudo reprepro -v -b /srv/repository --waitforlock 1000 --architecture \ + all remove "${REPOS}" "${binpackage}" || true +done + + +echo "*** including binary packages in repository $REPOS ***" +sudo reprepro -v -b /srv/repository --waitforlock 1000 --architecture "$architecture" \ + include "${REPOS}" "${WORKSPACE}/${BINARY_PACKAGE}"_*"${newest_version}_${architecture}.changes" + + +# include the source package only in *one* architecture, being amd64 +echo "*** including source package in repository $REPOS ***" +if [ "$architecture" = "amd64" ] ; then + sudo reprepro -v -b /srv/repository --waitforlock 1000 includedsc \ + "${REPOS}" "${HUDSON_HOME}/userContent/${PACKAGE}-source/${BINARY_PACKAGE}"_*"${newest_version}".dsc +fi + diff --git a/debian/changelog b/debian/changelog new file mode 100644 index 0000000..969b674 --- /dev/null +++ b/debian/changelog @@ -0,0 +1,5 @@ +jenkins-debian-glue (0.0-1) UNRELEASED; urgency=low + + * Initial release. + + -- Michael Prokop Fri, 05 Aug 2011 18:39:59 +0200 diff --git a/debian/compat b/debian/compat new file mode 100644 index 0000000..45a4fb7 --- /dev/null +++ b/debian/compat @@ -0,0 +1 @@ +8 diff --git a/debian/control b/debian/control new file mode 100644 index 0000000..e50cb27 --- /dev/null +++ b/debian/control @@ -0,0 +1,19 @@ +Source: jenkins-debian-glue +Section: admin +Priority: extra +Maintainer: Michael Prokop +Build-Depends: debhelper (>= 8.0.0) +Standards-Version: 3.9.2 +Homepage: https://github.com/mika/jenkins-debian-glue +Vcs-Git: git://github.com/mika/jenkins-debian-glue.git +Vcs-Browser: https://github.com/mika/jenkins-debian-glue + +Package: jenkins-debian-glue +Architecture: all +Depends: ${shlibs:Depends}, ${misc:Depends}, cowbuilder, devscripts, dpkg-dev, sudo +Description: glue scripts for building Debian packages inside Jenkins + This package provides scripts which should make building Debian + package inside Jenkins (a Continuous Integration suite) easier. + . + It's meant to make Q/A builds of Debian packages inside Jenkins + as manageable and homogeneous as possible. diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..66868af --- /dev/null +++ b/debian/copyright @@ -0,0 +1,28 @@ +Format: http://dep.debian.net/deps/dep5 +Upstream-Name: Michael Prokop +Source: https://github.com/mika/jenkins-debian-glue + +Files: * +Copyright: 2011 Michael Prokop +License: GPL-2.0+ + +Files: debian/* +Copyright: 2011 Michael Prokop +License: GPL-2.0+ + +License: GPL-2.0+ + This package is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + . + This package is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + . + You should have received a copy of the GNU General Public License + along with this program. If not, see + . + On Debian systems, the complete text of the GNU General + Public License version 2 can be found in "/usr/share/common-licenses/GPL-2". diff --git a/debian/docs b/debian/docs new file mode 100644 index 0000000..f20153b --- /dev/null +++ b/debian/docs @@ -0,0 +1 @@ +README.org diff --git a/debian/rules b/debian/rules new file mode 100755 index 0000000..68449f2 --- /dev/null +++ b/debian/rules @@ -0,0 +1,13 @@ +#!/usr/bin/make -f +# -*- makefile -*- +# Sample debian/rules that uses debhelper. +# This file was originally written by Joey Hess and Craig Small. +# As a special exception, when this file is copied by dh-make into a +# dh-make output file, you may use that output file without restriction. +# This special exception was added by Craig Small in version 0.37 of dh-make. + +# Uncomment this to turn on verbose mode. +# export DH_VERBOSE=1 + +%: + dh $@ diff --git a/debian/source/format b/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/generate-git-snapshot b/generate-git-snapshot new file mode 100755 index 0000000..ce8ddb8 --- /dev/null +++ b/generate-git-snapshot @@ -0,0 +1,48 @@ +#!/bin/sh + +set -x +set -e + +[ -n "$DEBEMAIL" ] || DEBEMAIL="jenkins.grml.org Autobuilder " +export DEBEMAIL + +if [ ! -d source ] ; then + echo "Please run the script in the jenkins workspace." >&2 + exit 1 +fi + +if [ -z "${BUILD_NUMBER}" ] ; then + echo "No BUILD_NUMBER defined, please run it in jenkins." >&2 + exit 1 +fi + +echo "*** source package build phase ***" +rm -f ./* || true + +cd source + +if [ "$1" = "auto" ] ; then + git-dch -S --auto --multimaint-merge --snapshot-number=${BUILD_NUMBER} --ignore-branch +else + tag=$(git describe $(git rev-list --tags='[^ju]*' --max-count=1)) + last_merge=$(git describe $(git rev-list --all --merges --max-count=1)) + since=${tag} + + if [ -n "$last_merge" ] ; then + m_date=$(git log ${last_merge} --pretty="format:%at" -1) + t_date=$(git log ${tag} --pretty="format:%at" -1) + if [ ${m_date} -gt ${t_date} ] ; then + since=${last_merge} + fi + fi + + git-dch -S -s "${since}" --multimaint-merge --snapshot-number=${BUILD_NUMBER} --ignore-branch +fi + +debchange --release "" + +cd .. +dpkg-source -b source + +# needed for deploying artifacts +mkdir -p ${JENKINS_HOME}/userContent/${JOB_NAME}/ diff --git a/generate-repository b/generate-local-repository old mode 100644 new mode 100755 similarity index 93% rename from generate-repository rename to generate-local-repository index 52f3ee3..a7c610a --- a/generate-repository +++ b/generate-local-repository @@ -1,4 +1,8 @@ #!/bin/sh + +set -x +set -e + apt-ftparchive packages . | gzip > Packages.gz apt-ftparchive contents . > Contents cat <Release diff --git a/generate-reprepro-codename b/generate-reprepro-codename new file mode 100755 index 0000000..ef1625c --- /dev/null +++ b/generate-reprepro-codename @@ -0,0 +1,44 @@ +#!/bin/sh + +set -x +set -e + +if ! [ -r /srv/repository/conf/distributions ] ; then + echo "Error: could not read /srv/repository/conf/distributions." >&2 + exit 1 +fi + +if [ "$#" -lt 1 ] ; then + echo "Usage: $0 " >&2 + exit 1 +fi + +# repository/codename that should be added +REPOS="$1" + +# support setting key id +if [ -n "$2" ] ; then + KEY_ID="$2" +else # using a default (TODO - suppport configuration from outside) + KEY_ID="52D4A654" +fi + +if grep -q "^Codename: ${REPOS}$" /srv/repository/conf/distributions ; then + echo "Codename/repository $REPOS exists already, ignoring request to add again." + exit 0 +fi + +cat >> /srv/repository/conf/distributions << EOF + +Codename: ${REPOS} +AlsoAcceptFor: unstable +Architectures: amd64 i386 source +Components: main +DebIndices: Packages Release . .gz +DscIndices: Sources Release .gz +Tracking: minimal +SignWith: ${KEY_ID} + +EOF + +echo "Added $REPOS as new codename/repos to the reprepro configuration." diff --git a/generate-snapshot b/generate-snapshot deleted file mode 100644 index 83b975b..0000000 --- a/generate-snapshot +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -if [ ! -d source ] ; then - echo Please run the script in the workspace - exit 1 -fi -if [ -z "${BUILD_NUMBER}" ] ; then - echo No BUILD_NUMBER defined, please run it in jenkins - exit 1 -fi - -cd source - -tag=$(git describe $(git rev-list --tags='[^ju]*' --max-count=1)) -last_merge=$(git describe $(git rev-list --all --merges --max-count=1)) -since=${tag} -if [ -n "$last_merge" ] ; then - m_date=$(git log ${last_merge} --pretty="format:%at" -1) - t_date=$(git log ${tag} --pretty="format:%at" -1) - if [ ${m_date} -gt ${t_date} ] ; then - since=${last_merge} - fi - -fi - -export DEBEMAIL="jenkins.grml.org Autobuilder " -git-dch -S -s "${since}" --multimaint-merge --snapshot-number=${BUILD_NUMBER} --ignore-branch -debchange --release "" - -# vs. -# git-dch -S --auto --multimaint-merge --snapshot-number=${BUILD_NUMBER} --ignore-branch -# debchange --release ""