Browse Source

initial slave stuff

tags/sipwise-2.4
Michael Prokop 10 years ago
parent
commit
3596f3c3e7
2 changed files with 256 additions and 12 deletions
  1. +26
    -12
      scripts/build-and-provide-package
  2. +230
    -0
      scripts/include-in-repos

+ 26
- 12
scripts/build-and-provide-package View File

@@ -31,8 +31,8 @@ clean_workspace() {
echo "*** The following files have been noticed in the workspace [$(pwd)]: ***"
ls -la ./

echo "*** Cleaning workspace in $(pwd) to make sure we're building from scratch. ***"
rm -f ./* || true
# echo "*** Cleaning workspace in $(pwd) to make sure we're building from scratch. ***"
# rm -f ./* || true
}

# make sure we don't leave files for next run
@@ -62,6 +62,25 @@ identify_package_name() {
fi
}

set_base_path() {
# when BASE_PATH is set in the build step then don't assume a default,
# this is useful when building on slave nodes, being used like:
# export BASE_PATH="$WORKSPACE/${JOB_NAME%-binaries/*}-source/"

if [ -n "${BASE_PATH:-}" ] ; then
echo "*** Using provided ${BASE_PATH} as BASE_PATH ***"
else
echo "*** No BASE_PATH set. ***"
if [ -z "$distribution:-}" ]; then
BASE_PATH="${JENKINS_HOME}/userContent/${PACKAGE}-source/"
echo "*** No BASE_PATH set but \$distribution is set, using $BASE_PATH as BASE_PATH ***"
else
BASE_PATH="${JENKINS_HOME}/userContent/${PACKAGE}-source/distribution=${distribution}/"
echo "*** Neither BASE_PATH nor \$distribution set, using $BASE_PATH as BASE_PATH ***"
fi
fi

}

build_info() {
if [ -n "${REPOS:-}" ] ; then
@@ -103,13 +122,7 @@ identify_sourcefile() {
echo "*** Identifying newest package version ***"
newest_version="0"

if [ -z "$distribution:-}" ]; then
local bpath="${JENKINS_HOME}/userContent/${PACKAGE}-source/"
else
local bpath="${JENKINS_HOME}/userContent/${PACKAGE}-source/distribution=${distribution}/"
fi

for file in "${bpath}"/*.dsc; do
for file in "${BASE_PATH}/"*.dsc ; do
SOURCE_PACKAGE="$(awk '/^Source: / {print $2}' $file)"
p="$(basename $file .dsc)"
if [ "$p" = '*' ] ; then
@@ -124,8 +137,8 @@ identify_sourcefile() {
done

echo "*** Found package version $newest_version ***"
sourcefile="${bpath}/${SOURCE_PACKAGE}"_*"${newest_version}".dsc
#archtype="$(awk '/^Architecture/ {print $2}' ${JENKINS_HOME}/userContent/${PACKAGE}-source/${SOURCE_PACKAGE}_*${newest_version}.dsc)"
sourcefile="${BASE_PATH}/${SOURCE_PACKAGE}"_*"${newest_version}".dsc
fi

echo "*** Using $sourcefile (version: ${newest_version}) [sources: $sources]"
@@ -195,7 +208,7 @@ identify_build_type() {

local TMPDIR=$(mktemp -d)
cd "$TMPDIR"
for file in ${JENKINS_HOME}/userContent/${PACKAGE}-source/${SOURCE_PACKAGE}_*.tar.* ; do
for file in ${BASE_PATH}/${SOURCE_PACKAGE}_*.tar.* ; do
if tar atf "$file" 2>/dev/null | grep -q debian/control ; then
# might be source/debian/control - so let's identify the path to debian/control
local control_file=$(tar atf "$file" 2>/dev/null | grep 'debian/control$')
@@ -398,6 +411,7 @@ trap bailout 1 2 3 3 6 9 14 15
checks_and_defaults
clean_workspace
identify_package_name
set_base_path
build_info
identify_sourcefile
dist_and_arch_settings


+ 230
- 0
scripts/include-in-repos View File

@@ -0,0 +1,230 @@
#!/bin/sh

set -x
set -u

# make sure cowbuilder/pbuilder/... are available
PATH='/bin:/sbin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin'

checks_and_defaults() {
if [ -r /etc/jenkins/debian_glue ] ; then
. /etc/jenkins/debian_glue
fi

if [ -z "${JOB_NAME:-}" ] ; then
echo "No JOB_NAME defined, please run it in jenkins." >&2
exit 1
fi

if [ -z "${architecture:-}" ] ; then
echo "*** No architecture defined. Consider running it with matrix configuration. ***"
architecture="$(dpkg-architecture -qDEB_HOST_ARCH)"
echo "*** Falling back to default, using host architecture ${architecture}. ***"
fi

if [ -z "${REPOSITORY:-}" ] ; then
REPOSITORY='/srv/repository'
fi
}

# make sure we don't leave files for next run
bailout() {
[ -n "${1:-}" ] && EXIT="${1}" || EXIT=0
[ -n "${2:-}" ] && echo "$2" >&2

exit $EXIT
}

get_base_directory() {
# FIXME
if [ -d "$WORKSPACE"/repos-data ] ; then
cd "$WORKSPACE"/repos-data/architecture=*,label=*/
BASE_DIR=$(pwd)
fi

# if [ -n "${workspace_binaries:-}" ] ; then
# cd $workspace_binaries
# cd ..
# archive=$(find configurations -name lastStable -type l | grep axis-label | head -1)/archive
# if [ -n "${archive:-}" ] ; then
# cd "$archive"
# BASE_DIR=$(pwd)
# fi
# fi
}

identify_package_name() {
PACKAGE=${JOB_NAME%-binaries*}
BINARY_PACKAGE=${PACKAGE%-test*}
if [ -z "${PACKAGE:-}" ] ; then
bailout 1 "Error: could not identify Debian package name based on job name ${JOB_NAME:-}."
else
echo "*** Building binary package $BINARY_PACKAGE ***"
fi
}


build_info() {
if [ -n "${REPOS:-}" ] ; then
echo "*** Using supplied repository name $REPOS ***"
else
REPOS="${JOB_NAME%-binaries*}"
if [ -z "${distribution:-}" ]; then
echo "*** No repository supplied, using repository name $REPOS ***"
else
REPOS="${REPOS}-${distribution}"
echo "*** No repository supplied but distribution has been set, using repository name $REPOS ***"
fi
fi
}

identify_sourcefile() {
echo "*** Identifying package file ***"
sourcefile="${BASE_DIR}/"*.dsc
echo "*** Using $sourcefile ***"
}

remove_packages() {
echo "*** Removing previous versions from repository ***"
for p in $(dcmd "${BASE_DIR}/"*".changes") ; do
file="$(basename $p)"
binpackage="${file%%_*}"
binary_list="${binary_list:-} ${binpackage}"

# note: "removesrc" would remove foreign arch files (of different builds)
echo "*** Removing existing package ${binpackage} from repository ${REPOS} ***"
${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" --waitforlock 1000 remove "${REPOS}" "${binpackage}"
done

}

remove_missing_binary_packages() {
for p in $(${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" --waitforlock 1000 listmatched "${REPOS}" '*' | awk '{print $2}' | sort -u); do
echo "$binary_list" | grep -q "$p" || missing_packages="${missing_packages:-} $p"
done

if echo "${missing_packages:-}" | grep -q '[a-z0-9]' ; then
echo "*** Binary package(s) found, missing in build version: ${missing_packages:-} ***"
for p in $missing_packages ; do
echo "*** Removing $p from $REPOS to avoid out-of-date data ***"
${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" --waitforlock 1000 remove "${REPOS}" "${p}"
done
fi
}

reprepro_wrapper() {
if ! [ -d "$REPOSITORY" ] ; then
bailout 1 "Error: repository ${REPOSITORY} does not exist."
fi

remove_packages
remove_missing_binary_packages

archall=false
case $architecture in
all) archall=true
architecture='*' # support as file expansion in reprepro cmdline
;;
esac

echo "*** Including binary packages in repository $REPOS ***"
${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" --waitforlock 1000 --ignore=wrongdistribution \
include "${REPOS}" "${BASE_DIR}/"*".changes"
[ $? -eq 0 ] || bailout 1 "Error: Failed to include binary package in $REPOS repository."

# include the source package only in *one* architecture, being amd64
if [ "$architecture" = "amd64" ] || $archall ; then
echo "*** Including source package in repository $REPOS ***"

RC=0
if ! ${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" --waitforlock 1000 --ignore=wrongdistribution \
includedsc "${REPOS}" ${sourcefile} ; then
RC=1

# iff section/priority is empty then reprepro will complain
# about "No section and no priority for" and error out in the
# cmdline above, therefore we retry with -S and -P being set
${SUDO_CMD:-} reprepro -v -S unstable -P extra -b "${REPOSITORY}" --waitforlock 1000 \
--ignore=wrongdistribution \
includedsc "${REPOS}" ${sourcefile}
RC=$?
fi
[ $RC -eq 0 ] || bailout 1 "Error: Failed to include source package in $REPOS repository."
fi
}

trunk_release() {
# setting TRUNK_RELEASE=true enables release-trunk repository,
# to always get a copy of the package(s) to a central place
if [ -n "${TRUNK_RELEASE:-}" ] ; then
generate-reprepro-codename "$TRUNK_RELEASE"

${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" --waitforlock 1000 --ignore=wrongdistribution copymatched "$TRUNK_RELEASE" "$REPOS" '*'
[ $? -eq 0 ] || bailout 1 "Error: Failed to copy packages from ${REPOS} to ${TRUNK_RELEASE}."
fi
}

release_repos() {
echo "*** Environment variable 'release' is set, running through release steps. ***"

mkdir -p "${REPOSITORY}/incoming/${release}"
mkdir -p "${REPOSITORY}/conf"

if [ -n "${SUDO_CMD:-}" ] ; then
${SUDO_CMD:-} mkdir -p "${REPOSITORY}/incoming/${release}"
${SUDO_CMD:-} mkdir -p "${REPOSITORY}/conf"
${SUDO_CMD:-} chown -R "$(id -un)" "${REPOSITORY}/conf"
${SUDO_CMD:-} chown -R "$(id -un)" "${REPOSITORY}/incoming/${release}"
fi

cp "${BASE_DIR}/"* "${REPOSITORY}/incoming/${release}/"
[ $? -eq 0 ] || bailout 1 "Error: Failed to copy binary packages to release directory."

REPOSITORY=$REPOSITORY generate-reprepro-codename "${release}"

if ! grep -q "^Name: $release$" "${REPOSITORY}/conf/incoming" 2>/dev/null ; then
cat >> "${REPOSITORY}/conf/incoming" << EOF

Name: $release
IncomingDir: incoming/$release
TempDir: tmp
LogDir: log
MorgueDir: ${REPOSITORY}/morgue
Allow: unstable>$release
Cleanup: unused_files on_deny on_error

EOF
fi

${SUDO_CMD:-} reprepro -v -b "${REPOSITORY}" processincoming "${release}"
local RC=$?

echo "*** Gettind rid of *.tar.* and *.dsc files in incoming directory to avoid data leakage. ***"
rm -f ${REPOSITORY}/incoming/$release/*.tar.*
rm -f ${REPOSITORY}/incoming/$release/*.dsc

if [ $RC -ne 0 ] ; then
bailout 1 "Error: Failed to execute processincoming for release ${release}."
fi
}

deploy_to_releases() {
if [ -n "${release:-}" ] && [ "$release" != "none" ] && [ "$release" != "trunk" ] ; then
release_repos
else
reprepro_wrapper
trunk_release
fi
}

# main execution
trap bailout 1 2 3 3 6 9 14 15

checks_and_defaults
get_base_directory
identify_package_name
build_info
identify_sourcefile
deploy_to_releases

bailout 0

Loading…
Cancel
Save