diff options
| -rwxr-xr-x | bin/reproducible_build.sh | 19 | ||||
| -rwxr-xr-x | bin/reproducible_common.sh | 4 | ||||
| -rwxr-xr-x | bin/reproducible_setup_pbuilder.sh (renamed from bin/reproducible_setup.sh) | 0 | ||||
| -rwxr-xr-x | bin/reproducible_setup_schroot.sh | 118 | ||||
| -rw-r--r-- | job-cfg/reproducible.yaml | 4 | ||||
| -rw-r--r-- | logparse/reproducible.rules | 1 |
6 files changed, 144 insertions, 2 deletions
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh index d9b558c3..4a7fc84b 100755 --- a/bin/reproducible_build.sh +++ b/bin/reproducible_build.sh @@ -134,10 +134,29 @@ else LOGFILE=$(ls ${SRCPACKAGE}_${EVERSION}.dsc) LOGFILE=$(echo ${LOGFILE%.dsc}.debbindiff.html) BUILDINFO=${SRCPACKAGE}_${EVERSION}_amd64.buildinfo + # the schroot for debbindiff gets updated once a day. wait patiently if that's the case + if [ -f $DBDCHROOT_WRITELOCK ] || [ -f $DBDCHROOT_READLOCK ] ; then + for i in $(seq 0 100) ; do + sleep 15 + echo "sleeping 15s, debbindiff schroot is locked." + if [ ! -f $DBDCHROOT_WRITELOCK ] && [ ! -f $DBDCHROOT_READLOCK ] ; then + break + fi + done + if [ -f $DBDCHROOT_WRITELOCK ] || [ -f $DBDCHROOT_READLOCK ] ; then + echo "Warning: lock $DBDCHROOT_WRITELOCK or [ -f $DBDCHROOT_READLOCK ] still exists, exiting." + exit 1 + fi + else + # we create (more) read-lock(s) but stop on write locks... + # write locks are only done by the schroot setup job + touch $DBDCHROOT_READLOCK + fi ( timeout 15m schroot --directory /tmp -c source:jenkins-reproducible-sid debbindiff -- --html $TMPDIR/${LOGFILE} $TMPDIR/b1/${SRCPACKAGE}_${EVERSION}_amd64.changes $TMPDIR/b2/${SRCPACKAGE}_${EVERSION}_amd64.changes ) 2>&1 >> ${RBUILDLOG} RESULT=$? set +x set -e + rm -f $DBDCHROOT_READLOCK echo | tee -a ${RBUILDLOG} if [ $RESULT -eq 124 ] ; then echo "$(date) - debbindiff was killed after running into timeouot... maybe there is still $JENKINS_URL/userContent/dbd/${LOGFILE}" | tee -a ${RBUILDLOG} diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 775033c7..f00a4797 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -100,6 +100,10 @@ elif [ ! -f ${PACKAGES_DB} ] ; then EOF fi +# common variables +DBDCHROOT_READLOCK=/var/lib/jenkins/reproducible-dbdchroot.readlock +DBDCHROOT_WRITELOCK=/var/lib/jenkins/reproducible-dbdchroot.writelock + # shop trailing slash JENKINS_URL=${JENKINS_URL:0:-1} diff --git a/bin/reproducible_setup.sh b/bin/reproducible_setup_pbuilder.sh index 6075789f..6075789f 100755 --- a/bin/reproducible_setup.sh +++ b/bin/reproducible_setup_pbuilder.sh diff --git a/bin/reproducible_setup_schroot.sh b/bin/reproducible_setup_schroot.sh new file mode 100755 index 00000000..70f40f5a --- /dev/null +++ b/bin/reproducible_setup_schroot.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# Copyright 2012-2014 Holger Levsen <holger@layer-acht.org> +# Copyright 2013 Antonio Terceiro <terceiro@debian.org> +# Copyright 2014 Joachim Breitner <nomeata@debian.org> +# released under the GPLv=2 + +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# bootstraps a new chroot for schroot, and then moves it into the right location + +# $1 = schroot name +# $2 = base distro +# $3 $4 ... = extra packages to install + +if [ $# -lt 2 ]; then + echo "usage: $0 TARGET DISTRO [backports] CMD [ARG1 ARG2 ...]" + exit 1 +fi +TARGET="$1" +shift +DISTRO="$1" +shift + +if [ "$1" == "backports" ] ; then + BACKPORTS="deb $MIRROR ${DISTRO}-backports main" + BACKPORTSSRC="deb-src $MIRROR ${DISTRO}-backports main" + shift +fi + +if [ ! -d "$CHROOT_BASE" ]; then + echo "Directory $CHROOT_BASE does not exist, aborting." + exit 1 +fi + +export CHROOT_TARGET=$(mktemp -d -p $CHROOT_BASE/ schroot-install-$TARGET-XXXX) +if [ -z "$CHROOT_TARGET" ]; then + echo "Could not create a directory to create the chroot in, aborting." + exit 1 +fi + +bootstrap() { + mkdir -p "$CHROOT_TARGET/etc/dpkg/dpkg.cfg.d" + echo force-unsafe-io > "$CHROOT_TARGET/etc/dpkg/dpkg.cfg.d/02dpkg-unsafe-io" + + echo "Bootstraping $DISTRO into $CHROOT_TARGET now." + sudo debootstrap $DISTRO $CHROOT_TARGET $MIRROR + + echo -e '#!/bin/sh\nexit 101' | sudo tee $CHROOT_TARGET/usr/sbin/policy-rc.d >/dev/null + sudo chmod +x $CHROOT_TARGET/usr/sbin/policy-rc.d + echo 'Acquire::http::Proxy "$http_proxy";' | sudo tee $CHROOT_TARGET/etc/apt/apt.conf.d/80proxy >/dev/null + echo "deb-src $MIRROR $DISTRO main" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list > /dev/null + echo "${BACKPORTS}" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list >/dev/null + echo "${BACKPORTSSRC}" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list >/dev/null + + sudo chroot $CHROOT_TARGET apt-get update + if [ -n "$1" ] ; then + sudo chroot $CHROOT_TARGET apt-get install -y --no-install-recommends "$@" + fi +} + +cleanup() { + if [ -d $CHROOT_TARGET ]; then + sudo rm -rf --one-file-system $CHROOT_TARGET || fuser -mv $CHROOT_TARGET + fi +} +trap cleanup INT TERM EXIT +bootstrap $@ + +trap - INT TERM EXIT + +# aquire a write lock in any case +touch $DBDCHROOT_WRITELOCK +if [ -f $DBDCHROOT_READLOCK ] ; then + # patiently wait for our users to using the schroot + for i in $(seq 0 100) ; do + sleep 15 + echo "sleeping 15s, debbindiff schroot is locked and used." + if [ ! -f $DBDCHROOT_READLOCK ] ; then + break + fi + done + if [ -f $DBDCHROOT_READLOCK ] ; then + echo "Warning: lock $DBDCHROOT_READLOCK still exists, exiting." + exit 1 + fi +fi + +# pivot the new schroot in place +rand=$RANDOM +if [ -d $SCHROOT_BASE/"$TARGET" ] +then + sudo mv $SCHROOT_BASE/"$TARGET" $SCHROOT_BASE/"$TARGET"-"$rand" +fi + +sudo mv $CHROOT_TARGET $SCHROOT_BASE/"$TARGET" + +if [ -d $SCHROOT_BASE/"$TARGET"-"$rand" ] +then + sudo rm -rf --one-file-system $SCHROOT_BASE/"$TARGET"-"$rand" +fi + +# write the schroot config +echo "Writing configuration" +sudo tee /etc/schroot/chroot.d/jenkins-"$TARGET" <<-__END__ + [jenkins-$TARGET] + description=Jenkins schroot $TARGET + directory=$SCHROOT_BASE/$TARGET + type=directory + root-users=jenkins + source-root-users=jenkins + union-type=aufs + __END__ + +# remove the lock +rm $DBDCHROOT_WRITELOCK + diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index 12f51ac4..b43a918f 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -183,12 +183,12 @@ - '{name}_setup_pbuilder': my_description: 'Setup pbuilder for reproducible builds as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' my_timed: '23 0 * * *' - my_shell: '/srv/jenkins/bin/reproducible_setup.sh' + my_shell: '/srv/jenkins/bin/reproducible_setup_pbuilder.sh' my_recipients: 'jenkins+debian-reproducible holger@layer-acht.org' - '{name}_setup_schroot': my_description: 'Setup schroot for running debbindiff in a sid environment, this is needed eg to be able to correctly investigate haskell binaries....' my_timed: '23 0 * * *' - my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible-sid sid debbindiff' + my_shell: '/srv/jenkins/bin/reproducible_setup_schroot.sh reproducible-sid sid debbindiff' my_recipients: 'holger@layer-acht.org' - '{name}_scheduler': my_description: 'Schedule packages to be tested on reproducible building.' diff --git a/logparse/reproducible.rules b/logparse/reproducible.rules index 08a0a289..03628085 100644 --- a/logparse/reproducible.rules +++ b/logparse/reproducible.rules @@ -5,4 +5,5 @@ warning /Warning: .+ is not a source package, or was removed or renamed. Please warning /Warning: Download of .+ sources failed./ warning /Warning: .+ contains invalid yaml, please fix./ warning /Warnung: old temp directories found in .+/ +warning /Warnung: lock .+ still exists, exiting./ |