summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rwxr-xr-xbin/reproducible_build.sh19
-rwxr-xr-xbin/reproducible_common.sh4
-rwxr-xr-xbin/reproducible_setup_pbuilder.sh (renamed from bin/reproducible_setup.sh)0
-rwxr-xr-xbin/reproducible_setup_schroot.sh118
-rw-r--r--job-cfg/reproducible.yaml4
-rw-r--r--logparse/reproducible.rules1
6 files changed, 144 insertions, 2 deletions
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index d9b558c3..4a7fc84b 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -134,10 +134,29 @@ else
LOGFILE=$(ls ${SRCPACKAGE}_${EVERSION}.dsc)
LOGFILE=$(echo ${LOGFILE%.dsc}.debbindiff.html)
BUILDINFO=${SRCPACKAGE}_${EVERSION}_amd64.buildinfo
+ # the schroot for debbindiff gets updated once a day. wait patiently if that's the case
+ if [ -f $DBDCHROOT_WRITELOCK ] || [ -f $DBDCHROOT_READLOCK ] ; then
+ for i in $(seq 0 100) ; do
+ sleep 15
+ echo "sleeping 15s, debbindiff schroot is locked."
+ if [ ! -f $DBDCHROOT_WRITELOCK ] && [ ! -f $DBDCHROOT_READLOCK ] ; then
+ break
+ fi
+ done
+ if [ -f $DBDCHROOT_WRITELOCK ] || [ -f $DBDCHROOT_READLOCK ] ; then
+ echo "Warning: lock $DBDCHROOT_WRITELOCK or [ -f $DBDCHROOT_READLOCK ] still exists, exiting."
+ exit 1
+ fi
+ else
+ # we create (more) read-lock(s) but stop on write locks...
+ # write locks are only done by the schroot setup job
+ touch $DBDCHROOT_READLOCK
+ fi
( timeout 15m schroot --directory /tmp -c source:jenkins-reproducible-sid debbindiff -- --html $TMPDIR/${LOGFILE} $TMPDIR/b1/${SRCPACKAGE}_${EVERSION}_amd64.changes $TMPDIR/b2/${SRCPACKAGE}_${EVERSION}_amd64.changes ) 2>&1 >> ${RBUILDLOG}
RESULT=$?
set +x
set -e
+ rm -f $DBDCHROOT_READLOCK
echo | tee -a ${RBUILDLOG}
if [ $RESULT -eq 124 ] ; then
echo "$(date) - debbindiff was killed after running into timeouot... maybe there is still $JENKINS_URL/userContent/dbd/${LOGFILE}" | tee -a ${RBUILDLOG}
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index 775033c7..f00a4797 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -100,6 +100,10 @@ elif [ ! -f ${PACKAGES_DB} ] ; then
EOF
fi
+# common variables
+DBDCHROOT_READLOCK=/var/lib/jenkins/reproducible-dbdchroot.readlock
+DBDCHROOT_WRITELOCK=/var/lib/jenkins/reproducible-dbdchroot.writelock
+
# shop trailing slash
JENKINS_URL=${JENKINS_URL:0:-1}
diff --git a/bin/reproducible_setup.sh b/bin/reproducible_setup_pbuilder.sh
index 6075789f..6075789f 100755
--- a/bin/reproducible_setup.sh
+++ b/bin/reproducible_setup_pbuilder.sh
diff --git a/bin/reproducible_setup_schroot.sh b/bin/reproducible_setup_schroot.sh
new file mode 100755
index 00000000..70f40f5a
--- /dev/null
+++ b/bin/reproducible_setup_schroot.sh
@@ -0,0 +1,118 @@
+#!/bin/bash
+
+# Copyright 2012-2014 Holger Levsen <holger@layer-acht.org>
+# Copyright 2013 Antonio Terceiro <terceiro@debian.org>
+# Copyright 2014 Joachim Breitner <nomeata@debian.org>
+# released under the GPLv=2
+
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# bootstraps a new chroot for schroot, and then moves it into the right location
+
+# $1 = schroot name
+# $2 = base distro
+# $3 $4 ... = extra packages to install
+
+if [ $# -lt 2 ]; then
+ echo "usage: $0 TARGET DISTRO [backports] CMD [ARG1 ARG2 ...]"
+ exit 1
+fi
+TARGET="$1"
+shift
+DISTRO="$1"
+shift
+
+if [ "$1" == "backports" ] ; then
+ BACKPORTS="deb $MIRROR ${DISTRO}-backports main"
+ BACKPORTSSRC="deb-src $MIRROR ${DISTRO}-backports main"
+ shift
+fi
+
+if [ ! -d "$CHROOT_BASE" ]; then
+ echo "Directory $CHROOT_BASE does not exist, aborting."
+ exit 1
+fi
+
+export CHROOT_TARGET=$(mktemp -d -p $CHROOT_BASE/ schroot-install-$TARGET-XXXX)
+if [ -z "$CHROOT_TARGET" ]; then
+ echo "Could not create a directory to create the chroot in, aborting."
+ exit 1
+fi
+
+bootstrap() {
+ mkdir -p "$CHROOT_TARGET/etc/dpkg/dpkg.cfg.d"
+ echo force-unsafe-io > "$CHROOT_TARGET/etc/dpkg/dpkg.cfg.d/02dpkg-unsafe-io"
+
+ echo "Bootstraping $DISTRO into $CHROOT_TARGET now."
+ sudo debootstrap $DISTRO $CHROOT_TARGET $MIRROR
+
+ echo -e '#!/bin/sh\nexit 101' | sudo tee $CHROOT_TARGET/usr/sbin/policy-rc.d >/dev/null
+ sudo chmod +x $CHROOT_TARGET/usr/sbin/policy-rc.d
+ echo 'Acquire::http::Proxy "$http_proxy";' | sudo tee $CHROOT_TARGET/etc/apt/apt.conf.d/80proxy >/dev/null
+ echo "deb-src $MIRROR $DISTRO main" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list > /dev/null
+ echo "${BACKPORTS}" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list >/dev/null
+ echo "${BACKPORTSSRC}" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list >/dev/null
+
+ sudo chroot $CHROOT_TARGET apt-get update
+ if [ -n "$1" ] ; then
+ sudo chroot $CHROOT_TARGET apt-get install -y --no-install-recommends "$@"
+ fi
+}
+
+cleanup() {
+ if [ -d $CHROOT_TARGET ]; then
+ sudo rm -rf --one-file-system $CHROOT_TARGET || fuser -mv $CHROOT_TARGET
+ fi
+}
+trap cleanup INT TERM EXIT
+bootstrap $@
+
+trap - INT TERM EXIT
+
+# aquire a write lock in any case
+touch $DBDCHROOT_WRITELOCK
+if [ -f $DBDCHROOT_READLOCK ] ; then
+ # patiently wait for our users to using the schroot
+ for i in $(seq 0 100) ; do
+ sleep 15
+ echo "sleeping 15s, debbindiff schroot is locked and used."
+ if [ ! -f $DBDCHROOT_READLOCK ] ; then
+ break
+ fi
+ done
+ if [ -f $DBDCHROOT_READLOCK ] ; then
+ echo "Warning: lock $DBDCHROOT_READLOCK still exists, exiting."
+ exit 1
+ fi
+fi
+
+# pivot the new schroot in place
+rand=$RANDOM
+if [ -d $SCHROOT_BASE/"$TARGET" ]
+then
+ sudo mv $SCHROOT_BASE/"$TARGET" $SCHROOT_BASE/"$TARGET"-"$rand"
+fi
+
+sudo mv $CHROOT_TARGET $SCHROOT_BASE/"$TARGET"
+
+if [ -d $SCHROOT_BASE/"$TARGET"-"$rand" ]
+then
+ sudo rm -rf --one-file-system $SCHROOT_BASE/"$TARGET"-"$rand"
+fi
+
+# write the schroot config
+echo "Writing configuration"
+sudo tee /etc/schroot/chroot.d/jenkins-"$TARGET" <<-__END__
+ [jenkins-$TARGET]
+ description=Jenkins schroot $TARGET
+ directory=$SCHROOT_BASE/$TARGET
+ type=directory
+ root-users=jenkins
+ source-root-users=jenkins
+ union-type=aufs
+ __END__
+
+# remove the lock
+rm $DBDCHROOT_WRITELOCK
+
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index 12f51ac4..b43a918f 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -183,12 +183,12 @@
- '{name}_setup_pbuilder':
my_description: 'Setup pbuilder for reproducible builds as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
my_timed: '23 0 * * *'
- my_shell: '/srv/jenkins/bin/reproducible_setup.sh'
+ my_shell: '/srv/jenkins/bin/reproducible_setup_pbuilder.sh'
my_recipients: 'jenkins+debian-reproducible holger@layer-acht.org'
- '{name}_setup_schroot':
my_description: 'Setup schroot for running debbindiff in a sid environment, this is needed eg to be able to correctly investigate haskell binaries....'
my_timed: '23 0 * * *'
- my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible-sid sid debbindiff'
+ my_shell: '/srv/jenkins/bin/reproducible_setup_schroot.sh reproducible-sid sid debbindiff'
my_recipients: 'holger@layer-acht.org'
- '{name}_scheduler':
my_description: 'Schedule packages to be tested on reproducible building.'
diff --git a/logparse/reproducible.rules b/logparse/reproducible.rules
index 08a0a289..03628085 100644
--- a/logparse/reproducible.rules
+++ b/logparse/reproducible.rules
@@ -5,4 +5,5 @@ warning /Warning: .+ is not a source package, or was removed or renamed. Please
warning /Warning: Download of .+ sources failed./
warning /Warning: .+ contains invalid yaml, please fix./
warning /Warnung: old temp directories found in .+/
+warning /Warnung: lock .+ still exists, exiting./