summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rwxr-xr-xbin/reproducible_build.sh49
-rwxr-xr-xbin/reproducible_setup.sh21
-rwxr-xr-xbin/schroot-create.sh13
-rw-r--r--etc/schroot/default/fstab21
-rw-r--r--job-cfg/reproducible.yaml50
-rwxr-xr-xupdate_jdn.sh2
6 files changed, 96 insertions, 60 deletions
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index 0876de9f..ac78ea52 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -26,13 +26,14 @@ unschedule_from_db() {
# unmark build as properly finished
sqlite3 -init $INIT ${PACKAGES_DB} "DELETE FROM sources_scheduled WHERE name = '$SRCPACKAGE';"
# update html page for package
- set -x
+ set +x
process_packages $SRCPACKAGE
- echo "Successfully updated the database and updated the html file for the package."
- echo "Enjoy $JENKINS_URL/userContent/rb-pkg/$SRCPACKAGE.html"
+ echo
+ echo "Successfully updated the database and updated $JENKINS_URL/userContent/rb-pkg/$SRCPACKAGE.html"
+ echo
}
-TMPDIR=$(mktemp --tmpdir=$PWD -d)
+TMPDIR=$(mktemp --tmpdir=/srv/reproducible-results -d)
trap cleanup_all INT TERM EXIT
cd $TMPDIR
RESULT=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT name,date_scheduled FROM sources_scheduled WHERE date_build_started = '' ORDER BY date_scheduled LIMIT 1")
@@ -99,8 +100,8 @@ else
break
fi
done
- set -x
if ! $SUITABLE ; then
+ set -x
sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"not for us\", \"$DATE\")"
set +x
echo "Package ${SRCPACKAGE} (${VERSION}) shall only be build on \"$(echo "${ARCHITECTURES}" | xargs echo )\" and thus was skipped." | tee -a ${RBUILDLOG}
@@ -108,8 +109,10 @@ else
exit 0
fi
set +e
+ set -x
NUM_CPU=$(cat /proc/cpuinfo |grep ^processor|wc -l)
- ( timeout 15m nice ionice -c 3 sudo DEB_BUILD_OPTIONS="parallel=$NUM_CPU" pbuilder --build --debbuildopts "-b" --basetgz /var/cache/pbuilder/base-reproducible.tgz --distribution sid ${SRCPACKAGE}_*.dsc ) 2>&1 | tee -a ${RBUILDLOG}
+ ( timeout 12h nice ionice -c 3 sudo DEB_BUILD_OPTIONS="parallel=$NUM_CPU" pbuilder --build --debbuildopts "-b" --basetgz /var/cache/pbuilder/base-reproducible.tgz --distribution sid ${SRCPACKAGE}_*.dsc ) 2>&1 | tee -a ${RBUILDLOG}
+ set +x
if [ -f /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}_amd64.changes ] ; then
mkdir b1 b2
dcmd cp /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}_amd64.changes b1
@@ -117,12 +120,12 @@ else
# so first delete files from .dsc, then from .changes file
sudo dcmd rm /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}.dsc
sudo dcmd rm /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}_amd64.changes
- set +x
echo "============================================================================="
echo "Re-building ${SRCPACKAGE} now."
echo "============================================================================="
set -x
timeout 12h nice ionice -c 3 sudo DEB_BUILD_OPTIONS="parallel=$NUM_CPU" pbuilder --build --debbuildopts "-b" --basetgz /var/cache/pbuilder/base-reproducible.tgz --distribution sid ${SRCPACKAGE}_${EVERSION}.dsc
+ set +x
dcmd cp /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}_amd64.changes b2
# and again (see comment 5 lines above)
sudo dcmd rm /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}.dsc
@@ -131,25 +134,30 @@ else
LOGFILE=$(ls ${SRCPACKAGE}_${EVERSION}.dsc)
LOGFILE=$(echo ${LOGFILE%.dsc}.debbindiff.html)
BUILDINFO=${SRCPACKAGE}_${EVERSION}_amd64.buildinfo
- ( timeout 15m /var/lib/jenkins/debbindiff.git/debbindiff.py --html ./${LOGFILE} b1/${SRCPACKAGE}_${EVERSION}_amd64.changes b2/${SRCPACKAGE}_${EVERSION}_amd64.changes ) 2>&1 >> ${RBUILDLOG}
+ ( timeout 15m schroot --directory /tmp -c source:jenkins-reproducible-sid debbindiff -- --html $TMPDIR/${LOGFILE} $TMPDIR/b1/${SRCPACKAGE}_${EVERSION}_amd64.changes $TMPDIR/b2/${SRCPACKAGE}_${EVERSION}_amd64.changes ) 2>&1 >> ${RBUILDLOG}
RESULT=$?
+ set +x
set -e
echo | tee -a ${RBUILDLOG}
if [ $RESULT -eq 124 ] ; then
- echo "$(date) - debbindiff.py was killed after running into timeouot..." | tee -a ${RBUILDLOG}
+ echo "$(date) - debbindiff was killed after running into timeouot... maybe there is still $JENKINS_URL/userContent/dbd/${LOGFILE}" | tee -a ${RBUILDLOG}
elif [ $RESULT -eq 1 ] ; then
- echo "$(date) - debbindiff.py found issues, please investigate $JENKINS_URL/userContent/dbd/${LOGFILE}" | tee -a ${RBUILDLOG}
+ DEBBINDIFFOUT="debbindiff found issues, please investigate $JENKINS_URL/userContent/dbd/${LOGFILE}"
fi
if [ ! -f ./${LOGFILE} ] && [ -f b1/${BUILDINFO} ] ; then
- cp b1/${BUILDINFO} /var/lib/jenkins/userContent/buildinfo/
+ cp b1/${BUILDINFO} /var/lib/jenkins/userContent/buildinfo/ 2>&1 > /dev/null
figlet ${SRCPACKAGE}
- echo "debbindiff.py found no differences in the changes files, and a .buildinfo file also exist." | tee -a ${RBUILDLOG}
+ echo
+ echo "debbindiff found no differences in the changes files, and a .buildinfo file also exist." | tee -a ${RBUILDLOG}
echo "${SRCPACKAGE} built successfully and reproducibly." | tee -a ${RBUILDLOG}
sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"reproducible\", \"$DATE\")"
unschedule_from_db
else
- cp b1/${BUILDINFO} /var/lib/jenkins/userContent/buildinfo/ || true
+ echo | tee -a ${RBUILDLOG}
+ echo -n "$(date) - ${SRCPACKAGE} failed to build reproducibly " | tee -a ${RBUILDLOG}
+ cp b1/${BUILDINFO} /var/lib/jenkins/userContent/buildinfo/ 2>&1 > /dev/null || true
if [ -f ./${LOGFILE} ] ; then
+ echo -n "$DEBBINDIFFOUT" | tee -a ${RBUILDLOG}
# FIXME: work around debbindiff not having external CSS support (#764470)
# should really be fixed in debbindiff and just moved....
if grep -q "Generated by debbindiff 3" ./${LOGFILE} ; then
@@ -157,27 +165,26 @@ else
else
mv ./${LOGFILE} /var/lib/jenkins/userContent/dbd/
fi
+ else
+ echo -n ", debbindiff produced no output (which is strange)"
fi
- sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"unreproducible\", \"$DATE\")"
- unschedule_from_db
- set +x
- echo -n "${SRCPACKAGE} failed to build reproducibly" | tee -a ${RBUILDLOG}
if [ ! -f b1/${BUILDINFO} ] ; then
- echo "and a .buildinfo file is missing too." | tee -a ${RBUILDLOG}
+ echo " and a .buildinfo file is missing." | tee -a ${RBUILDLOG}
else
echo "." | tee -a ${RBUILDLOG}
fi
+ sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"unreproducible\", \"$DATE\")"
+ unschedule_from_db
fi
else
- sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"FTBFS\", \"$DATE\")"
- unschedule_from_db
set +x
echo "${SRCPACKAGE} failed to build from source."
+ sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"FTBFS\", \"$DATE\")"
+ unschedule_from_db
fi
fi
fi
-set -x
cd ..
cleanup_all
trap - INT TERM EXIT
diff --git a/bin/reproducible_setup.sh b/bin/reproducible_setup.sh
index c89be793..f5cab3d8 100755
--- a/bin/reproducible_setup.sh
+++ b/bin/reproducible_setup.sh
@@ -23,23 +23,10 @@ for PKG in linux cups zurl openclipart eigen3 xmds2 ; do
fi
done
-# FIXME: needed as long as there is no backport for debbindiff
-# or as long as we dont run it within pbuilder...
-# fetch git repo for the debbindiff command used in reproducible_build.sh
-WORKSPACE=$PWD
-cd /var/lib/jenkins
-if [ -d debbindiff.git ] ; then
- cd debbindiff.git
- git pull
-else
- git clone git://git.debian.org/git/reproducible/debbindiff.git debbindiff.git
-fi
-cd $WORKSPACE
-
+#
+# create script to configure a pbuilder chroot
+#
create_setup_tmpfile() {
- #
- # script to configure a pbuilder chroot
- #
cat > ${TMPFILE} <<- EOF
#
# this script is run within the pbuilder environment to further customize it
@@ -84,10 +71,10 @@ echo
EOF
}
-setup_pbuilder() {
#
# setup pbuilder for reproducible builds
#
+setup_pbuilder() {
echo "$(date) - creating /var/cache/pbuilder/${1}.tgz now..."
TMPFILE=$(mktemp)
create_setup_tmpfile
diff --git a/bin/schroot-create.sh b/bin/schroot-create.sh
index f59010b1..5f31e3cd 100755
--- a/bin/schroot-create.sh
+++ b/bin/schroot-create.sh
@@ -53,6 +53,11 @@ bootstrap() {
echo "deb-src $MIRROR $DISTRO main" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list > /dev/null
echo "${BACKPORTS}" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list >/dev/null
echo "${BACKPORTSSRC}" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list >/dev/null
+
+ sudo chroot $CHROOT_TARGET apt-get update
+ if [ -n "$1" ] ; then
+ sudo chroot $CHROOT_TARGET apt-get install -y --no-install-recommends "$@"
+ fi
}
cleanup() {
@@ -61,7 +66,7 @@ cleanup() {
fi
}
trap cleanup INT TERM EXIT
-bootstrap
+bootstrap $@
trap - INT TERM EXIT
@@ -91,8 +96,4 @@ sudo tee /etc/schroot/chroot.d/jenkins-"$TARGET" <<-__END__
union-type=aufs
__END__
-schroot --directory /root -c "source:jenkins-$TARGET" -u root -- apt-get update
-if [ -n "$1" ]
-then
- schroot --directory /root -c "source:jenkins-$TARGET" -u root -- apt-get install -y --no-install-recommends "$@"
-fi
+
diff --git a/etc/schroot/default/fstab b/etc/schroot/default/fstab
new file mode 100644
index 00000000..e54d0fb4
--- /dev/null
+++ b/etc/schroot/default/fstab
@@ -0,0 +1,21 @@
+# fstab: static file system information for chroots.
+# Note that the mount point will be prefixed by the chroot path
+# (CHROOT_PATH)
+#
+# <file system> <mount point> <type> <options> <dump> <pass>
+/proc /proc none rw,bind 0 0
+/sys /sys none rw,bind 0 0
+/dev /dev none rw,bind 0 0
+/dev/pts /dev/pts none rw,bind 0 0
+/home /home none rw,bind 0 0
+#/tmp /tmp none rw,bind 0 0
+/srv /srv none rw,bind 0 0
+
+# It may be desirable to have access to /run, especially if you wish
+# to run additional services in the chroot. However, note that this
+# may potentially cause undesirable behaviour on upgrades, such as
+# killing services on the host.
+#/run /run none rw,bind 0 0
+#/run/lock /run/lock none rw,bind 0 0
+#/dev/shm /dev/shm none rw,bind 0 0
+#/run/shm /run/shm none rw,bind 0 0
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index e04dad7e..f68c51b7 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -117,7 +117,15 @@
- job-template:
defaults: reproducible
- name: '{name}_setup'
+ name: '{name}_setup_pbuilder'
+
+- job-template:
+ defaults: reproducible
+ name: '{name}_setup_schroot'
+
+- job-template:
+ defaults: reproducible
+ name: '{name}_scheduler'
- job-template:
defaults: reproducible
@@ -147,28 +155,41 @@
defaults: reproducible_builder
name: '{name}_builder_gamma'
-- job-template:
- defaults: reproducible
- name: '{name}_scheduler'
+#
+# sometimes used for development:
+#
+#- job-template:
+# defaults: reproducible_builder
+# name: '{name}_builder_delta'
- project:
name: reproducible
jobs:
- - '{name}_setup':
+ - '{name}_setup_pbuilder':
my_description: 'Setup pbuilder for reproducible builds as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
my_timed: '23 0 * * *'
my_shell: '/srv/jenkins/bin/reproducible_setup.sh'
my_recipients: 'jenkins+debian-reproducible holger@layer-acht.org'
- - '{name}_html_indexes':
- my_description: 'Generates HTML results (indexes) for reproducible builds.'
- my_timed: '59 */2 * * *'
- my_shell: '/srv/jenkins/bin/reproducible_html_indexes.sh'
+ - '{name}_setup_schroot':
+ my_description: 'Setup schroot for running debbindiff in a sid environment, this is needed eg to be able to correctly investigate haskell binaries....'
+ my_timed: '23 0 * * *'
+ my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible-sid sid debbindiff'
+ my_recipients: 'holger@layer-acht.org'
+ - '{name}_scheduler':
+ my_description: 'Schedule packages to be tested on reproducible building.'
+ my_timed: '42 * * * *'
+ my_shell: '/srv/jenkins/bin/reproducible_scheduler.sh'
my_recipients: 'holger@layer-acht.org'
- '{name}_html_graphs':
my_description: 'Generates HTML results (stats with graphs) for reproducible builds.'
my_timed: '0 * * * *'
my_shell: '/srv/jenkins/bin/reproducible_html_graphs.sh'
my_recipients: 'holger@layer-acht.org'
+ - '{name}_html_indexes':
+ my_description: 'Generates HTML results (indexes) for reproducible builds.'
+ my_timed: '59 */2 * * *'
+ my_shell: '/srv/jenkins/bin/reproducible_html_indexes.sh'
+ my_recipients: 'holger@layer-acht.org'
- '{name}_html_dd_list':
my_description: 'Generates HTML results (dd-list) for reproducible builds.'
my_timed: '55 23 * * *'
@@ -194,9 +215,8 @@
my_timed: 'H/2 * * * *'
my_shell: '/srv/jenkins/bin/reproducible_build.sh'
my_recipients: 'holger@layer-acht.org'
- - '{name}_scheduler':
- my_description: 'Schedule packages to be tested on reproducible building.'
- my_timed: '42 * * * *'
- my_shell: '/srv/jenkins/bin/reproducible_scheduler.sh'
- my_recipients: 'holger@layer-acht.org'
-
+# - '{name}_builder_delta':
+# my_description: 'This is a builder job only used for development.'
+# my_timed: ''
+# my_shell: '/srv/jenkins/bin/reproducible_build_delta.sh'
+# my_recipients: 'holger@layer-acht.org'
diff --git a/update_jdn.sh b/update_jdn.sh
index eee5c280..34a22c0f 100755
--- a/update_jdn.sh
+++ b/update_jdn.sh
@@ -28,7 +28,7 @@ if ! mountpoint -q /srv/workspace; then
fi
# make sure needed directories exists
-for directory in /srv/jenkins /schroots; do
+for directory in /srv/jenkins /schroots /srv/reproducible-results ; do
if [ ! -d $directory ] ; then
sudo mkdir $directory
sudo chown jenkins.jenkins $directory
generated by cgit v1.2.3-54-g00ecf (git 2.44.0) at 2024-06-03 06:24:33 +0000