reproducible: update pbuilder in maintenance job too, run maintenance jobs more often, run pbuilder setup jobs less often

4 files changed, 86 insertions, 65 deletions

diff --git a/TODO b/TODO
index 865b27ac..dcfd2c01 100644
--- a/TODO
+++ b/TODO
@@ -182,7 +182,7 @@ properties:
 * lesser prio
 ** get the content for freebsd/netbsd/openwrt/coreboot "<h2>status of $1</h2>" from notes.git/friends.yaml or such
 ** document (in README) the multihost setup
-** the setup pbuilder+schroot scripts should detect network problems such as "Bad gateway" and sleep 5m and retry once
+** the setup pbuilder+schroot setup scripts should detect network problems such as "Bad gateway" and sleep 5m and retry once
 ** more graphs:
 *** graph average build duration by day
 *** graph packages in testing+unstable which need to be fixed
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index 890185f0..7b3ae8bc 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -260,14 +260,17 @@ if ! $DIRTY ; then
 	echo
 fi
 
-echo "$(date -u) - updating the schroots now..."
+echo "$(date -u) - updating the schroots and pbuilder now..."
+set +e
 ARCH=$(dpkg --print-architecture)
 for s in $SUITES ; do
 	if [ "$ARCH" = "armhf" ] && [ "$s" != "unstable" ] ; then
 		continue
 	fi
+	#
+	# schroot update
+	#
 	echo "$(date -u) - updating the $s/$ARCH schroot now."
-	set +e
 	for i in 1 2 3 ; do
 		schroot --directory /root -u root -c source:jenkins-reproducible-$s -- apt-get update
 		RESULT=$?
@@ -283,4 +286,31 @@ for s in $SUITES ; do
 	if [ $RESULT -eq 1 ] ; then
 		echo "Warning: failed to update the $s/$ARCH schroot."
 	fi
+	#
+	# pbuilder update
+	#
+	echo "$(date -u) - updating pbuilder for $s/$ARCH now."
+	# use host apt proxy configuration for pbuilder
+	if [ ! -z "$http_proxy" ] ; then
+		pbuilder_http_proxy="--http-proxy $http_proxy"
+	fi
+	for i in 1 2 3 ; do
+		sudo pbuilder --update $pbuilder_http_proxy --basetgz /var/cache/pbuilder/$s-reproducible-base.tgz
+		RESULT=$?
+		if [ $RESULT -eq 1 ] ; then
+			# sleep 31-100 secs
+			echo "Sleeping some time... (to workaround network problems like 'Hash Sum mismatch'...)"
+			/bin/sleep $(echo "scale=1 ; ($(shuf -i 1-700 -n 1)/10)+30" | bc )
+			echo "$(date -u) - Retrying to update pbuilder for $s/$ARCH."
+		elif [ $RESULT -eq 0 ] ; then
+			continue
+		fi
+	done
+	if [ $RESULT -eq 1 ] ; then
+		echo "Warning: failed to update pbuilder for $s/$ARCH."
+	fi
 done
+set -e
+echo "$(date -u) - the end."
+
+
diff --git a/bin/reproducible_setup_pbuilder.sh b/bin/reproducible_setup_pbuilder.sh
index 837f3b4f..f30d686b 100755
--- a/bin/reproducible_setup_pbuilder.sh
+++ b/bin/reproducible_setup_pbuilder.sh
@@ -114,14 +114,6 @@ setup_pbuilder() {
 }
 
 #
-# update pbuilder for reproducible builds
-#
-update_pbuilder() {
-	NAME=$1
-	sudo pbuilder --update --http-proxy $http_proxy --basetgz /var/cache/pbuilder/${NAME}.tgz
-}
-
-#
 # main
 #
 BASETGZ=/var/cache/pbuilder/$SUITE-reproducible-base.tgz
@@ -135,7 +127,6 @@ if [ -n "$OLDSTAMP" ] || [ ! -f $BASETGZ ] || [ ! -f $STAMP ] ; then
 	fi
 	setup_pbuilder $SUITE $SUITE-reproducible-base dpkg dpkg-dev debhelper
 else
-	echo "Updating $BASETGZ..."
-	update_pbuilder $SUITE-reproducible-base
+	echo "Echo $BASETGZ not old enough, doing nothing..."
 fi
 echo
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index a7791948..623a6640 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -440,128 +440,128 @@
     name: reproducible
     jobs:
         - '{name}_maintenance':
-            my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
-            my_timed: '5 0,4,8,12,16,20 * * *'
+            my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+            my_timed: '5 0,3,6,9,12,15,18,21 * * *'
             my_shell: '/srv/jenkins/bin/reproducible_maintenance.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: ''
         - '{name}_maintenance_armhf_bpi0':
-            my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
-            my_timed: '5 0,4,8,12,16,20 * * *'
+            my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+            my_timed: '5 0,3,6,9,12,15,18,21 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'bpi0'
         - '{name}_maintenance_armhf_cbxi4pro0':
-            my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
-            my_timed: '5 0,4,8,12,16,20 * * *'
+            my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+            my_timed: '5 0,3,6,9,12,15,18,21 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'cbxi4pro0'
         - '{name}_maintenance_armhf_hb0':
-            my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
-            my_timed: '5 0,4,8,12,16,20 * * *'
+            my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+            my_timed: '5 0,3,6,9,12,15,18,21 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'hb0'
         - '{name}_maintenance_armhf_wbq0':
-            my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
-            my_timed: '5 0,4,8,12,16,20 * * *'
+            my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+            my_timed: '5 0,3,6,9,12,15,18,21 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'wbq0'
         - '{name}_maintenance_amd64_profitbricks1':
-            my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
-            my_timed: '5 0,4,8,12,16,20 * * *'
+            my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+            my_timed: '5 0,3,6,9,12,15,18,21 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks1'
         - '{name}_maintenance_amd64_profitbricks2':
-            my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.'
-            my_timed: '5 0,4,8,12,16,20 * * *'
+            my_description: 'Do some maintenance: check for old directories laying around, do backups, update pbuilder and schroots, etc.'
+            my_timed: '5 0,3,6,9,12,15,18,21 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks2'
         - '{name}_setup_pbuilder_testing':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '23 7 * * *'
             my_shell: '/srv/jenkins/bin/reproducible_setup_pbuilder.sh testing'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: ''
         - '{name}_setup_pbuilder_unstable_armhf_bpi0':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '42 7 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'bpi0'
         - '{name}_setup_pbuilder_unstable_armhf_cbxi4pro0':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '42 7 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'cbxi4pro0'
         - '{name}_setup_pbuilder_unstable_armhf_hb0':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '42 7 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'hb0'
         - '{name}_setup_pbuilder_unstable_armhf_wbq0':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '42 7 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'wbq0'
         - '{name}_setup_pbuilder_testing_amd64_profitbricks1':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '23 7 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks1'
         - '{name}_setup_pbuilder_testing_amd64_profitbricks2':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '23 7 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks2'
         - '{name}_setup_pbuilder_unstable':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '42 7 * * *'
             my_shell: '/srv/jenkins/bin/reproducible_setup_pbuilder.sh unstable'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: ''
         - '{name}_setup_pbuilder_unstable_amd64_profitbricks1':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '42 7 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks1'
         - '{name}_setup_pbuilder_unstable_amd64_profitbricks2':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '42 7 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks2'
         - '{name}_setup_pbuilder_experimental':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '33 7 * * *'
             my_shell: '/srv/jenkins/bin/reproducible_setup_pbuilder.sh experimental'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: ''
         - '{name}_setup_pbuilder_experimental_amd64_profitbricks1':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '33 7 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks1'
         - '{name}_setup_pbuilder_experimental_amd64_profitbricks2':
-            my_description: 'Setup and update pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
-            my_timed: '23 0,4,8,12,16,20 * * *'
+            my_description: 'Setup pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
+            my_timed: '33 7 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks2'
         - '{name}_setup_schroot_unstable_diffoscope_amd64_jenkins':
             my_description: 'Setup schroot for running diffoscope in a unstable environment (this is needed to be able to correctly investigate haskell binaries...)'
-            my_timed: '23 0 * * *'
+            my_timed: '42 0 * * *'
             my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable-diffoscope unstable diffoscope locales-all'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: ''
@@ -573,19 +573,19 @@
             my_node: ''
         - '{name}_setup_schroot_unstable':
             my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
-            my_timed: '23 1 * * *'
+            my_timed: '42 1 * * *'
             my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable unstable botch'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: ''
         - '{name}_setup_schroot_unstable_amd64_profitbricks1':
             my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
-            my_timed: '23 1 * * *'
+            my_timed: '42 1 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks1'
         - '{name}_setup_schroot_unstable_amd64_profitbricks2':
             my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
-            my_timed: '23 1 * * *'
+            my_timed: '42 1 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks2'
@@ -597,31 +597,31 @@
             my_node: ''
         - '{name}_setup_schroot_unstable_armhf_bpi0':
             my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
-            my_timed: '23 1 * * *'
+            my_timed: '42 1 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'bpi0'
         - '{name}_setup_schroot_unstable_armhf_cbxi4pro0':
             my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
-            my_timed: '23 1 * * *'
+            my_timed: '42 1 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'cbxi4pro0'
         - '{name}_setup_schroot_unstable_armhf_hb0':
             my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
-            my_timed: '23 1 * * *'
+            my_timed: '42 1 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'hb0'
         - '{name}_setup_schroot_unstable_armhf_wbq0':
             my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.'
-            my_timed: '23 1 * * *'
+            my_timed: '42 1 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'wbq0'
         - '{name}_setup_schroot_testing_amd64_profitbricks1':
             my_description: 'Setup testing schroot for fetching source packages for the builder jobs.'
-            my_timed: '23 1 * * *'
+            my_timed: '42 1 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks1'
@@ -633,19 +633,19 @@
             my_node: 'profitbricks2'
         - '{name}_setup_schroot_experimental':
             my_description: 'Setup experimental schroot for fetching source packages for the builder jobs.'
-            my_timed: '23 1 * * *'
+            my_timed: '33 1 * * *'
             my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible reproducible-experimental experimental'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: ''
         - '{name}_setup_schroot_experimental_amd64_profitbricks1':
             my_description: 'Setup experimental schroot for fetching source packages for the builder jobs.'
-            my_timed: '23 1 * * *'
+            my_timed: '33 1 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks1'
         - '{name}_setup_schroot_experimental_amd64_profitbricks2':
             my_description: 'Setup experimental schroot for fetching source packages for the builder jobs.'
-            my_timed: '23 1 * * *'
+            my_timed: '33 1 * * *'
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: 'profitbricks2'