summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHolger Levsen <holger@layer-acht.org>2015-09-21 14:17:10 +0200
committerHolger Levsen <holger@layer-acht.org>2015-09-21 14:17:10 +0200
commitf521b26f9c308ca6c1baa4ee0589d78870717457 (patch)
tree50e4e88a40207b0db988025019be57aa1c2574fb
parentdb68a8c9089e85182dd01aaf3b469d68c69b2aa2 (diff)
downloadjenkins.debian.net-f521b26f9c308ca6c1baa4ee0589d78870717457.tar.xz
reproducible: remove stale builds from at the beginning of the build and send a daily mail about these from the maintenance job
-rw-r--r--TODO1
-rwxr-xr-xbin/reproducible_build.sh9
-rwxr-xr-xbin/reproducible_html_live_status.py2
-rwxr-xr-xbin/reproducible_maintenance.sh10
4 files changed, 20 insertions, 2 deletions
diff --git a/TODO b/TODO
index 007f16a5..efaf4fc1 100644
--- a/TODO
+++ b/TODO
@@ -181,7 +181,6 @@ properties:
** new page: packages which are orphaned but have a reproducible usertagged patch
** cleanup old testing + diffscope schroots on armhf nodes, diffoscope schroots on pb-amd64 nodes and old pbuilder setups on jenkins
** reproducible_build.sh changes needed:
-*** should look for other builds "running" as this build job in the db on start and cleanup these from the db and prepare bits for a daily mail ...
*** fix: "DIFFOSCOPE='E: Failed to change to directory ‘/tmp’: Permission denied" - maybe by making sure the cause is gone…
*** race condition with itself? https://jenkins.debian.net/job/reproducible_builder_armhf_6/716/console says Warning, package php5 in unstable on armhf is probably already building elsewhere, exiting. Please check https://jenkins.debian.net/job/reproducible_builder_armhf_6/716/ which does not compute.
*** diffoscope needs to be run on the target arch...
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index 6919b768..2d6bce0a 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -415,6 +415,15 @@ init_package_build() {
echo "============================================================================="
echo "Initialising reproducibly build of ${SRCPACKAGE} in ${SUITE} on ${ARCH} on $(hostname -f) now. $ANNOUNCE"
echo "============================================================================="
+ # remove previous build attempts which didnt finish correctly
+ BUILDER_PREFIX="${JOB_NAME#reproducible_builder_}/"
+ BAD_BUILDS=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT package_id, date_build_started, builder FROM schedule WHERE builder LIKE '${BUILDER_PREFIX}%'")
+ if [ ! -z "$BAD_BUILDS" ] ; then
+ # stale_builds.txt is mailed once a day by reproducible_maintenance.sh
+ echo "$(date -u) - stale builds detected, cleaning up database from these entries:" | tee -a /var/lib/jenkins/stale_builds.txt
+ echo -n $BAD_BUILDS | tee -a /var/lib/jenkins/stale_builds.txt
+ sqlite3 -init $INIT ${PACKAGES_DB} "UPDATE schedule SET date_build_started='' WHERE builder LIKE '${BUILDER_PREFIX}%'"
+ fi
# mark build attempt
if [ -z "$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT date_build_started FROM schedule WHERE package_id = '$SRCPKGID'")" ] ; then
sqlite3 -init $INIT ${PACKAGES_DB} "UPDATE schedule SET date_build_started='$DATE', builder='$BUILDER' WHERE package_id = '$SRCPKGID'"
diff --git a/bin/reproducible_html_live_status.py b/bin/reproducible_html_live_status.py
index 334a31b5..1d55bb39 100755
--- a/bin/reproducible_html_live_status.py
+++ b/bin/reproducible_html_live_status.py
@@ -78,7 +78,7 @@ def generate_live_status_table(arch):
counter += 1
# the numbers 17 and 9 should really be derived from /var/lib/jenkins/jobs/reproducible_builder_${arch}_* instead of being hard-coded here...
if ( arch == 'amd64' and counter == 17 ) or ( arch == 'armhf' and counter == 9 ):
- html += '<tr><td colspan="10">There are more builds marked as currently building in the database than there are ' + arch + ' build jobs. This does not compute. Please cleanup and please automate cleanup.</td></tr>'
+ html += '<tr><td colspan="10">There are more builds marked as currently building in the database than there are ' + arch + ' build jobs. This does not compute, please investigate and fix the cause.</td></tr>'
suite = row[1]
arch = row[2]
pkg = row[3]
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index c577a659..0fd38938 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -336,6 +336,16 @@ if [ ! -z "$BADPERMS" ] ; then
echo
fi
+# once a day, send mail about stale builds
+if [ "$HOSTNAME" = "jenkins" ] && [ $(date -u +%H) -eq 0 ] ; then
+ if [ -s /var/lib/jenkins/stale_builds.txt ] ; then
+ TMPFILE=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXXXX)
+ mv /var/lib/jenkins/stale_builds.txt $TMPFILE
+ cat $TMPFILE | mail -s "stale builds found" qa-jenkins-scm@lists.alioth.debian.org
+ rm -f $TMPFILE
+ fi
+fi
+
if ! $DIRTY ; then
echo "$(date -u ) - Everything seems to be fine."
echo