reproducible: remove stale builds from at the beginning of the build and send a daily mail about these from the maintenance job

4 files changed, 20 insertions, 2 deletions

diff --git a/TODO b/TODO
index 007f16a5..efaf4fc1 100644
--- a/TODO
+++ b/TODO
@@ -181,7 +181,6 @@ properties:
 ** new page: packages which are orphaned but have a reproducible usertagged patch
 ** cleanup old testing + diffscope schroots on armhf nodes, diffoscope schroots on pb-amd64 nodes and old pbuilder setups on jenkins
 ** reproducible_build.sh changes needed:
-*** should look for other builds "running" as this build job in the db on start and cleanup these from the db and prepare bits for a daily mail ...
 *** fix: "DIFFOSCOPE='E: Failed to change to directory ‘/tmp’: Permission denied" - maybe by making sure the cause is gone…
 *** race condition with itself? https://jenkins.debian.net/job/reproducible_builder_armhf_6/716/console says Warning, package php5 in unstable on armhf is probably already building elsewhere, exiting. Please check https://jenkins.debian.net/job/reproducible_builder_armhf_6/716/ which does not compute.
 *** diffoscope needs to be run on the target arch...
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index 6919b768..2d6bce0a 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -415,6 +415,15 @@ init_package_build() {
 	echo "============================================================================="
 	echo "Initialising reproducibly build of ${SRCPACKAGE} in ${SUITE} on ${ARCH} on $(hostname -f) now. $ANNOUNCE"
 	echo "============================================================================="
+	# remove previous build attempts which didnt finish correctly
+	BUILDER_PREFIX="${JOB_NAME#reproducible_builder_}/"
+	BAD_BUILDS=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT package_id, date_build_started, builder FROM schedule WHERE builder LIKE '${BUILDER_PREFIX}%'")
+	if [ ! -z "$BAD_BUILDS" ] ; then
+		# stale_builds.txt is mailed once a day by reproducible_maintenance.sh
+		echo "$(date -u) - stale builds detected, cleaning up database from these entries:" | tee -a /var/lib/jenkins/stale_builds.txt
+		echo -n $BAD_BUILDS | tee -a /var/lib/jenkins/stale_builds.txt
+		sqlite3 -init $INIT ${PACKAGES_DB} "UPDATE schedule SET date_build_started='' WHERE builder LIKE '${BUILDER_PREFIX}%'"
+	fi
 	# mark build attempt
 	if [ -z "$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT date_build_started FROM schedule WHERE package_id = '$SRCPKGID'")" ] ; then
 		sqlite3 -init $INIT ${PACKAGES_DB} "UPDATE schedule SET date_build_started='$DATE', builder='$BUILDER' WHERE package_id = '$SRCPKGID'"
diff --git a/bin/reproducible_html_live_status.py b/bin/reproducible_html_live_status.py
index 334a31b5..1d55bb39 100755
--- a/bin/reproducible_html_live_status.py
+++ b/bin/reproducible_html_live_status.py
@@ -78,7 +78,7 @@ def generate_live_status_table(arch):
         counter += 1
         # the numbers 17 and 9 should really be derived from /var/lib/jenkins/jobs/reproducible_builder_${arch}_* instead of being hard-coded here...
         if ( arch == 'amd64' and counter == 17 ) or ( arch == 'armhf' and counter == 9 ):
-             html += '<tr><td colspan="10">There are more builds marked as currently building in the database than there are ' + arch + ' build jobs. This does not compute. Please cleanup and please automate cleanup.</td></tr>'
+             html += '<tr><td colspan="10">There are more builds marked as currently building in the database than there are ' + arch + ' build jobs. This does not compute, please investigate and fix the cause.</td></tr>'
         suite = row[1]
         arch = row[2]
         pkg = row[3]
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index c577a659..0fd38938 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -336,6 +336,16 @@ if [ ! -z "$BADPERMS" ] ; then
     echo
 fi
 
+# once a day, send mail about stale builds
+if [ "$HOSTNAME" = "jenkins" ] && [ $(date -u +%H) -eq 0 ]  ; then
+	if [ -s /var/lib/jenkins/stale_builds.txt ] ; then
+		TMPFILE=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXXXX)
+		mv /var/lib/jenkins/stale_builds.txt $TMPFILE
+		cat $TMPFILE | mail -s "stale builds found" qa-jenkins-scm@lists.alioth.debian.org
+		rm -f $TMPFILE
+	fi
+fi
+
 if ! $DIRTY ; then
 	echo "$(date -u ) - Everything seems to be fine."
 	echo