reproducible maintenance: detect temporary diffoscope schroot problems and (prepare to) reschedule automatically (not doing so yet)

3 files changed, 28 insertions, 3 deletions

diff --git a/TODO b/TODO
index 476c9af3..e13e2d53 100644
--- a/TODO
+++ b/TODO
@@ -134,8 +134,6 @@ The plan is to run a jenkins.d.o host, which is maintained by DSA, but we are ma
 *** new table in pkg/test history page: schedule - if that package is currently scheduled
 *** add link to pkg set(s) if pkg is member of some
 ** link pkg sets and issues, that is: at least show packages without issues on pkg set pages, maybe also some issues which need actions (like uninvestigated test failures)
-** make maintenance job detect and reschedule logs with: 'E: 10mount: error: Directory '.*' does not exist'
-** make maintenance job detect and reschedule logs with: '^Bus Error$'
 ** use schroot tarballs (gzipped), moves are atomic then
 ** notes related:
 *** #786396: classify issue by "toolchain" or "package" fix needed: show bugs which block a bug
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index d6ec25f8..acf00556 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -170,7 +170,7 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then
 	# only grep through the last 5h (300 minutes) of builds...
 	# (ignore "*None.rbuild.log" because these are build which were just started)
 	# this job runs every 4h
-	echo "$(date -u) - Rescheduling failed builds."
+	echo "$(date -u) - Rescheduling failed builds due to network issues."
 	FAILED_BUILDS=$(find $BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: Failed to fetch.*(Unable to connect to|Connection failed|Size mismatch|Cannot initiate the connection to|Bad Gateway)' {} \; || true)
 	if [ ! -z "$FAILED_BUILDS" ] ; then
 		echo
@@ -197,6 +197,32 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then
 	fi
 
 	#
+	# find failed builds due to diffoscope schroot problems and reschedule them
+	#
+	# only grep through the last 5h (300 minutes) of builds...
+	# (ignore "*None.rbuild.log" because these are build which were just started)
+	# this job runs every 4h
+	echo "$(date -u) - Rescheduling failed builds due to diffoscope schroot issues."
+	FAILED_BUILDS=$(find $BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: 10mount: error: Directory' {} \; || true)
+	if [ ! -z "$FAILED_BUILDS" ] ; then
+		echo
+		echo "Warning: The following builds have failed due to diffoscope schroot problems and will be rescheduled now:"
+		echo "$FAILED_BUILDS"
+		echo "Actually not doing so yet… please investigate manually."
+		echo
+		echo "Rescheduling packages: "
+		REQUESTER="jenkins maintenance job"
+		REASON="maintenance reschedule: reschedule builds which failed due to diffoscope schroot errors"
+		for SUITE in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f8 | sort -u) ; do
+			for ARCH in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f9 | sort -u) ; do
+				CANDIDATES=$(for PKG in $(echo $FAILED_BUILDS | sed "s# #\n#g" | grep "/$SUITE/$ARCH/" | cut -d "/" -f10 | cut -d "_" -f1) ; do echo "$PKG" ; done)
+				echo "_would_ schedule_packages $CANDIDATES"
+			done
+		done
+		DIRTY=true
+	fi
+
+	#
 	# find packages which build didnt end correctly
 	#
 	echo "$(date -u) - Rescheduling builds which didn't end correctly."
diff --git a/logparse/reproducible.rules b/logparse/reproducible.rules
index 09ccbfe5..6536f4c1 100644
--- a/logparse/reproducible.rules
+++ b/logparse/reproducible.rules
@@ -15,3 +15,4 @@ warning /Warning: too much difference for .+, aborting. Please investigate and u
 warning /Warning: couldn't delete old files from.+/
 warning /Warning: failed to update the .+ schroot./
 warning /Warning: failed to update Archlinux schroot./
+warning /Warning: The following builds have failed due to diffoscope schroot problems and will be rescheduled.+/