From fa67582a14dffb28963cac6d3ad9ed1ae2eb9600 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Sat, 9 Jan 2016 14:33:46 +0100 Subject: reproducible maintenance: detect temporary diffoscope schroot problems and (prepare to) reschedule automatically (not doing so yet) --- bin/reproducible_maintenance.sh | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh index d6ec25f8..acf00556 100755 --- a/bin/reproducible_maintenance.sh +++ b/bin/reproducible_maintenance.sh @@ -170,7 +170,7 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then # only grep through the last 5h (300 minutes) of builds... # (ignore "*None.rbuild.log" because these are build which were just started) # this job runs every 4h - echo "$(date -u) - Rescheduling failed builds." + echo "$(date -u) - Rescheduling failed builds due to network issues." FAILED_BUILDS=$(find $BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: Failed to fetch.*(Unable to connect to|Connection failed|Size mismatch|Cannot initiate the connection to|Bad Gateway)' {} \; || true) if [ ! -z "$FAILED_BUILDS" ] ; then echo @@ -196,6 +196,32 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then DIRTY=true fi + # + # find failed builds due to diffoscope schroot problems and reschedule them + # + # only grep through the last 5h (300 minutes) of builds... + # (ignore "*None.rbuild.log" because these are build which were just started) + # this job runs every 4h + echo "$(date -u) - Rescheduling failed builds due to diffoscope schroot issues." + FAILED_BUILDS=$(find $BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: 10mount: error: Directory' {} \; || true) + if [ ! -z "$FAILED_BUILDS" ] ; then + echo + echo "Warning: The following builds have failed due to diffoscope schroot problems and will be rescheduled now:" + echo "$FAILED_BUILDS" + echo "Actually not doing so yet… please investigate manually." + echo + echo "Rescheduling packages: " + REQUESTER="jenkins maintenance job" + REASON="maintenance reschedule: reschedule builds which failed due to diffoscope schroot errors" + for SUITE in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f8 | sort -u) ; do + for ARCH in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f9 | sort -u) ; do + CANDIDATES=$(for PKG in $(echo $FAILED_BUILDS | sed "s# #\n#g" | grep "/$SUITE/$ARCH/" | cut -d "/" -f10 | cut -d "_" -f1) ; do echo "$PKG" ; done) + echo "_would_ schedule_packages $CANDIDATES" + done + done + DIRTY=true + fi + # # find packages which build didnt end correctly # -- cgit v1.2.3-70-g09d2