summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHolger Levsen <holger@layer-acht.org>2016-01-09 14:33:46 +0100
committerHolger Levsen <holger@layer-acht.org>2016-01-09 14:33:46 +0100
commitfa67582a14dffb28963cac6d3ad9ed1ae2eb9600 (patch)
treed37b9d84f431424ea00ac8f267819cfbcb04c4c7
parent32b9be7509b1d8b7860d99b812dae6e4943d4d9c (diff)
downloadjenkins.debian.net-fa67582a14dffb28963cac6d3ad9ed1ae2eb9600.tar.xz
reproducible maintenance: detect temporary diffoscope schroot problems and (prepare to) reschedule automatically (not doing so yet)
-rw-r--r--TODO2
-rwxr-xr-xbin/reproducible_maintenance.sh28
-rw-r--r--logparse/reproducible.rules1
3 files changed, 28 insertions, 3 deletions
diff --git a/TODO b/TODO
index 476c9af3..e13e2d53 100644
--- a/TODO
+++ b/TODO
@@ -134,8 +134,6 @@ The plan is to run a jenkins.d.o host, which is maintained by DSA, but we are ma
*** new table in pkg/test history page: schedule - if that package is currently scheduled
*** add link to pkg set(s) if pkg is member of some
** link pkg sets and issues, that is: at least show packages without issues on pkg set pages, maybe also some issues which need actions (like uninvestigated test failures)
-** make maintenance job detect and reschedule logs with: 'E: 10mount: error: Directory '.*' does not exist'
-** make maintenance job detect and reschedule logs with: '^Bus Error$'
** use schroot tarballs (gzipped), moves are atomic then
** notes related:
*** #786396: classify issue by "toolchain" or "package" fix needed: show bugs which block a bug
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index d6ec25f8..acf00556 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -170,7 +170,7 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then
# only grep through the last 5h (300 minutes) of builds...
# (ignore "*None.rbuild.log" because these are build which were just started)
# this job runs every 4h
- echo "$(date -u) - Rescheduling failed builds."
+ echo "$(date -u) - Rescheduling failed builds due to network issues."
FAILED_BUILDS=$(find $BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: Failed to fetch.*(Unable to connect to|Connection failed|Size mismatch|Cannot initiate the connection to|Bad Gateway)' {} \; || true)
if [ ! -z "$FAILED_BUILDS" ] ; then
echo
@@ -197,6 +197,32 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then
fi
#
+ # find failed builds due to diffoscope schroot problems and reschedule them
+ #
+ # only grep through the last 5h (300 minutes) of builds...
+ # (ignore "*None.rbuild.log" because these are build which were just started)
+ # this job runs every 4h
+ echo "$(date -u) - Rescheduling failed builds due to diffoscope schroot issues."
+ FAILED_BUILDS=$(find $BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: 10mount: error: Directory' {} \; || true)
+ if [ ! -z "$FAILED_BUILDS" ] ; then
+ echo
+ echo "Warning: The following builds have failed due to diffoscope schroot problems and will be rescheduled now:"
+ echo "$FAILED_BUILDS"
+ echo "Actually not doing so yet… please investigate manually."
+ echo
+ echo "Rescheduling packages: "
+ REQUESTER="jenkins maintenance job"
+ REASON="maintenance reschedule: reschedule builds which failed due to diffoscope schroot errors"
+ for SUITE in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f8 | sort -u) ; do
+ for ARCH in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f9 | sort -u) ; do
+ CANDIDATES=$(for PKG in $(echo $FAILED_BUILDS | sed "s# #\n#g" | grep "/$SUITE/$ARCH/" | cut -d "/" -f10 | cut -d "_" -f1) ; do echo "$PKG" ; done)
+ echo "_would_ schedule_packages $CANDIDATES"
+ done
+ done
+ DIRTY=true
+ fi
+
+ #
# find packages which build didnt end correctly
#
echo "$(date -u) - Rescheduling builds which didn't end correctly."
diff --git a/logparse/reproducible.rules b/logparse/reproducible.rules
index 09ccbfe5..6536f4c1 100644
--- a/logparse/reproducible.rules
+++ b/logparse/reproducible.rules
@@ -15,3 +15,4 @@ warning /Warning: too much difference for .+, aborting. Please investigate and u
warning /Warning: couldn't delete old files from.+/
warning /Warning: failed to update the .+ schroot./
warning /Warning: failed to update Archlinux schroot./
+warning /Warning: The following builds have failed due to diffoscope schroot problems and will be rescheduled.+/