diff options
| -rw-r--r-- | TODO | 2 | ||||
| -rwxr-xr-x | bin/reproducible_maintenance.sh | 28 | ||||
| -rw-r--r-- | logparse/reproducible.rules | 1 |
3 files changed, 28 insertions, 3 deletions
@@ -134,8 +134,6 @@ The plan is to run a jenkins.d.o host, which is maintained by DSA, but we are ma *** new table in pkg/test history page: schedule - if that package is currently scheduled *** add link to pkg set(s) if pkg is member of some ** link pkg sets and issues, that is: at least show packages without issues on pkg set pages, maybe also some issues which need actions (like uninvestigated test failures) -** make maintenance job detect and reschedule logs with: 'E: 10mount: error: Directory '.*' does not exist' -** make maintenance job detect and reschedule logs with: '^Bus Error$' ** use schroot tarballs (gzipped), moves are atomic then ** notes related: *** #786396: classify issue by "toolchain" or "package" fix needed: show bugs which block a bug diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh index d6ec25f8..acf00556 100755 --- a/bin/reproducible_maintenance.sh +++ b/bin/reproducible_maintenance.sh @@ -170,7 +170,7 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then # only grep through the last 5h (300 minutes) of builds... # (ignore "*None.rbuild.log" because these are build which were just started) # this job runs every 4h - echo "$(date -u) - Rescheduling failed builds." + echo "$(date -u) - Rescheduling failed builds due to network issues." FAILED_BUILDS=$(find $BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: Failed to fetch.*(Unable to connect to|Connection failed|Size mismatch|Cannot initiate the connection to|Bad Gateway)' {} \; || true) if [ ! -z "$FAILED_BUILDS" ] ; then echo @@ -197,6 +197,32 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then fi # + # find failed builds due to diffoscope schroot problems and reschedule them + # + # only grep through the last 5h (300 minutes) of builds... + # (ignore "*None.rbuild.log" because these are build which were just started) + # this job runs every 4h + echo "$(date -u) - Rescheduling failed builds due to diffoscope schroot issues." + FAILED_BUILDS=$(find $BASE/rbuild -type f ! -name "*None.rbuild.log" ! -mmin +300 -exec zgrep -l -E 'E: 10mount: error: Directory' {} \; || true) + if [ ! -z "$FAILED_BUILDS" ] ; then + echo + echo "Warning: The following builds have failed due to diffoscope schroot problems and will be rescheduled now:" + echo "$FAILED_BUILDS" + echo "Actually not doing so yet… please investigate manually." + echo + echo "Rescheduling packages: " + REQUESTER="jenkins maintenance job" + REASON="maintenance reschedule: reschedule builds which failed due to diffoscope schroot errors" + for SUITE in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f8 | sort -u) ; do + for ARCH in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f9 | sort -u) ; do + CANDIDATES=$(for PKG in $(echo $FAILED_BUILDS | sed "s# #\n#g" | grep "/$SUITE/$ARCH/" | cut -d "/" -f10 | cut -d "_" -f1) ; do echo "$PKG" ; done) + echo "_would_ schedule_packages $CANDIDATES" + done + done + DIRTY=true + fi + + # # find packages which build didnt end correctly # echo "$(date -u) - Rescheduling builds which didn't end correctly." diff --git a/logparse/reproducible.rules b/logparse/reproducible.rules index 09ccbfe5..6536f4c1 100644 --- a/logparse/reproducible.rules +++ b/logparse/reproducible.rules @@ -15,3 +15,4 @@ warning /Warning: too much difference for .+, aborting. Please investigate and u warning /Warning: couldn't delete old files from.+/ warning /Warning: failed to update the .+ schroot./ warning /Warning: failed to update Archlinux schroot./ +warning /Warning: The following builds have failed due to diffoscope schroot problems and will be rescheduled.+/ |