diff options
| author | Holger Levsen <holger@layer-acht.org> | 2015-10-10 02:16:09 +0200 |
|---|---|---|
| committer | Holger Levsen <holger@layer-acht.org> | 2015-10-10 02:16:09 +0200 |
| commit | f11b3cc452f10a7e69746d9c3837243545332ea9 (patch) | |
| tree | 80c4d2e0bd706e2b6da1df61d4679e68e8198123 | |
| parent | 17621e8bd607252c085c35f8812c3a46946658d1 (diff) | |
| download | jenkins.debian.net-f11b3cc452f10a7e69746d9c3837243545332ea9.tar.xz | |
reproducible: refactor, turn remote error irc notifications into mails
| -rwxr-xr-x | bin/reproducible_build.sh | 14 | ||||
| -rwxr-xr-x | bin/reproducible_maintenance.sh | 2 |
2 files changed, 13 insertions, 3 deletions
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh index e131d87f..b96c1d7a 100755 --- a/bin/reproducible_build.sh +++ b/bin/reproducible_build.sh @@ -275,12 +275,22 @@ handle_unhandled() { MESSAGE="$BUILD_URL met an unhandled $1, please check." echo "$MESSAGE" irc_message "$MESSAGE" + # no need to slow down + exec /srv/jenkins/bin/abort.sh + exit 0 +} + +handle_remote_error() { + unregister_build + MESSAGE="$BUILD_URL got remote $1" + echo "$(date -u ) - $MESSAGE" | tee -a /var/log/jenkins/reproducible-remote-error.log echo "Sleeping 5m before aborting the job." sleep 5m exec /srv/jenkins/bin/abort.sh exit 0 } + handle_enospace() { unregister_build MESSAGE="$BUILD_URL hit diskspace issues with $SRCPACKAGE on $SUITE/$ARCH, sleeping 30m." @@ -607,7 +617,7 @@ remote_build() { if [ $RESULT -eq 148 ] ; then handle_404 elif [ $RESULT -ne 0 ] ; then - handle_unhandled "exit code from remote build job" + handle_remote_error "with exit code $RESULT from $NODE for build #$BUILDNR for ${SRCPACKAGE} on ${SUITE}/${ARCH}" fi rsync -e "ssh -p $PORT" -r $NODE:$TMPDIR/b$BUILDNR $TMPDIR/ RESULT=$? @@ -617,7 +627,7 @@ remote_build() { rsync -e "ssh -p $PORT" -r $NODE:$TMPDIR/b$BUILDNR $TMPDIR/ RESULT=$? if [ $RESULT -ne 0 ] ; then - handle_unhandled "error when rsyncing remote build results" + handle_remote_error "when rsyncing remote build #$BUILDNR results from $NODE" fi fi ls -R $TMPDIR diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh index 88ea9dfa..7f683128 100755 --- a/bin/reproducible_maintenance.sh +++ b/bin/reproducible_maintenance.sh @@ -329,7 +329,7 @@ fi # daily mails if [ "$HOSTNAME" = "$MAINNODE" ] && [ $(date -u +%H) -eq 0 ] ; then # once a day, send mail about builder problems - for PROBLEM in /var/log/jenkins/stale_builds.log /var/log/jenkins/reproducible-race-conditions.log /var/log/jenkins/reproducible-diskspace-issues.log ; do + for PROBLEM in /var/log/jenkins/stale_builds.log /var/log/jenkins/reproducible-race-conditions.log /var/log/jenkins/reproducible-diskspace-issues.log /var/log/jenkins/reproducible-remote-error.log; do if [ -s $PROBLEM ] ; then TMPFILE=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXXXX) mv $PROBLEM $TMPFILE |