From f11b3cc452f10a7e69746d9c3837243545332ea9 Mon Sep 17 00:00:00 2001
From: Holger Levsen <holger@layer-acht.org>
Date: Sat, 10 Oct 2015 02:16:09 +0200
Subject: reproducible: refactor, turn remote error irc notifications into
 mails

---
 bin/reproducible_build.sh       | 14 ++++++++++++--
 bin/reproducible_maintenance.sh |  2 +-
 2 files changed, 13 insertions(+), 3 deletions(-)

(limited to 'bin')

diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index e131d87f..b96c1d7a 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -275,12 +275,22 @@ handle_unhandled() {
 	MESSAGE="$BUILD_URL met an unhandled $1, please check."
 	echo "$MESSAGE"
 	irc_message "$MESSAGE"
+	# no need to slow down
+	exec /srv/jenkins/bin/abort.sh
+	exit 0
+}
+
+handle_remote_error() {
+	unregister_build
+	MESSAGE="$BUILD_URL got remote $1"
+	echo "$(date -u ) - $MESSAGE" | tee -a /var/log/jenkins/reproducible-remote-error.log
 	echo "Sleeping 5m before aborting the job."
 	sleep 5m
 	exec /srv/jenkins/bin/abort.sh
 	exit 0
 }
 
+
 handle_enospace() {
 	unregister_build
 	MESSAGE="$BUILD_URL hit diskspace issues with $SRCPACKAGE on $SUITE/$ARCH, sleeping 30m."
@@ -607,7 +617,7 @@ remote_build() {
 	if [ $RESULT -eq 148 ] ; then
 		handle_404
 	elif [ $RESULT -ne 0 ] ; then
-		handle_unhandled "exit code from remote build job"
+		handle_remote_error "with exit code $RESULT from $NODE for build #$BUILDNR for ${SRCPACKAGE} on ${SUITE}/${ARCH}"
 	fi
 	rsync -e "ssh -p $PORT" -r $NODE:$TMPDIR/b$BUILDNR $TMPDIR/
 	RESULT=$?
@@ -617,7 +627,7 @@ remote_build() {
 		rsync -e "ssh -p $PORT" -r $NODE:$TMPDIR/b$BUILDNR $TMPDIR/
 		RESULT=$?
 		if [ $RESULT -ne 0 ] ; then
-			handle_unhandled "error when rsyncing remote build results"
+			handle_remote_error "when rsyncing remote build #$BUILDNR results from $NODE"
 		fi
 	fi
 	ls -R $TMPDIR
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index 88ea9dfa..7f683128 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -329,7 +329,7 @@ fi
 # daily mails
 if [ "$HOSTNAME" = "$MAINNODE" ] && [ $(date -u +%H) -eq 0 ]  ; then
 	# once a day, send mail about builder problems
-	for PROBLEM in /var/log/jenkins/stale_builds.log /var/log/jenkins/reproducible-race-conditions.log /var/log/jenkins/reproducible-diskspace-issues.log ; do
+	for PROBLEM in /var/log/jenkins/stale_builds.log /var/log/jenkins/reproducible-race-conditions.log /var/log/jenkins/reproducible-diskspace-issues.log /var/log/jenkins/reproducible-remote-error.log; do
 		if [ -s $PROBLEM ] ; then
 			TMPFILE=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXXXX)
 			mv $PROBLEM $TMPFILE
-- 
cgit v1.2.3-54-g00ecf