node wrapper: check if a node is up, else sleep 15m and abort the job

1 files changed, 24 insertions, 7 deletions

diff --git a/bin/jenkins_master_wrapper.sh b/bin/jenkins_master_wrapper.sh
index 77c90026..12dcb142 100755
--- a/bin/jenkins_master_wrapper.sh
+++ b/bin/jenkins_master_wrapper.sh
@@ -2,29 +2,46 @@
 
 set -u
 set -e
+PARAMS="$JOB_NAME"
 
 # these nodes also need to be listed in bin/reproducible_common.sh where they define $BUILD_NODES
 case "$NODE_NAME" in
   bpi0-armhf-rb.debian.net)
-    exec ssh -p 2222 $NODE_NAME "$JOB_NAME"
+    PORT=2222
     ;;
   hb0-armhf-rb.debian.net)
-    exec ssh -p 2224 $NODE_NAME "$JOB_NAME"
+    PORT=2224
     ;;
   wbq0-armhf-rb.debian.net)
-    exec ssh -p 2225 $NODE_NAME "$JOB_NAME"
+    PORT=2225
     ;;
   cbxi4pro0-armhf-rb.debian.net)
-    exec ssh -p 2226 $NODE_NAME "$JOB_NAME"
+    PORT=2226
     ;;
   profitbricks-build?-amd64.debian.net)
+    PORT=22
     if [[ "$JOB_NAME" =~ rebootstrap_.* ]] ; then
-	    exec ssh $NODE_NAME "$JOB_NAME $@"
-    else
-	    exec ssh $NODE_NAME "$JOB_NAME"
+	   PARAMS="$PARAMS $@"
     fi
     ;;
   *)
     echo >&2 "Unknown node $NODE_NAME."
     exit 1
 esac
+
+#
+# main
+#
+set +e
+ssh -p $PORT $NODE_NAME /bin/true
+RESULT=$?
+# abort job if host is down
+if [ $RESULT -ne 0 ] then
+	echo "$(date -u) - $NODE_NAME seems to be down, sleeping 15min before aborting this job."
+	sleep 15m
+	/srv/jenkins/bin/abort.sh
+fi
+set -e
+# finally
+exec ssh -p $PORT $NODE_NAME "$PARAMS"
+