node wrapper: check if a node is up, else sleep 15m and abort the job

2 files changed, 27 insertions, 8 deletions

diff --git a/bin/jenkins_master_wrapper.sh b/bin/jenkins_master_wrapper.sh
index 77c90026..12dcb142 100755
--- a/bin/jenkins_master_wrapper.sh
+++ b/bin/jenkins_master_wrapper.sh
@@ -2,29 +2,46 @@
 
 set -u
 set -e
+PARAMS="$JOB_NAME"
 
 # these nodes also need to be listed in bin/reproducible_common.sh where they define $BUILD_NODES
 case "$NODE_NAME" in
   bpi0-armhf-rb.debian.net)
-    exec ssh -p 2222 $NODE_NAME "$JOB_NAME"
+    PORT=2222
     ;;
   hb0-armhf-rb.debian.net)
-    exec ssh -p 2224 $NODE_NAME "$JOB_NAME"
+    PORT=2224
     ;;
   wbq0-armhf-rb.debian.net)
-    exec ssh -p 2225 $NODE_NAME "$JOB_NAME"
+    PORT=2225
     ;;
   cbxi4pro0-armhf-rb.debian.net)
-    exec ssh -p 2226 $NODE_NAME "$JOB_NAME"
+    PORT=2226
     ;;
   profitbricks-build?-amd64.debian.net)
+    PORT=22
     if [[ "$JOB_NAME" =~ rebootstrap_.* ]] ; then
-	    exec ssh $NODE_NAME "$JOB_NAME $@"
-    else
-	    exec ssh $NODE_NAME "$JOB_NAME"
+	   PARAMS="$PARAMS $@"
     fi
     ;;
   *)
     echo >&2 "Unknown node $NODE_NAME."
     exit 1
 esac
+
+#
+# main
+#
+set +e
+ssh -p $PORT $NODE_NAME /bin/true
+RESULT=$?
+# abort job if host is down
+if [ $RESULT -ne 0 ] then
+	echo "$(date -u) - $NODE_NAME seems to be down, sleeping 15min before aborting this job."
+	sleep 15m
+	/srv/jenkins/bin/abort.sh
+fi
+set -e
+# finally
+exec ssh -p $PORT $NODE_NAME "$PARAMS"
+
diff --git a/bin/jenkins_node_wrapper.sh b/bin/jenkins_node_wrapper.sh
index b4706697..b6ea9eb0 100755
--- a/bin/jenkins_node_wrapper.sh
+++ b/bin/jenkins_node_wrapper.sh
@@ -58,7 +58,9 @@ shift
 
 allowed_cmds=()
 
-if [[ "$*" =~ /bin/nc\ localhost\ 4949 ]] ; then
+if [[ "$*" =~ /bin/true ]] ; then
+	exec /bin/true ; croak "Exec failed";
+elif [[ "$*" =~ /bin/nc\ localhost\ 4949 ]] ; then
 	exec /bin/nc localhost 4949 ; croak "Exec failed";
 elif [[ "$*" =~ rebootstrap_.* ]] ; then
 	shift