reproducible: add new script, reproducible_cleanup_nodes.sh, to kill lonely build processes on all nodes

16 files changed, 116 insertions, 12 deletions

diff --git a/bin/jenkins_master_wrapper.sh b/bin/jenkins_master_wrapper.sh
index 79c78f33..ed7902ba 100755
--- a/bin/jenkins_master_wrapper.sh
+++ b/bin/jenkins_master_wrapper.sh
@@ -46,6 +46,11 @@ case "$NODE_NAME" in
     exit 1
 esac
 
+# pseudo job used to cleanup nodes
+if [ "$JOB_NAME" = "cleanup_nodes" ] ; then
+	   PARAMS="$PARAMS $@"
+fi
+
 #
 # main
 #
diff --git a/bin/jenkins_node_wrapper.sh b/bin/jenkins_node_wrapper.sh
index c830377f..4d41a256 100755
--- a/bin/jenkins_node_wrapper.sh
+++ b/bin/jenkins_node_wrapper.sh
@@ -60,6 +60,8 @@ allowed_cmds=()
 
 if [[ "$*" =~ /bin/true ]] ; then
 	exec /bin/true ; croak "Exec failed";
+elif [[ "$*" = "/srv/jenkins/bin/reproducible_slay.sh" ]] ; then
+	exec /srv/jenkins/bin/reproducible_slay.sh ; croak "Exec failed";
 elif [[ "$*" =~ /bin/nc\ localhost\ 4949 ]] ; then
 	exec /bin/nc localhost 4949 ; croak "Exec failed";
 elif [[ "$*" =~ rebootstrap_.* ]] ; then
diff --git a/bin/reproducible_cleanup_nodes.sh b/bin/reproducible_cleanup_nodes.sh
new file mode 100755
index 00000000..c6e0473a
--- /dev/null
+++ b/bin/reproducible_cleanup_nodes.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Copyright © 2015 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv=2
+
+DEBUG=false
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code defining db access
+. /srv/jenkins/bin/reproducible_common.sh
+
+# only run this on the main node
+if [ "$HOSTNAME" != "jenkins" ] ; then
+	echo "this script must only be run on the main node, aborting."
+	exit 1
+fi
+
+# deny running this if jenkins is still running
+RESULT=$(ps fax|grep '/usr/share/jenkins/jenkins.war'|grep -v grep||true)
+if [ -n "$RESULT" ] ; then
+	echo "jenkins is still running, aborting."
+	exit 1
+else
+	echo "jenkins is not running, ok, let's go."
+
+# simple confirmation needed
+echo
+echo "Warning: running this will kill all processes by the 1111, 2222 and jenkins"
+echo "         users. Press return if you want this, else better press CTRL-C now."
+echo
+read
+
+export JOB_NAME="cleanup_nodes"
+for NODE in $BUILD_NODES ; do
+	# call jenkins_master_wrapper.sh so we only need to track different ssh ports in one place
+	# jenkins_master_wrapper.sh needs NODE_NAME and JOB_NAME
+	export NODE_NAME=$NODE
+	echo "$(date -u) - Killing build processes on $NODE now:"
+	/srv/jenkins/bin/jenkins_master_wrapper.sh /srv/jenkins/bin/reproducible_slay.sh
+	echo "$(date -u) - done killing processes on $NODE."
+done
+
diff --git a/bin/reproducible_slay.sh b/bin/reproducible_slay.sh
new file mode 100755
index 00000000..922bea27
--- /dev/null
+++ b/bin/reproducible_slay.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Copyright 2015 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv=2
+
+set -e
+
+# usually called by /srv/jenkins/bin/reproducible_cleanup_nodes.sh
+# this script just kills everyone…
+sudo slay -clean 1111
+sudo slay -clean 2222
+sleep 2
+ps fax
+# only slay jenkins on the build nodes…
+if [ "$HOSTNAME" != "jenkins" ] ; then
+	sudo slay -clean jenkins
+fi
+
diff --git a/hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins b/hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL=  \
 	/bin/rm /var/cache/pbuilder/result/*, \
 	/usr/bin/dcmd rm *.changes, \
 	/usr/bin/dcmd rm *.dsc, \
-	/usr/bin/apt-get update
+	/usr/bin/apt-get update, \
+	/usr/sbin/slay -clean 1111, \
+	/usr/sbin/slay -clean 2222, \
+	/usr/sbin/slay -clean jenkins
 
 # keep these environment variables
 Defaults        env_keep += "http_proxy", env_reset
diff --git a/hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins b/hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL=  \
 	/bin/rm /var/cache/pbuilder/result/*, \
 	/usr/bin/dcmd rm *.changes, \
 	/usr/bin/dcmd rm *.dsc, \
-	/usr/bin/apt-get update
+	/usr/bin/apt-get update, \
+	/usr/sbin/slay -clean 1111, \
+	/usr/sbin/slay -clean 2222, \
+	/usr/sbin/slay -clean jenkins
 
 # keep these environment variables
 Defaults        env_keep += "http_proxy", env_reset
diff --git a/hosts/hb0-armhf-rb/etc/sudoers.d/jenkins b/hosts/hb0-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/hb0-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/hb0-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL=  \
 	/bin/rm /var/cache/pbuilder/result/*, \
 	/usr/bin/dcmd rm *.changes, \
 	/usr/bin/dcmd rm *.dsc, \
-	/usr/bin/apt-get update
+	/usr/bin/apt-get update, \
+	/usr/sbin/slay -clean 1111, \
+	/usr/sbin/slay -clean 2222, \
+	/usr/sbin/slay -clean jenkins
 
 # keep these environment variables
 Defaults        env_keep += "http_proxy", env_reset
diff --git a/hosts/jenkins/etc/sudoers.d/jenkins b/hosts/jenkins/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/jenkins/etc/sudoers.d/jenkins
+++ b/hosts/jenkins/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL=  \
 	/bin/rm /var/cache/pbuilder/result/*, \
 	/usr/bin/dcmd rm *.changes, \
 	/usr/bin/dcmd rm *.dsc, \
-	/usr/bin/apt-get update
+	/usr/bin/apt-get update, \
+	/usr/sbin/slay -clean 1111, \
+	/usr/sbin/slay -clean 2222, \
+	/usr/sbin/slay -clean jenkins
 
 # keep these environment variables
 Defaults        env_keep += "http_proxy", env_reset
diff --git a/hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins b/hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL=  \
 	/bin/rm /var/cache/pbuilder/result/*, \
 	/usr/bin/dcmd rm *.changes, \
 	/usr/bin/dcmd rm *.dsc, \
-	/usr/bin/apt-get update
+	/usr/bin/apt-get update, \
+	/usr/sbin/slay -clean 1111, \
+	/usr/sbin/slay -clean 2222, \
+	/usr/sbin/slay -clean jenkins
 
 # keep these environment variables
 Defaults        env_keep += "http_proxy", env_reset
diff --git a/hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins
+++ b/hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL=  \
 	/bin/rm /var/cache/pbuilder/result/*, \
 	/usr/bin/dcmd rm *.changes, \
 	/usr/bin/dcmd rm *.dsc, \
-	/usr/bin/apt-get update
+	/usr/bin/apt-get update, \
+	/usr/sbin/slay -clean 1111, \
+	/usr/sbin/slay -clean 2222, \
+	/usr/sbin/slay -clean jenkins
 
 # keep these environment variables
 Defaults        env_keep += "http_proxy", env_reset
diff --git a/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins
+++ b/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL=  \
 	/bin/rm /var/cache/pbuilder/result/*, \
 	/usr/bin/dcmd rm *.changes, \
 	/usr/bin/dcmd rm *.dsc, \
-	/usr/bin/apt-get update
+	/usr/bin/apt-get update, \
+	/usr/sbin/slay -clean 1111, \
+	/usr/sbin/slay -clean 2222, \
+	/usr/sbin/slay -clean jenkins
 
 # keep these environment variables
 Defaults        env_keep += "http_proxy", env_reset
diff --git a/hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins
+++ b/hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL=  \
 	/bin/rm /var/cache/pbuilder/result/*, \
 	/usr/bin/dcmd rm *.changes, \
 	/usr/bin/dcmd rm *.dsc, \
-	/usr/bin/apt-get update
+	/usr/bin/apt-get update, \
+	/usr/sbin/slay -clean 1111, \
+	/usr/sbin/slay -clean 2222, \
+	/usr/sbin/slay -clean jenkins
 
 # keep these environment variables
 Defaults        env_keep += "http_proxy", env_reset
diff --git a/hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins
+++ b/hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL=  \
 	/bin/rm /var/cache/pbuilder/result/*, \
 	/usr/bin/dcmd rm *.changes, \
 	/usr/bin/dcmd rm *.dsc, \
-	/usr/bin/apt-get update
+	/usr/bin/apt-get update, \
+	/usr/sbin/slay -clean 1111, \
+	/usr/sbin/slay -clean 2222, \
+	/usr/sbin/slay -clean jenkins
 
 # keep these environment variables
 Defaults        env_keep += "http_proxy", env_reset
diff --git a/hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins b/hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL=  \
 	/bin/rm /var/cache/pbuilder/result/*, \
 	/usr/bin/dcmd rm *.changes, \
 	/usr/bin/dcmd rm *.dsc, \
-	/usr/bin/apt-get update
+	/usr/bin/apt-get update, \
+	/usr/sbin/slay -clean 1111, \
+	/usr/sbin/slay -clean 2222, \
+	/usr/sbin/slay -clean jenkins
 
 # keep these environment variables
 Defaults        env_keep += "http_proxy", env_reset
diff --git a/hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins b/hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL=  \
 	/bin/rm /var/cache/pbuilder/result/*, \
 	/usr/bin/dcmd rm *.changes, \
 	/usr/bin/dcmd rm *.dsc, \
-	/usr/bin/apt-get update
+	/usr/bin/apt-get update, \
+	/usr/sbin/slay -clean 1111, \
+	/usr/sbin/slay -clean 2222, \
+	/usr/sbin/slay -clean jenkins
 
 # keep these environment variables
 Defaults        env_keep += "http_proxy", env_reset
diff --git a/hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins b/hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL=  \
 	/bin/rm /var/cache/pbuilder/result/*, \
 	/usr/bin/dcmd rm *.changes, \
 	/usr/bin/dcmd rm *.dsc, \
-	/usr/bin/apt-get update
+	/usr/bin/apt-get update, \
+	/usr/sbin/slay -clean 1111, \
+	/usr/sbin/slay -clean 2222, \
+	/usr/sbin/slay -clean jenkins
 
 # keep these environment variables
 Defaults        env_keep += "http_proxy", env_reset