summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHolger Levsen <holger@layer-acht.org>2015-12-09 12:35:48 +0100
committerHolger Levsen <holger@layer-acht.org>2015-12-09 12:35:48 +0100
commit9f20c74c7cc4434b298c40f2786dbd9d54981bd8 (patch)
treebff7f1a319257143ada444aea9b5cd77f3f532b4
parent3c2f65d77d08ab47ecc6ff75eecc729e64575a0b (diff)
downloadjenkins.debian.net-9f20c74c7cc4434b298c40f2786dbd9d54981bd8.tar.xz
reproducible: add new script, reproducible_cleanup_nodes.sh, to kill lonely build processes on all nodes
-rwxr-xr-xbin/jenkins_master_wrapper.sh5
-rwxr-xr-xbin/jenkins_node_wrapper.sh2
-rwxr-xr-xbin/reproducible_cleanup_nodes.sh43
-rwxr-xr-xbin/reproducible_slay.sh18
-rw-r--r--hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins5
-rw-r--r--hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins5
-rw-r--r--hosts/hb0-armhf-rb/etc/sudoers.d/jenkins5
-rw-r--r--hosts/jenkins/etc/sudoers.d/jenkins5
-rw-r--r--hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins5
-rw-r--r--hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins5
-rw-r--r--hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins5
-rw-r--r--hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins5
-rw-r--r--hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins5
-rw-r--r--hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins5
-rw-r--r--hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins5
-rw-r--r--hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins5
16 files changed, 116 insertions, 12 deletions
diff --git a/bin/jenkins_master_wrapper.sh b/bin/jenkins_master_wrapper.sh
index 79c78f33..ed7902ba 100755
--- a/bin/jenkins_master_wrapper.sh
+++ b/bin/jenkins_master_wrapper.sh
@@ -46,6 +46,11 @@ case "$NODE_NAME" in
exit 1
esac
+# pseudo job used to cleanup nodes
+if [ "$JOB_NAME" = "cleanup_nodes" ] ; then
+ PARAMS="$PARAMS $@"
+fi
+
#
# main
#
diff --git a/bin/jenkins_node_wrapper.sh b/bin/jenkins_node_wrapper.sh
index c830377f..4d41a256 100755
--- a/bin/jenkins_node_wrapper.sh
+++ b/bin/jenkins_node_wrapper.sh
@@ -60,6 +60,8 @@ allowed_cmds=()
if [[ "$*" =~ /bin/true ]] ; then
exec /bin/true ; croak "Exec failed";
+elif [[ "$*" = "/srv/jenkins/bin/reproducible_slay.sh" ]] ; then
+ exec /srv/jenkins/bin/reproducible_slay.sh ; croak "Exec failed";
elif [[ "$*" =~ /bin/nc\ localhost\ 4949 ]] ; then
exec /bin/nc localhost 4949 ; croak "Exec failed";
elif [[ "$*" =~ rebootstrap_.* ]] ; then
diff --git a/bin/reproducible_cleanup_nodes.sh b/bin/reproducible_cleanup_nodes.sh
new file mode 100755
index 00000000..c6e0473a
--- /dev/null
+++ b/bin/reproducible_cleanup_nodes.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Copyright © 2015 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv=2
+
+DEBUG=false
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code defining db access
+. /srv/jenkins/bin/reproducible_common.sh
+
+# only run this on the main node
+if [ "$HOSTNAME" != "jenkins" ] ; then
+ echo "this script must only be run on the main node, aborting."
+ exit 1
+fi
+
+# deny running this if jenkins is still running
+RESULT=$(ps fax|grep '/usr/share/jenkins/jenkins.war'|grep -v grep||true)
+if [ -n "$RESULT" ] ; then
+ echo "jenkins is still running, aborting."
+ exit 1
+else
+ echo "jenkins is not running, ok, let's go."
+
+# simple confirmation needed
+echo
+echo "Warning: running this will kill all processes by the 1111, 2222 and jenkins"
+echo " users. Press return if you want this, else better press CTRL-C now."
+echo
+read
+
+export JOB_NAME="cleanup_nodes"
+for NODE in $BUILD_NODES ; do
+ # call jenkins_master_wrapper.sh so we only need to track different ssh ports in one place
+ # jenkins_master_wrapper.sh needs NODE_NAME and JOB_NAME
+ export NODE_NAME=$NODE
+ echo "$(date -u) - Killing build processes on $NODE now:"
+ /srv/jenkins/bin/jenkins_master_wrapper.sh /srv/jenkins/bin/reproducible_slay.sh
+ echo "$(date -u) - done killing processes on $NODE."
+done
+
diff --git a/bin/reproducible_slay.sh b/bin/reproducible_slay.sh
new file mode 100755
index 00000000..922bea27
--- /dev/null
+++ b/bin/reproducible_slay.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Copyright 2015 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv=2
+
+set -e
+
+# usually called by /srv/jenkins/bin/reproducible_cleanup_nodes.sh
+# this script just kills everyone…
+sudo slay -clean 1111
+sudo slay -clean 2222
+sleep 2
+ps fax
+# only slay jenkins on the build nodes…
+if [ "$HOSTNAME" != "jenkins" ] ; then
+ sudo slay -clean jenkins
+fi
+
diff --git a/hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins b/hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL= \
/bin/rm /var/cache/pbuilder/result/*, \
/usr/bin/dcmd rm *.changes, \
/usr/bin/dcmd rm *.dsc, \
- /usr/bin/apt-get update
+ /usr/bin/apt-get update, \
+ /usr/sbin/slay -clean 1111, \
+ /usr/sbin/slay -clean 2222, \
+ /usr/sbin/slay -clean jenkins
# keep these environment variables
Defaults env_keep += "http_proxy", env_reset
diff --git a/hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins b/hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL= \
/bin/rm /var/cache/pbuilder/result/*, \
/usr/bin/dcmd rm *.changes, \
/usr/bin/dcmd rm *.dsc, \
- /usr/bin/apt-get update
+ /usr/bin/apt-get update, \
+ /usr/sbin/slay -clean 1111, \
+ /usr/sbin/slay -clean 2222, \
+ /usr/sbin/slay -clean jenkins
# keep these environment variables
Defaults env_keep += "http_proxy", env_reset
diff --git a/hosts/hb0-armhf-rb/etc/sudoers.d/jenkins b/hosts/hb0-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/hb0-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/hb0-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL= \
/bin/rm /var/cache/pbuilder/result/*, \
/usr/bin/dcmd rm *.changes, \
/usr/bin/dcmd rm *.dsc, \
- /usr/bin/apt-get update
+ /usr/bin/apt-get update, \
+ /usr/sbin/slay -clean 1111, \
+ /usr/sbin/slay -clean 2222, \
+ /usr/sbin/slay -clean jenkins
# keep these environment variables
Defaults env_keep += "http_proxy", env_reset
diff --git a/hosts/jenkins/etc/sudoers.d/jenkins b/hosts/jenkins/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/jenkins/etc/sudoers.d/jenkins
+++ b/hosts/jenkins/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL= \
/bin/rm /var/cache/pbuilder/result/*, \
/usr/bin/dcmd rm *.changes, \
/usr/bin/dcmd rm *.dsc, \
- /usr/bin/apt-get update
+ /usr/bin/apt-get update, \
+ /usr/sbin/slay -clean 1111, \
+ /usr/sbin/slay -clean 2222, \
+ /usr/sbin/slay -clean jenkins
# keep these environment variables
Defaults env_keep += "http_proxy", env_reset
diff --git a/hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins b/hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL= \
/bin/rm /var/cache/pbuilder/result/*, \
/usr/bin/dcmd rm *.changes, \
/usr/bin/dcmd rm *.dsc, \
- /usr/bin/apt-get update
+ /usr/bin/apt-get update, \
+ /usr/sbin/slay -clean 1111, \
+ /usr/sbin/slay -clean 2222, \
+ /usr/sbin/slay -clean jenkins
# keep these environment variables
Defaults env_keep += "http_proxy", env_reset
diff --git a/hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins
+++ b/hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL= \
/bin/rm /var/cache/pbuilder/result/*, \
/usr/bin/dcmd rm *.changes, \
/usr/bin/dcmd rm *.dsc, \
- /usr/bin/apt-get update
+ /usr/bin/apt-get update, \
+ /usr/sbin/slay -clean 1111, \
+ /usr/sbin/slay -clean 2222, \
+ /usr/sbin/slay -clean jenkins
# keep these environment variables
Defaults env_keep += "http_proxy", env_reset
diff --git a/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins
+++ b/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL= \
/bin/rm /var/cache/pbuilder/result/*, \
/usr/bin/dcmd rm *.changes, \
/usr/bin/dcmd rm *.dsc, \
- /usr/bin/apt-get update
+ /usr/bin/apt-get update, \
+ /usr/sbin/slay -clean 1111, \
+ /usr/sbin/slay -clean 2222, \
+ /usr/sbin/slay -clean jenkins
# keep these environment variables
Defaults env_keep += "http_proxy", env_reset
diff --git a/hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins
+++ b/hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL= \
/bin/rm /var/cache/pbuilder/result/*, \
/usr/bin/dcmd rm *.changes, \
/usr/bin/dcmd rm *.dsc, \
- /usr/bin/apt-get update
+ /usr/bin/apt-get update, \
+ /usr/sbin/slay -clean 1111, \
+ /usr/sbin/slay -clean 2222, \
+ /usr/sbin/slay -clean jenkins
# keep these environment variables
Defaults env_keep += "http_proxy", env_reset
diff --git a/hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins
+++ b/hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL= \
/bin/rm /var/cache/pbuilder/result/*, \
/usr/bin/dcmd rm *.changes, \
/usr/bin/dcmd rm *.dsc, \
- /usr/bin/apt-get update
+ /usr/bin/apt-get update, \
+ /usr/sbin/slay -clean 1111, \
+ /usr/sbin/slay -clean 2222, \
+ /usr/sbin/slay -clean jenkins
# keep these environment variables
Defaults env_keep += "http_proxy", env_reset
diff --git a/hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins b/hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL= \
/bin/rm /var/cache/pbuilder/result/*, \
/usr/bin/dcmd rm *.changes, \
/usr/bin/dcmd rm *.dsc, \
- /usr/bin/apt-get update
+ /usr/bin/apt-get update, \
+ /usr/sbin/slay -clean 1111, \
+ /usr/sbin/slay -clean 2222, \
+ /usr/sbin/slay -clean jenkins
# keep these environment variables
Defaults env_keep += "http_proxy", env_reset
diff --git a/hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins b/hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL= \
/bin/rm /var/cache/pbuilder/result/*, \
/usr/bin/dcmd rm *.changes, \
/usr/bin/dcmd rm *.dsc, \
- /usr/bin/apt-get update
+ /usr/bin/apt-get update, \
+ /usr/sbin/slay -clean 1111, \
+ /usr/sbin/slay -clean 2222, \
+ /usr/sbin/slay -clean jenkins
# keep these environment variables
Defaults env_keep += "http_proxy", env_reset
diff --git a/hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins b/hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins
index 22751009..911015a5 100644
--- a/hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins
+++ b/hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins
@@ -41,7 +41,10 @@ jenkins ALL= \
/bin/rm /var/cache/pbuilder/result/*, \
/usr/bin/dcmd rm *.changes, \
/usr/bin/dcmd rm *.dsc, \
- /usr/bin/apt-get update
+ /usr/bin/apt-get update, \
+ /usr/sbin/slay -clean 1111, \
+ /usr/sbin/slay -clean 2222, \
+ /usr/sbin/slay -clean jenkins
# keep these environment variables
Defaults env_keep += "http_proxy", env_reset