From 9f20c74c7cc4434b298c40f2786dbd9d54981bd8 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Wed, 9 Dec 2015 12:35:48 +0100 Subject: reproducible: add new script, reproducible_cleanup_nodes.sh, to kill lonely build processes on all nodes --- bin/jenkins_master_wrapper.sh | 5 +++ bin/jenkins_node_wrapper.sh | 2 + bin/reproducible_cleanup_nodes.sh | 43 ++++++++++++++++++++++ bin/reproducible_slay.sh | 18 +++++++++ hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins | 5 ++- hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins | 5 ++- hosts/hb0-armhf-rb/etc/sudoers.d/jenkins | 5 ++- hosts/jenkins/etc/sudoers.d/jenkins | 5 ++- hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins | 5 ++- .../etc/sudoers.d/jenkins | 5 ++- .../etc/sudoers.d/jenkins | 5 ++- .../etc/sudoers.d/jenkins | 5 ++- .../etc/sudoers.d/jenkins | 5 ++- hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins | 5 ++- hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins | 5 ++- hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins | 5 ++- 16 files changed, 116 insertions(+), 12 deletions(-) create mode 100755 bin/reproducible_cleanup_nodes.sh create mode 100755 bin/reproducible_slay.sh diff --git a/bin/jenkins_master_wrapper.sh b/bin/jenkins_master_wrapper.sh index 79c78f33..ed7902ba 100755 --- a/bin/jenkins_master_wrapper.sh +++ b/bin/jenkins_master_wrapper.sh @@ -46,6 +46,11 @@ case "$NODE_NAME" in exit 1 esac +# pseudo job used to cleanup nodes +if [ "$JOB_NAME" = "cleanup_nodes" ] ; then + PARAMS="$PARAMS $@" +fi + # # main # diff --git a/bin/jenkins_node_wrapper.sh b/bin/jenkins_node_wrapper.sh index c830377f..4d41a256 100755 --- a/bin/jenkins_node_wrapper.sh +++ b/bin/jenkins_node_wrapper.sh @@ -60,6 +60,8 @@ allowed_cmds=() if [[ "$*" =~ /bin/true ]] ; then exec /bin/true ; croak "Exec failed"; +elif [[ "$*" = "/srv/jenkins/bin/reproducible_slay.sh" ]] ; then + exec /srv/jenkins/bin/reproducible_slay.sh ; croak "Exec failed"; elif [[ "$*" =~ /bin/nc\ localhost\ 4949 ]] ; then exec /bin/nc localhost 4949 ; croak "Exec failed"; elif [[ "$*" =~ rebootstrap_.* ]] ; then diff --git a/bin/reproducible_cleanup_nodes.sh b/bin/reproducible_cleanup_nodes.sh new file mode 100755 index 00000000..c6e0473a --- /dev/null +++ b/bin/reproducible_cleanup_nodes.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Copyright © 2015 Holger Levsen +# released under the GPLv=2 + +DEBUG=false +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code defining db access +. /srv/jenkins/bin/reproducible_common.sh + +# only run this on the main node +if [ "$HOSTNAME" != "jenkins" ] ; then + echo "this script must only be run on the main node, aborting." + exit 1 +fi + +# deny running this if jenkins is still running +RESULT=$(ps fax|grep '/usr/share/jenkins/jenkins.war'|grep -v grep||true) +if [ -n "$RESULT" ] ; then + echo "jenkins is still running, aborting." + exit 1 +else + echo "jenkins is not running, ok, let's go." + +# simple confirmation needed +echo +echo "Warning: running this will kill all processes by the 1111, 2222 and jenkins" +echo " users. Press return if you want this, else better press CTRL-C now." +echo +read + +export JOB_NAME="cleanup_nodes" +for NODE in $BUILD_NODES ; do + # call jenkins_master_wrapper.sh so we only need to track different ssh ports in one place + # jenkins_master_wrapper.sh needs NODE_NAME and JOB_NAME + export NODE_NAME=$NODE + echo "$(date -u) - Killing build processes on $NODE now:" + /srv/jenkins/bin/jenkins_master_wrapper.sh /srv/jenkins/bin/reproducible_slay.sh + echo "$(date -u) - done killing processes on $NODE." +done + diff --git a/bin/reproducible_slay.sh b/bin/reproducible_slay.sh new file mode 100755 index 00000000..922bea27 --- /dev/null +++ b/bin/reproducible_slay.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Copyright 2015 Holger Levsen +# released under the GPLv=2 + +set -e + +# usually called by /srv/jenkins/bin/reproducible_cleanup_nodes.sh +# this script just kills everyone… +sudo slay -clean 1111 +sudo slay -clean 2222 +sleep 2 +ps fax +# only slay jenkins on the build nodes… +if [ "$HOSTNAME" != "jenkins" ] ; then + sudo slay -clean jenkins +fi + diff --git a/hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins b/hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins index 22751009..911015a5 100644 --- a/hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins +++ b/hosts/bpi0-armhf-rb/etc/sudoers.d/jenkins @@ -41,7 +41,10 @@ jenkins ALL= \ /bin/rm /var/cache/pbuilder/result/*, \ /usr/bin/dcmd rm *.changes, \ /usr/bin/dcmd rm *.dsc, \ - /usr/bin/apt-get update + /usr/bin/apt-get update, \ + /usr/sbin/slay -clean 1111, \ + /usr/sbin/slay -clean 2222, \ + /usr/sbin/slay -clean jenkins # keep these environment variables Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins b/hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins index 22751009..911015a5 100644 --- a/hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins +++ b/hosts/cbxi4pro0-armhf-rb/etc/sudoers.d/jenkins @@ -41,7 +41,10 @@ jenkins ALL= \ /bin/rm /var/cache/pbuilder/result/*, \ /usr/bin/dcmd rm *.changes, \ /usr/bin/dcmd rm *.dsc, \ - /usr/bin/apt-get update + /usr/bin/apt-get update, \ + /usr/sbin/slay -clean 1111, \ + /usr/sbin/slay -clean 2222, \ + /usr/sbin/slay -clean jenkins # keep these environment variables Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/hb0-armhf-rb/etc/sudoers.d/jenkins b/hosts/hb0-armhf-rb/etc/sudoers.d/jenkins index 22751009..911015a5 100644 --- a/hosts/hb0-armhf-rb/etc/sudoers.d/jenkins +++ b/hosts/hb0-armhf-rb/etc/sudoers.d/jenkins @@ -41,7 +41,10 @@ jenkins ALL= \ /bin/rm /var/cache/pbuilder/result/*, \ /usr/bin/dcmd rm *.changes, \ /usr/bin/dcmd rm *.dsc, \ - /usr/bin/apt-get update + /usr/bin/apt-get update, \ + /usr/sbin/slay -clean 1111, \ + /usr/sbin/slay -clean 2222, \ + /usr/sbin/slay -clean jenkins # keep these environment variables Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/jenkins/etc/sudoers.d/jenkins b/hosts/jenkins/etc/sudoers.d/jenkins index 22751009..911015a5 100644 --- a/hosts/jenkins/etc/sudoers.d/jenkins +++ b/hosts/jenkins/etc/sudoers.d/jenkins @@ -41,7 +41,10 @@ jenkins ALL= \ /bin/rm /var/cache/pbuilder/result/*, \ /usr/bin/dcmd rm *.changes, \ /usr/bin/dcmd rm *.dsc, \ - /usr/bin/apt-get update + /usr/bin/apt-get update, \ + /usr/sbin/slay -clean 1111, \ + /usr/sbin/slay -clean 2222, \ + /usr/sbin/slay -clean jenkins # keep these environment variables Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins b/hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins index 22751009..911015a5 100644 --- a/hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins +++ b/hosts/odxu4-armhf-rb/etc/sudoers.d/jenkins @@ -41,7 +41,10 @@ jenkins ALL= \ /bin/rm /var/cache/pbuilder/result/*, \ /usr/bin/dcmd rm *.changes, \ /usr/bin/dcmd rm *.dsc, \ - /usr/bin/apt-get update + /usr/bin/apt-get update, \ + /usr/sbin/slay -clean 1111, \ + /usr/sbin/slay -clean 2222, \ + /usr/sbin/slay -clean jenkins # keep these environment variables Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins index 22751009..911015a5 100644 --- a/hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins +++ b/hosts/profitbricks-build1-amd64/etc/sudoers.d/jenkins @@ -41,7 +41,10 @@ jenkins ALL= \ /bin/rm /var/cache/pbuilder/result/*, \ /usr/bin/dcmd rm *.changes, \ /usr/bin/dcmd rm *.dsc, \ - /usr/bin/apt-get update + /usr/bin/apt-get update, \ + /usr/sbin/slay -clean 1111, \ + /usr/sbin/slay -clean 2222, \ + /usr/sbin/slay -clean jenkins # keep these environment variables Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins index 22751009..911015a5 100644 --- a/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins +++ b/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins @@ -41,7 +41,10 @@ jenkins ALL= \ /bin/rm /var/cache/pbuilder/result/*, \ /usr/bin/dcmd rm *.changes, \ /usr/bin/dcmd rm *.dsc, \ - /usr/bin/apt-get update + /usr/bin/apt-get update, \ + /usr/sbin/slay -clean 1111, \ + /usr/sbin/slay -clean 2222, \ + /usr/sbin/slay -clean jenkins # keep these environment variables Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins index 22751009..911015a5 100644 --- a/hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins +++ b/hosts/profitbricks-build5-amd64/etc/sudoers.d/jenkins @@ -41,7 +41,10 @@ jenkins ALL= \ /bin/rm /var/cache/pbuilder/result/*, \ /usr/bin/dcmd rm *.changes, \ /usr/bin/dcmd rm *.dsc, \ - /usr/bin/apt-get update + /usr/bin/apt-get update, \ + /usr/sbin/slay -clean 1111, \ + /usr/sbin/slay -clean 2222, \ + /usr/sbin/slay -clean jenkins # keep these environment variables Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins index 22751009..911015a5 100644 --- a/hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins +++ b/hosts/profitbricks-build6-amd64/etc/sudoers.d/jenkins @@ -41,7 +41,10 @@ jenkins ALL= \ /bin/rm /var/cache/pbuilder/result/*, \ /usr/bin/dcmd rm *.changes, \ /usr/bin/dcmd rm *.dsc, \ - /usr/bin/apt-get update + /usr/bin/apt-get update, \ + /usr/sbin/slay -clean 1111, \ + /usr/sbin/slay -clean 2222, \ + /usr/sbin/slay -clean jenkins # keep these environment variables Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins b/hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins index 22751009..911015a5 100644 --- a/hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins +++ b/hosts/rpi2b-armhf-rb/etc/sudoers.d/jenkins @@ -41,7 +41,10 @@ jenkins ALL= \ /bin/rm /var/cache/pbuilder/result/*, \ /usr/bin/dcmd rm *.changes, \ /usr/bin/dcmd rm *.dsc, \ - /usr/bin/apt-get update + /usr/bin/apt-get update, \ + /usr/sbin/slay -clean 1111, \ + /usr/sbin/slay -clean 2222, \ + /usr/sbin/slay -clean jenkins # keep these environment variables Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins b/hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins index 22751009..911015a5 100644 --- a/hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins +++ b/hosts/wbd0-armhf-rb/etc/sudoers.d/jenkins @@ -41,7 +41,10 @@ jenkins ALL= \ /bin/rm /var/cache/pbuilder/result/*, \ /usr/bin/dcmd rm *.changes, \ /usr/bin/dcmd rm *.dsc, \ - /usr/bin/apt-get update + /usr/bin/apt-get update, \ + /usr/sbin/slay -clean 1111, \ + /usr/sbin/slay -clean 2222, \ + /usr/sbin/slay -clean jenkins # keep these environment variables Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins b/hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins index 22751009..911015a5 100644 --- a/hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins +++ b/hosts/wbq0-armhf-rb/etc/sudoers.d/jenkins @@ -41,7 +41,10 @@ jenkins ALL= \ /bin/rm /var/cache/pbuilder/result/*, \ /usr/bin/dcmd rm *.changes, \ /usr/bin/dcmd rm *.dsc, \ - /usr/bin/apt-get update + /usr/bin/apt-get update, \ + /usr/sbin/slay -clean 1111, \ + /usr/sbin/slay -clean 2222, \ + /usr/sbin/slay -clean jenkins # keep these environment variables Defaults env_keep += "http_proxy", env_reset -- cgit v1.2.3-54-g00ecf