reproducible Debian: add new node health check jobs, splitt off from maintenance jobs

Signed-off-by: Holger Levsen <holger@layer-acht.org>

5 files changed, 146 insertions, 57 deletions

diff --git a/bin/jenkins_node_wrapper.sh b/bin/jenkins_node_wrapper.sh
index 52cdecc9..27006a1e 100755
--- a/bin/jenkins_node_wrapper.sh
+++ b/bin/jenkins_node_wrapper.sh
@@ -109,6 +109,8 @@ elif [[ "$*" =~ ^reproducible_setup_pbuilder_experimental_.*_.* ]] ; then
 	exec /srv/jenkins/bin/reproducible_setup_pbuilder.sh experimental ; croak "Exec failed";
 elif [[ "$*" =~ ^reproducible_maintenance_.*_.* ]] ; then
 	exec /srv/jenkins/bin/reproducible_maintenance.sh ; croak "Exec failed";
+elif [[ "$*" =~ ^reproducible_node_health_check_.*_.* ]] ; then
+	exec /srv/jenkins/bin/reproducible_node_health_check.sh ; croak "Exec failed";
 elif [[ "$*" =~ ^reproducible_setup_schroot_unstable_diffoscope_.*_.* ]] ; then
 	exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable-diffoscope unstable diffoscope locales-all ; croak "Exec failed";
 elif [[ "$*" =~ ^reproducible_setup_schroot_unstable_.*_.* ]] ; then
diff --git a/bin/reproducible_html_nodes_info.sh b/bin/reproducible_html_nodes_info.sh
index eaf3fba3..fe0d19bf 100755
--- a/bin/reproducible_html_nodes_info.sh
+++ b/bin/reproducible_html_nodes_info.sh
@@ -30,7 +30,7 @@ write_page "<p style=\"clear:both;\">"
 for ARCH in ${ARCHS} ; do
 	write_page "<h3>$ARCH nodes</h3>"
 	write_page "<table>"
-	write_page "<tr><th>Name</th><th>maintenance</th><th>worker.log links</th>"
+	write_page "<tr><th>Name</th><th>health check</th><th>maintenance</th><th>worker.log links</th>"
 		for SUITE in ${SUITES} ; do
 			if [ "$SUITE" = "experimental" ] ; then
 				continue
@@ -58,6 +58,9 @@ for ARCH in ${ARCHS} ; do
 			esac
 		fi
 		write_page "<tr><td>$JENKINS_NODENAME</td>"
+		URL="https://jenkins.debian.net/view/reproducible/view/Node_node_health_check/job/reproducible_node_health_check_${ARCH}_${JENKINS_NODENAME}"
+		BADGE="$URL/badge/icon"
+		write_page "<td><a href='$URL'><img src='$BADGE' /></a></td>"
 		URL="https://jenkins.debian.net/view/reproducible/view/Node_maintenance/job/reproducible_maintenance_${ARCH}_${JENKINS_NODENAME}"
 		BADGE="$URL/badge/icon"
 		write_page "<td><a href='$URL'><img src='$BADGE' /></a></td>"
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index ea55adef..e71f3dc4 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -52,62 +52,6 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then
 	fi
 fi
 
-show_fstab_and_mounts() {
-	echo "################################"
-	echo "/dev/shm and /run/shm on $HOSTNAME"
-	echo "################################"
-	ls -lartd /run/shm /dev/shm/
-	echo "################################"
-	echo "/etc/fstab on $HOSTNAME"
-	echo "################################"
-	cat /etc/fstab
-	echo "################################"
-	echo "mount output on $HOSTNAME"
-	echo "################################"
-	mount
-	echo "################################"
-}
-
-# for Debian, first run some checks…
-set +e
-# check for working proxy
-echo "$(date -u) - testing whether the proxy works..."
-curl http://www.debian.org > /dev/null
-if [ $? -ne 0 ] ; then
-	irc_message debian-reproducible "Proxy is down for $HOSTNAME, please tell the jenkins admins to fix this. (sudo service squid3 restart)"
-	exit 1
-fi
-# check for /dev/shm being mounted properly
-echo "$(date -u) - testing whether /dev/shm is mounted correctly..."
-mount | egrep -q "^tmpfs on /dev/shm"
-if [ $? -ne 0 ] ; then
-	echo "Warning: /dev/shm is not mounted correctly on $HOSTNAME, it should be a tmpfs, please tell the jenkins admins to fix this."
-	show_fstab_and_mounts
-fi
-test "$(stat -c %a -L /dev/shm)" = 1777
-if [ $? -ne 0 ] ; then
-	echo "Warning: /dev/shm is not mounted correctly on $HOSTNAME, it should be mounted with 1777 permissions, please tell the jenkins admins to fix this."
-	show_fstab_and_mounts
-fi
-# check for /run/shm being a link to /dev/shm
-echo "$(date -u) - testing whether /run/shm is a link..."
-if ! test -L /run/shm ; then
-	echo "Warning: /run/shm is not a link on $HOSTNAME, please tell the jenkins admins to fix this."
-	show_fstab_and_mounts
-elif [ "$(readlink /run/shm)" != "/dev/shm" ] ; then
-	echo "Warning: /run/shm is a link, but not pointing to /dev/shm on $HOSTNAME, please tell the jenkins admins to fix this."
-	show_fstab_and_mounts
-fi
-# check for correct MTU
-echo "$(date -u) - testing whether the network interfaces MTU is 1500..."
-if [ "$(ip link | sed -n '/LOOPBACK\|NOARP/!s/.* mtu \([0-9]*\) .*/\1/p' | sort -u)" != "1500" ] ; then
-	ip link
-	echo "$(date -u) - network interfaces MTU != 1500 - this is wrong.  => please \`sudo ifconfig eth0 mtu 1500\`"
-	# should probably turn this into a warning if this becomes to annoying
-	irc_message debian-reproducible "$HOSTNAME has wrong MTU, please tell the jenkins admins to fix this.  (sudo ifconfig eth0 mtu 1500)"
-	exit 1
-fi
-
 echo "$(date -u) - updating the schroots and pbuilder now..."
 # use host architecture (only)
 ARCH=$(dpkg --print-architecture)
diff --git a/bin/reproducible_node_health_check.sh b/bin/reproducible_node_health_check.sh
new file mode 100755
index 00000000..3e9665e8
--- /dev/null
+++ b/bin/reproducible_node_health_check.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+# Copyright 2014-2017 Holger Levsen <holger@layer-acht.org>
+#         © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# released under the GPLv=2
+
+DEBUG=false
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code defining db access
+. /srv/jenkins/bin/reproducible_common.sh
+
+DIRTY=false
+REP_RESULTS=/srv/reproducible-results
+
+show_fstab_and_mounts() {
+	echo "################################"
+	echo "/dev/shm and /run/shm on $HOSTNAME"
+	echo "################################"
+	ls -lartd /run/shm /dev/shm/
+	echo "################################"
+	echo "/etc/fstab on $HOSTNAME"
+	echo "################################"
+	cat /etc/fstab
+	echo "################################"
+	echo "mount output on $HOSTNAME"
+	echo "################################"
+	mount
+	echo "################################"
+	DIRTY=true
+}
+
+# for Debian, first run some checks…
+set +e
+# check for working proxy
+echo "$(date -u) - testing whether the proxy works..."
+curl http://www.debian.org > /dev/null
+if [ $? -ne 0 ] ; then
+	irc_message debian-reproducible "Proxy is down for $HOSTNAME, please tell the jenkins admins to fix this. (sudo service squid3 restart)"
+	exit 1
+fi
+# check for /dev/shm being mounted properly
+echo "$(date -u) - testing whether /dev/shm is mounted correctly..."
+mount | egrep -q "^tmpfs on /dev/shm"
+if [ $? -ne 0 ] ; then
+	echo "Warning: /dev/shm is not mounted correctly on $HOSTNAME, it should be a tmpfs, please tell the jenkins admins to fix this."
+	show_fstab_and_mounts
+fi
+test "$(stat -c %a -L /dev/shm)" = 1777
+if [ $? -ne 0 ] ; then
+	echo "Warning: /dev/shm is not mounted correctly on $HOSTNAME, it should be mounted with 1777 permissions, please tell the jenkins admins to fix this."
+	show_fstab_and_mounts
+fi
+# check for /run/shm being a link to /dev/shm
+echo "$(date -u) - testing whether /run/shm is a link..."
+if ! test -L /run/shm ; then
+	echo "Warning: /run/shm is not a link on $HOSTNAME, please tell the jenkins admins to fix this."
+	show_fstab_and_mounts
+elif [ "$(readlink /run/shm)" != "/dev/shm" ] ; then
+	echo "Warning: /run/shm is a link, but not pointing to /dev/shm on $HOSTNAME, please tell the jenkins admins to fix this."
+	show_fstab_and_mounts
+fi
+# check for correct MTU
+echo "$(date -u) - testing whether the network interfaces MTU is 1500..."
+if [ "$(ip link | sed -n '/LOOPBACK\|NOARP/!s/.* mtu \([0-9]*\) .*/\1/p' | sort -u)" != "1500" ] ; then
+	ip link
+	echo "$(date -u) - network interfaces MTU != 1500 - this is wrong.  => please \`sudo ifconfig eth0 mtu 1500\`"
+	# should probably turn this into a warning if this becomes to annoying
+	irc_message debian-reproducible "$HOSTNAME has wrong MTU, please tell the jenkins admins to fix this.  (sudo ifconfig eth0 mtu 1500)"
+	exit 1
+fi
+if ! $DIRTY ; then
+	echo "$(date -u ) - Everything seems to be fine."
+	echo
+fi
+
+echo "$(date -u) - the end."
+
+
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index cc51a268..38330b21 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -258,6 +258,14 @@
             my_hname:
                 - 'jenkins':        { my_arch: 'amd64' }
             my_node: ''
+            my_task:
+                - 'node_health_check':
+                    my_description: 'Do some health checks.'
+                    my_timed: '3 * * * *'
+            my_shell: '/srv/jenkins/bin/reproducible_node_health_check.sh'
+            my_hname:
+                - 'jenkins':        { my_arch: 'amd64' }
+            my_node: ''
 
         - '{name}_{my_task}_{my_arch}_{my_hname}':
             my_task:
@@ -311,6 +319,58 @@
                 - 'codethink15':    { my_arch: 'arm64' }
                 - 'codethink16':    { my_arch: 'arm64' }
             my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
+        - '{name}_{my_task}_{my_arch}_{my_hname}':
+            my_task:
+                - 'node_health_check':
+                    my_description: 'Do some health checks'
+                    my_timed: '3 * * * *'
+            my_hname:
+                - 'bbx15':          { my_arch: 'armhf' }
+                - 'bpi0':           { my_arch: 'armhf' }
+                - 'cbxi4a':         { my_arch: 'armhf' }
+                - 'cbxi4b':         { my_arch: 'armhf' }
+                - 'cbxi4pro0':      { my_arch: 'armhf' }
+                - 'cb3a':           { my_arch: 'armhf' }
+                - 'ff2a':           { my_arch: 'armhf' }
+                - 'ff2b':           { my_arch: 'armhf' }
+                - 'ff4a':           { my_arch: 'armhf' }
+                - 'ff64a':          { my_arch: 'armhf' }
+                - 'hb0':            { my_arch: 'armhf' }
+                - 'jtk1a':          { my_arch: 'armhf' }
+                - 'jtx1a':          { my_arch: 'armhf' }
+                - 'odc2a':          { my_arch: 'armhf' }
+                - 'odxu4':          { my_arch: 'armhf' }
+                - 'odxu4b':         { my_arch: 'armhf' }
+                - 'odxu4c':         { my_arch: 'armhf' }
+                - 'odu3a':          { my_arch: 'armhf' }
+                - 'opi2a':          { my_arch: 'armhf' }
+                - 'opi2b':          { my_arch: 'armhf' }
+                - 'opi2c':          { my_arch: 'armhf' }
+                - 'rpi2b':          { my_arch: 'armhf' }
+                - 'rpi2c':          { my_arch: 'armhf' }
+                - 'p64b':           { my_arch: 'armhf' }
+                - 'p64c':           { my_arch: 'armhf' }
+                - 'wbd0':           { my_arch: 'armhf' }
+                - 'wbq0':           { my_arch: 'armhf' }
+                - 'profitbricks1':  { my_arch: 'amd64' }
+                - 'profitbricks2':  { my_arch: 'i386'  }
+                - 'profitbricks3':  { my_arch: 'amd64' }
+                - 'profitbricks4':  { my_arch: 'amd64' }
+                - 'profitbricks5':  { my_arch: 'amd64' }
+                - 'profitbricks6':  { my_arch: 'i386'  }
+                - 'profitbricks11': { my_arch: 'amd64' }
+                - 'profitbricks12': { my_arch: 'i386'  }
+                - 'profitbricks15': { my_arch: 'amd64' }
+                - 'profitbricks16': { my_arch: 'i386'  }
+                - 'codethink9':     { my_arch: 'arm64' }
+                - 'codethink10':    { my_arch: 'arm64' }
+                - 'codethink11':    { my_arch: 'arm64' }
+                - 'codethink12':    { my_arch: 'arm64' }
+                - 'codethink13':    { my_arch: 'arm64' }
+                - 'codethink14':    { my_arch: 'arm64' }
+                - 'codethink15':    { my_arch: 'arm64' }
+                - 'codethink16':    { my_arch: 'arm64' }
+            my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh'
 
         - '{name}_{my_task}_{my_arch}_{my_hname}':
             my_task: