diff options
| -rwxr-xr-x | bin/jenkins_node_wrapper.sh | 2 | ||||
| -rwxr-xr-x | bin/reproducible_html_nodes_info.sh | 5 | ||||
| -rwxr-xr-x | bin/reproducible_maintenance.sh | 56 | ||||
| -rwxr-xr-x | bin/reproducible_node_health_check.sh | 80 | ||||
| -rw-r--r-- | job-cfg/reproducible.yaml | 60 |
5 files changed, 146 insertions, 57 deletions
diff --git a/bin/jenkins_node_wrapper.sh b/bin/jenkins_node_wrapper.sh index 52cdecc9..27006a1e 100755 --- a/bin/jenkins_node_wrapper.sh +++ b/bin/jenkins_node_wrapper.sh @@ -109,6 +109,8 @@ elif [[ "$*" =~ ^reproducible_setup_pbuilder_experimental_.*_.* ]] ; then exec /srv/jenkins/bin/reproducible_setup_pbuilder.sh experimental ; croak "Exec failed"; elif [[ "$*" =~ ^reproducible_maintenance_.*_.* ]] ; then exec /srv/jenkins/bin/reproducible_maintenance.sh ; croak "Exec failed"; +elif [[ "$*" =~ ^reproducible_node_health_check_.*_.* ]] ; then + exec /srv/jenkins/bin/reproducible_node_health_check.sh ; croak "Exec failed"; elif [[ "$*" =~ ^reproducible_setup_schroot_unstable_diffoscope_.*_.* ]] ; then exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable-diffoscope unstable diffoscope locales-all ; croak "Exec failed"; elif [[ "$*" =~ ^reproducible_setup_schroot_unstable_.*_.* ]] ; then diff --git a/bin/reproducible_html_nodes_info.sh b/bin/reproducible_html_nodes_info.sh index eaf3fba3..fe0d19bf 100755 --- a/bin/reproducible_html_nodes_info.sh +++ b/bin/reproducible_html_nodes_info.sh @@ -30,7 +30,7 @@ write_page "<p style=\"clear:both;\">" for ARCH in ${ARCHS} ; do write_page "<h3>$ARCH nodes</h3>" write_page "<table>" - write_page "<tr><th>Name</th><th>maintenance</th><th>worker.log links</th>" + write_page "<tr><th>Name</th><th>health check</th><th>maintenance</th><th>worker.log links</th>" for SUITE in ${SUITES} ; do if [ "$SUITE" = "experimental" ] ; then continue @@ -58,6 +58,9 @@ for ARCH in ${ARCHS} ; do esac fi write_page "<tr><td>$JENKINS_NODENAME</td>" + URL="https://jenkins.debian.net/view/reproducible/view/Node_node_health_check/job/reproducible_node_health_check_${ARCH}_${JENKINS_NODENAME}" + BADGE="$URL/badge/icon" + write_page "<td><a href='$URL'><img src='$BADGE' /></a></td>" URL="https://jenkins.debian.net/view/reproducible/view/Node_maintenance/job/reproducible_maintenance_${ARCH}_${JENKINS_NODENAME}" BADGE="$URL/badge/icon" write_page "<td><a href='$URL'><img src='$BADGE' /></a></td>" diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh index ea55adef..e71f3dc4 100755 --- a/bin/reproducible_maintenance.sh +++ b/bin/reproducible_maintenance.sh @@ -52,62 +52,6 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then fi fi -show_fstab_and_mounts() { - echo "################################" - echo "/dev/shm and /run/shm on $HOSTNAME" - echo "################################" - ls -lartd /run/shm /dev/shm/ - echo "################################" - echo "/etc/fstab on $HOSTNAME" - echo "################################" - cat /etc/fstab - echo "################################" - echo "mount output on $HOSTNAME" - echo "################################" - mount - echo "################################" -} - -# for Debian, first run some checks… -set +e -# check for working proxy -echo "$(date -u) - testing whether the proxy works..." -curl http://www.debian.org > /dev/null -if [ $? -ne 0 ] ; then - irc_message debian-reproducible "Proxy is down for $HOSTNAME, please tell the jenkins admins to fix this. (sudo service squid3 restart)" - exit 1 -fi -# check for /dev/shm being mounted properly -echo "$(date -u) - testing whether /dev/shm is mounted correctly..." -mount | egrep -q "^tmpfs on /dev/shm" -if [ $? -ne 0 ] ; then - echo "Warning: /dev/shm is not mounted correctly on $HOSTNAME, it should be a tmpfs, please tell the jenkins admins to fix this." - show_fstab_and_mounts -fi -test "$(stat -c %a -L /dev/shm)" = 1777 -if [ $? -ne 0 ] ; then - echo "Warning: /dev/shm is not mounted correctly on $HOSTNAME, it should be mounted with 1777 permissions, please tell the jenkins admins to fix this." - show_fstab_and_mounts -fi -# check for /run/shm being a link to /dev/shm -echo "$(date -u) - testing whether /run/shm is a link..." -if ! test -L /run/shm ; then - echo "Warning: /run/shm is not a link on $HOSTNAME, please tell the jenkins admins to fix this." - show_fstab_and_mounts -elif [ "$(readlink /run/shm)" != "/dev/shm" ] ; then - echo "Warning: /run/shm is a link, but not pointing to /dev/shm on $HOSTNAME, please tell the jenkins admins to fix this." - show_fstab_and_mounts -fi -# check for correct MTU -echo "$(date -u) - testing whether the network interfaces MTU is 1500..." -if [ "$(ip link | sed -n '/LOOPBACK\|NOARP/!s/.* mtu \([0-9]*\) .*/\1/p' | sort -u)" != "1500" ] ; then - ip link - echo "$(date -u) - network interfaces MTU != 1500 - this is wrong. => please \`sudo ifconfig eth0 mtu 1500\`" - # should probably turn this into a warning if this becomes to annoying - irc_message debian-reproducible "$HOSTNAME has wrong MTU, please tell the jenkins admins to fix this. (sudo ifconfig eth0 mtu 1500)" - exit 1 -fi - echo "$(date -u) - updating the schroots and pbuilder now..." # use host architecture (only) ARCH=$(dpkg --print-architecture) diff --git a/bin/reproducible_node_health_check.sh b/bin/reproducible_node_health_check.sh new file mode 100755 index 00000000..3e9665e8 --- /dev/null +++ b/bin/reproducible_node_health_check.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# Copyright 2014-2017 Holger Levsen <holger@layer-acht.org> +# © 2015 Mattia Rizzolo <mattia@mapreri.org> +# released under the GPLv=2 + +DEBUG=false +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code defining db access +. /srv/jenkins/bin/reproducible_common.sh + +DIRTY=false +REP_RESULTS=/srv/reproducible-results + +show_fstab_and_mounts() { + echo "################################" + echo "/dev/shm and /run/shm on $HOSTNAME" + echo "################################" + ls -lartd /run/shm /dev/shm/ + echo "################################" + echo "/etc/fstab on $HOSTNAME" + echo "################################" + cat /etc/fstab + echo "################################" + echo "mount output on $HOSTNAME" + echo "################################" + mount + echo "################################" + DIRTY=true +} + +# for Debian, first run some checks… +set +e +# check for working proxy +echo "$(date -u) - testing whether the proxy works..." +curl http://www.debian.org > /dev/null +if [ $? -ne 0 ] ; then + irc_message debian-reproducible "Proxy is down for $HOSTNAME, please tell the jenkins admins to fix this. (sudo service squid3 restart)" + exit 1 +fi +# check for /dev/shm being mounted properly +echo "$(date -u) - testing whether /dev/shm is mounted correctly..." +mount | egrep -q "^tmpfs on /dev/shm" +if [ $? -ne 0 ] ; then + echo "Warning: /dev/shm is not mounted correctly on $HOSTNAME, it should be a tmpfs, please tell the jenkins admins to fix this." + show_fstab_and_mounts +fi +test "$(stat -c %a -L /dev/shm)" = 1777 +if [ $? -ne 0 ] ; then + echo "Warning: /dev/shm is not mounted correctly on $HOSTNAME, it should be mounted with 1777 permissions, please tell the jenkins admins to fix this." + show_fstab_and_mounts +fi +# check for /run/shm being a link to /dev/shm +echo "$(date -u) - testing whether /run/shm is a link..." +if ! test -L /run/shm ; then + echo "Warning: /run/shm is not a link on $HOSTNAME, please tell the jenkins admins to fix this." + show_fstab_and_mounts +elif [ "$(readlink /run/shm)" != "/dev/shm" ] ; then + echo "Warning: /run/shm is a link, but not pointing to /dev/shm on $HOSTNAME, please tell the jenkins admins to fix this." + show_fstab_and_mounts +fi +# check for correct MTU +echo "$(date -u) - testing whether the network interfaces MTU is 1500..." +if [ "$(ip link | sed -n '/LOOPBACK\|NOARP/!s/.* mtu \([0-9]*\) .*/\1/p' | sort -u)" != "1500" ] ; then + ip link + echo "$(date -u) - network interfaces MTU != 1500 - this is wrong. => please \`sudo ifconfig eth0 mtu 1500\`" + # should probably turn this into a warning if this becomes to annoying + irc_message debian-reproducible "$HOSTNAME has wrong MTU, please tell the jenkins admins to fix this. (sudo ifconfig eth0 mtu 1500)" + exit 1 +fi +if ! $DIRTY ; then + echo "$(date -u ) - Everything seems to be fine." + echo +fi + +echo "$(date -u) - the end." + + diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index cc51a268..38330b21 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -258,6 +258,14 @@ my_hname: - 'jenkins': { my_arch: 'amd64' } my_node: '' + my_task: + - 'node_health_check': + my_description: 'Do some health checks.' + my_timed: '3 * * * *' + my_shell: '/srv/jenkins/bin/reproducible_node_health_check.sh' + my_hname: + - 'jenkins': { my_arch: 'amd64' } + my_node: '' - '{name}_{my_task}_{my_arch}_{my_hname}': my_task: @@ -311,6 +319,58 @@ - 'codethink15': { my_arch: 'arm64' } - 'codethink16': { my_arch: 'arm64' } my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh' + - '{name}_{my_task}_{my_arch}_{my_hname}': + my_task: + - 'node_health_check': + my_description: 'Do some health checks' + my_timed: '3 * * * *' + my_hname: + - 'bbx15': { my_arch: 'armhf' } + - 'bpi0': { my_arch: 'armhf' } + - 'cbxi4a': { my_arch: 'armhf' } + - 'cbxi4b': { my_arch: 'armhf' } + - 'cbxi4pro0': { my_arch: 'armhf' } + - 'cb3a': { my_arch: 'armhf' } + - 'ff2a': { my_arch: 'armhf' } + - 'ff2b': { my_arch: 'armhf' } + - 'ff4a': { my_arch: 'armhf' } + - 'ff64a': { my_arch: 'armhf' } + - 'hb0': { my_arch: 'armhf' } + - 'jtk1a': { my_arch: 'armhf' } + - 'jtx1a': { my_arch: 'armhf' } + - 'odc2a': { my_arch: 'armhf' } + - 'odxu4': { my_arch: 'armhf' } + - 'odxu4b': { my_arch: 'armhf' } + - 'odxu4c': { my_arch: 'armhf' } + - 'odu3a': { my_arch: 'armhf' } + - 'opi2a': { my_arch: 'armhf' } + - 'opi2b': { my_arch: 'armhf' } + - 'opi2c': { my_arch: 'armhf' } + - 'rpi2b': { my_arch: 'armhf' } + - 'rpi2c': { my_arch: 'armhf' } + - 'p64b': { my_arch: 'armhf' } + - 'p64c': { my_arch: 'armhf' } + - 'wbd0': { my_arch: 'armhf' } + - 'wbq0': { my_arch: 'armhf' } + - 'profitbricks1': { my_arch: 'amd64' } + - 'profitbricks2': { my_arch: 'i386' } + - 'profitbricks3': { my_arch: 'amd64' } + - 'profitbricks4': { my_arch: 'amd64' } + - 'profitbricks5': { my_arch: 'amd64' } + - 'profitbricks6': { my_arch: 'i386' } + - 'profitbricks11': { my_arch: 'amd64' } + - 'profitbricks12': { my_arch: 'i386' } + - 'profitbricks15': { my_arch: 'amd64' } + - 'profitbricks16': { my_arch: 'i386' } + - 'codethink9': { my_arch: 'arm64' } + - 'codethink10': { my_arch: 'arm64' } + - 'codethink11': { my_arch: 'arm64' } + - 'codethink12': { my_arch: 'arm64' } + - 'codethink13': { my_arch: 'arm64' } + - 'codethink14': { my_arch: 'arm64' } + - 'codethink15': { my_arch: 'arm64' } + - 'codethink16': { my_arch: 'arm64' } + my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh' - '{name}_{my_task}_{my_arch}_{my_hname}': my_task: |