diff options
30 files changed, 1399 insertions, 7 deletions
@@ -138,10 +138,10 @@ Installation tests inside chroot environments. * Currently, three suites are tested on 'amd64', 'i386' and 'armhf' architectures: 'testing', 'unstable' and 'experimental'. The tests are done using 'pbuilder' using link:https://wiki.debian.org/ReproducibleBuilds/ExperimentalToolchain[our toolchain] through concurrent builder jobs, 16 for 'amd64' and 'i386' each, and 57 for 'armhf', which are each constantly testing packages and saving the results of these tests. ** These builds on remote nodes run on very different hardware: for 'amd64' and 'i386' we are using two virtual machines each, profitbricks-build(1+5)-amd64 and profitbricks-build(2+6)-i386, which have 18 or 17 cores and 48gb ram each and are sponsored by link:https://jenkins.debian.net/userContent/thanks.html[Profitbricks]. -** To test 'armhf' we are using 21 small boards donated by vagrant@d.o: +** To test 'armhf' we are using 22 small boards donated by vagrant@d.o: *** five quad-cores (cbxi4a, cbxi4b, and ff4a) with 4gb ram, *** three octo-cores (odxu4, odxu4b and odxu4c) with 2gb ram, -*** eight quad-cores (wbq0, cbxi4pro0, ff2a, ff2b, odu3a, opi2b, opi2c and jtk1a) with 2gb ram, +*** nine quad-cores (wbq0, cbxi4pro0, ff2a, ff2b, odu3a, opi2a, opi2b, opi2c and jtk1a) with 2gb ram, *** two dual-core (bbx15 and cb3a) with 2gb ram and, *** two quad-cores (rpi2b and rpi2c) with 1gb ram and *** three dual-cores (bpi0, hb0 and wbd0) with 1gb ram, each. diff --git a/THANKS.head b/THANKS.head index b0f6f143..36edac62 100644 --- a/THANKS.head +++ b/THANKS.head @@ -18,10 +18,10 @@ Debian packages for t.r-b.o, running in the future ** 4 cores and 8 GB memory for profitbricks-build9-amd64.debian.net used for rebootstrap jobs ** 4 cores and 7 GB memory for profitbricks-build10-amd64.debian.net used for lvc jobs ** 3 cores and 6 GB memory for freebsd-jenkins.debian.net (also running on Profitbricks virtual hardware), used for building FreeBSD for t.r-b.o - * link:https://qa.debian.org/developer.php?login=vagrant%40debian.org[Vagrant] provides and hosts 21 'armhf' systems, used for building armhf Debian packages for t.r-b.o: + * link:https://qa.debian.org/developer.php?login=vagrant%40debian.org[Vagrant] provides and hosts 22 'armhf' systems, used for building armhf Debian packages for t.r-b.o: ** three quad-cores with 4 GB RAM each, ** three octo-cores with 2 GB RAM each, - ** eight quad-cores with 2 GB RAM each, + ** nine quad-cores with 2 GB RAM each, ** two dual-core with 2 GB RAM, ** two quad-cores with 1 GB RAM and ** three dual-cores with 1 GB RAM. diff --git a/bin/jenkins_node_definitions.sh b/bin/jenkins_node_definitions.sh index 5b09cedd..6750d0b8 100755 --- a/bin/jenkins_node_definitions.sh +++ b/bin/jenkins_node_definitions.sh @@ -34,6 +34,7 @@ ff2a-armhf-rb.debian.net ff2b-armhf-rb.debian.net ff4a-armhf-rb.debian.net jtk1a-armhf-rb.debian.net +opi2a-armhf-rb.debian.net opi2b-armhf-rb.debian.net opi2c-armhf-rb.debian.net" @@ -89,6 +90,9 @@ get_node_ssh_port() { ff4a-armhf-rb.debian.net) PORT=2241 ;; + opi2a-armhf-rb.debian.net) + PORT=2236 + ;; opi2b-armhf-rb.debian.net) PORT=2238 ;; diff --git a/deploy_everywhere b/deploy_everywhere index b7930a4b..e9121d62 100755 --- a/deploy_everywhere +++ b/deploy_everywhere @@ -25,7 +25,7 @@ START=$(date +'%s') GIT_REPO="git://anonscm.debian.org/qa/jenkins.debian.net.git" -export HOSTS="profitbricks-build1-amd64.debian.net profitbricks-build2-i386.debian.net profitbricks-build3-amd64.debian.net profitbricks-build4-amd64.debian.net profitbricks-build5-amd64.debian.net profitbricks-build6-i386.debian.net profitbricks-build7-amd64.debian.net profitbricks-build9-amd64.debian.net profitbricks-build10-amd64.debian.net bbx15-armhf-rb.debian.net bpi0-armhf-rb.debian.net hb0-armhf-rb.debian.net wbq0-armhf-rb.debian.net cbxi4pro0-armhf-rb.debian.net cbxi4a-armhf-rb.debian.net cbxi4b-armhf-rb.debian.net odxu4-armhf-rb.debian.net wbd0-armhf-rb.debian.net rpi2b-armhf-rb.debian.net rpi2c-armhf-rb.debian.net root@jenkins.debian.net odxu4b-armhf-rb.debian.net odxu4c-armhf-rb.debian.net ff2a-armhf-rb.debian.net ff2b-armhf-rb.debian.net ff4a-armhf-rb.debian.net opi2b-armhf-rb.debian.net odu3a-armhf-rb.debian.net cb3a-armhf-rb.debian.net opi2c-armhf-rb.debian.net jtk1a-armhf-rb.debian.net" +export HOSTS="profitbricks-build1-amd64.debian.net profitbricks-build2-i386.debian.net profitbricks-build3-amd64.debian.net profitbricks-build4-amd64.debian.net profitbricks-build5-amd64.debian.net profitbricks-build6-i386.debian.net profitbricks-build7-amd64.debian.net profitbricks-build9-amd64.debian.net profitbricks-build10-amd64.debian.net bbx15-armhf-rb.debian.net bpi0-armhf-rb.debian.net hb0-armhf-rb.debian.net wbq0-armhf-rb.debian.net cbxi4pro0-armhf-rb.debian.net cbxi4a-armhf-rb.debian.net cbxi4b-armhf-rb.debian.net odxu4-armhf-rb.debian.net wbd0-armhf-rb.debian.net rpi2b-armhf-rb.debian.net rpi2c-armhf-rb.debian.net root@jenkins.debian.net odxu4b-armhf-rb.debian.net odxu4c-armhf-rb.debian.net ff2a-armhf-rb.debian.net ff2b-armhf-rb.debian.net ff4a-armhf-rb.debian.net opi2a-armhf-rb.debian.net opi2b-armhf-rb.debian.net odu3a-armhf-rb.debian.net cb3a-armhf-rb.debian.net opi2c-armhf-rb.debian.net jtk1a-armhf-rb.debian.net" echo echo -n "$(date) - " # real command, for running manually: cd ~jenkins-adm/jenkins.debian.net/ ; sudo -u jenkins-adm git pull ; ./update_jdn.sh diff --git a/hosts/jenkins/etc/munin/munin.conf b/hosts/jenkins/etc/munin/munin.conf index 4a6374bf..6e11b18f 100644 --- a/hosts/jenkins/etc/munin/munin.conf +++ b/hosts/jenkins/etc/munin/munin.conf @@ -364,6 +364,16 @@ contact.me.command mail -s "Munin notification ${var:host}" root diskstats_utilization.graph no diskstats_iops.graph no +[opi2a-armhf-rb.debian.net] + address ssh://jenkins@opi2a-armhf-rb.debian.net:2236/bin/nc localhost 4949 + use_node_name yes + diskstats_latency.sda.avgwrwait.warning 0:20 + diskstats_latency.sda.avgrdwait.warning 0:10 + diskstats_latency.graph no + diskstats_throughput.graph no + diskstats_utilization.graph no + diskstats_iops.graph no + [opi2b-armhf-rb.debian.net] address ssh://jenkins@opi2b-armhf-rb.debian.net:2238/bin/nc localhost 4949 use_node_name yes @@ -497,5 +507,5 @@ contact.me.command mail -s "Munin notification ${var:host}" root # node_order Totals fii.foo.com fay.foo.com # [debian.net;] - node_order jenkins.debian.net profitbricks-build1-amd64.debian.net profitbricks-build2-i386.debian.net profitbricks-build3-amd64.debian.net profitbricks-build4-amd64.debian.net profitbricks-build5-amd64.debian.net profitbricks-build6-i386.debian.net profitbricks-build9-amd64.debian.net profitbricks-build10-amd64.debian.net bbx15-armhf-rb.debian.net bpi0-armhf-rb.debian.net cbxi4a-armhf-rb.debian.net cbxi4b-armhf-rb.debian.net cbxi4pro0-armhf-rb.debian.net cb3a-armhf-rb.debian.net ff2a-armhf-rb.debian.net ff2b-armhf-rb.debian.net ff4a-armhf-rb.debian.net hb0-armhf-rb.debian.net odxu4-armhf-rb.debian.net odxu4b-armhf-rb.debian.net odxu4c-armhf-rb.debian.net odu3a-armhf-rb.debian.net jtk1a-armhf-rb.debian.net opi2b-armhf-rb.debian.net opi2c-armhf-rb.debian.net rpi2b-armhf-rb.debian.net rpi2c-armhf-rb.debian.net wbd0-armhf-rb.debian.net wbq0-armhf-rb.debian.net freebsd-jenkins.debian.net jenkins-test-vm.debian.net + node_order jenkins.debian.net profitbricks-build1-amd64.debian.net profitbricks-build2-i386.debian.net profitbricks-build3-amd64.debian.net profitbricks-build4-amd64.debian.net profitbricks-build5-amd64.debian.net profitbricks-build6-i386.debian.net profitbricks-build9-amd64.debian.net profitbricks-build10-amd64.debian.net bbx15-armhf-rb.debian.net bpi0-armhf-rb.debian.net cbxi4a-armhf-rb.debian.net cbxi4b-armhf-rb.debian.net cbxi4pro0-armhf-rb.debian.net cb3a-armhf-rb.debian.net ff2a-armhf-rb.debian.net ff2b-armhf-rb.debian.net ff4a-armhf-rb.debian.net hb0-armhf-rb.debian.net odxu4-armhf-rb.debian.net odxu4b-armhf-rb.debian.net odxu4c-armhf-rb.debian.net odu3a-armhf-rb.debian.net jtk1a-armhf-rb.debian.net opi2a-armhf-rb.debian.net opi2b-armhf-rb.debian.net opi2c-armhf-rb.debian.net rpi2b-armhf-rb.debian.net rpi2c-armhf-rb.debian.net wbd0-armhf-rb.debian.net wbq0-armhf-rb.debian.net freebsd-jenkins.debian.net jenkins-test-vm.debian.net diff --git a/hosts/opi2a b/hosts/opi2a new file mode 120000 index 00000000..48aff003 --- /dev/null +++ b/hosts/opi2a @@ -0,0 +1 @@ +opi2a-armhf-rb
\ No newline at end of file diff --git a/hosts/opi2a-armhf-rb/etc/apt/apt.conf.d/10no-package-descriptions b/hosts/opi2a-armhf-rb/etc/apt/apt.conf.d/10no-package-descriptions new file mode 100644 index 00000000..2318f84e --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/apt/apt.conf.d/10no-package-descriptions @@ -0,0 +1 @@ +Acquire::Languages "none"; diff --git a/hosts/opi2a-armhf-rb/etc/apt/apt.conf.d/80proxy b/hosts/opi2a-armhf-rb/etc/apt/apt.conf.d/80proxy new file mode 100644 index 00000000..9e738254 --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/apt/apt.conf.d/80proxy @@ -0,0 +1,2 @@ +Acquire::http::Proxy "http://10.0.0.15:8000/"; + diff --git a/hosts/opi2a-armhf-rb/etc/apt/listchanges.conf b/hosts/opi2a-armhf-rb/etc/apt/listchanges.conf new file mode 100644 index 00000000..8b598c0a --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/apt/listchanges.conf @@ -0,0 +1,6 @@ +[apt] +frontend=mail +email_address=root +confirm=0 +save_seen=/var/lib/apt/listchanges.db +which=both diff --git a/hosts/opi2a-armhf-rb/etc/apt/sources.list b/hosts/opi2a-armhf-rb/etc/apt/sources.list new file mode 100644 index 00000000..70669281 --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/apt/sources.list @@ -0,0 +1,11 @@ +deb http://ftp.us.debian.org/debian/ jessie main contrib non-free +#deb-src http://ftp.us.debian.org/debian/ jessie main contrib non-free + +deb http://ftp.us.debian.org/debian/ jessie-updates main contrib non-free +#deb-src http://ftp.us.debian.org/debian/ jessie-updates main contrib non-free + +deb http://security.debian.org/ jessie/updates main contrib non-free +#deb-src http://security.debian.org/ jessie/updates main contrib non-free + +deb http://ftp.us.debian.org/debian/ jessie-backports main contrib non-free +#deb-src http://ftp.us.debian.org/debian/ jessie-backports main contrib non-free diff --git a/hosts/opi2a-armhf-rb/etc/cron.d/dsa b/hosts/opi2a-armhf-rb/etc/cron.d/dsa new file mode 100755 index 00000000..9be64c36 --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/cron.d/dsa @@ -0,0 +1,10 @@ +# m h dom mon dow (0|7=sun,1=mon) command + +# +# cron-jobs for jenkins.debian.net and nodes +# + +MAILTO=root + +0 1,13 * * * nobody /usr/bin/chronic /usr/local/bin/dsa-check-running-kernel +2 1,13 * * * nobody /usr/bin/chronic /usr/local/bin/dsa-check-packages diff --git a/hosts/opi2a-armhf-rb/etc/munin/munin-node.conf b/hosts/opi2a-armhf-rb/etc/munin/munin-node.conf new file mode 100644 index 00000000..65ba8e0f --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/munin/munin-node.conf @@ -0,0 +1,66 @@ +# +# Example config-file for munin-node +# + +log_level 4 +log_file /var/log/munin/munin-node.log +pid_file /var/run/munin/munin-node.pid + +background 1 +setsid 1 + +user root +group root + +# This is the timeout for the whole transaction. +# Units are in sec. Default is 15 min +# +# global_timeout 900 + +# This is the timeout for each plugin. +# Units are in sec. Default is 1 min +# +# timeout 60 + +# Regexps for files to ignore +ignore_file [\#~]$ +ignore_file DEADJOE$ +ignore_file \.bak$ +ignore_file %$ +ignore_file \.dpkg-(tmp|new|old|dist)$ +ignore_file \.rpm(save|new)$ +ignore_file \.pod$ + +# Set this if the client doesn't report the correct hostname when +# telnetting to localhost, port 4949 +# +#host_name localhost.localdomain + +# A list of addresses that are allowed to connect. This must be a +# regular expression, since Net::Server does not understand CIDR-style +# network notation unless the perl module Net::CIDR is installed. You +# may repeat the allow line as many times as you'd like + +allow ^127\.0\.0\.1$ +allow ^::1$ + +# If you have installed the Net::CIDR perl module, you can use one or more +# cidr_allow and cidr_deny address/mask patterns. A connecting client must +# match any cidr_allow, and not match any cidr_deny. Note that a netmask +# *must* be provided, even if it's /32 +# +# Example: +# +# cidr_allow 127.0.0.1/32 +# cidr_allow 192.0.2.0/24 +# cidr_deny 192.0.2.42/32 + +# Which address to bind to; +host * +# host 127.0.0.1 + +# And which port +port 4949 + +allow ^78\.137\.96\.196 +hostname opi2a-armhf-rb.debian.net diff --git a/hosts/opi2a-armhf-rb/etc/munin/plugin-conf.d/df b/hosts/opi2a-armhf-rb/etc/munin/plugin-conf.d/df new file mode 100644 index 00000000..b3fdadcb --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/munin/plugin-conf.d/df @@ -0,0 +1,6 @@ +[df*] +env.exclude none unknown iso9660 squashfs udf romfs ramfs debugfs devtmpfs sysfs +env.exclude_re /srv/workspace/pbuilder /run /dev/disk/by /var/lib/schroot/mount /srv/workspace/varlibschroot /dev/shm /sys/fs/cgroup +env.warning 92 +env.critical 98 + diff --git a/hosts/opi2a-armhf-rb/etc/munin/plugin-conf.d/diskstats b/hosts/opi2a-armhf-rb/etc/munin/plugin-conf.d/diskstats new file mode 100644 index 00000000..2d11f397 --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/munin/plugin-conf.d/diskstats @@ -0,0 +1,4 @@ +[diskstats] +env.trim_labels yes +env.include_only /dev/sda + diff --git a/hosts/opi2a-armhf-rb/etc/munin/plugin-conf.d/munin-node b/hosts/opi2a-armhf-rb/etc/munin/plugin-conf.d/munin-node new file mode 100644 index 00000000..e766928f --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/munin/plugin-conf.d/munin-node @@ -0,0 +1,117 @@ +# This file is used to configure how the plugins are invoked. +# Place in /etc/munin/plugin-conf.d/ or corresponding directory. +# +# PLEASE NOTE: Changes in the plugin-conf.d directory are only +# read at munin-node startup, so restart at any changes. +# +# user <user> # Set the user to run the plugin as. +# group <group> # Set the group to run the plugin as. +# command <command> # Run <command> instead of the plugin. %c expands to +# what would normally be run. +# env.<variable> <value> # Sets <variable> in the plugin's environment, see the +# individual plugins to find out which variables they +# care about. + + +[amavis] +group adm +env.MUNIN_MKTEMP /bin/mktemp -p /tmp/ $1 +env.amavislog /var/log/mail.info + +[apt] +user root + +[courier_mta_mailqueue] +group daemon + +[courier_mta_mailstats] +group adm + +[courier_mta_mailvolume] +group adm + +[cps*] +user root + +[exim_mailqueue] +group adm, (Debian-exim) + +[exim_mailstats] +group adm, (Debian-exim) +env.logdir /var/log/exim4/ +env.logname mainlog + +[fw_conntrack] +user root + +[fw_forwarded_local] +user root + +[hddtemp_smartctl] +user root + +[hddtemp2] +user root + +[if_*] +user root + +[if_err_*] +user nobody + +[ip_*] +user root + +[ipmi_*] +user root + +[mysql*] +user root +env.mysqlopts --defaults-file=/etc/mysql/debian.cnf +env.mysqluser debian-sys-maint +env.mysqlconnection DBI:mysql:mysql;mysql_read_default_file=/etc/mysql/debian.cnf + +[postfix_mailqueue] +user postfix + +[postfix_mailstats] +group adm + +[postfix_mailvolume] +group adm +env.logfile mail.log + +[smart_*] +user root + +[vlan*] +user root + +[ejabberd*] +user ejabberd +env.statuses available away chat xa +env.days 1 7 30 + +[dhcpd3] +user root +env.leasefile /var/lib/dhcp3/dhcpd.leases +env.configfile /etc/dhcp3/dhcpd.conf + +[jmx_*] +env.ip 127.0.0.1 +env.port 5400 + +[samba] +user root + +[munin_stats] +user munin +group munin + +[postgres_*] +user postgres +env.PGUSER postgres +env.PGPORT 5432 + +[fail2ban] +user root diff --git a/hosts/opi2a-armhf-rb/etc/munin/plugins/jenkins_reproducible_builds b/hosts/opi2a-armhf-rb/etc/munin/plugins/jenkins_reproducible_builds new file mode 100755 index 00000000..e990c127 --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/munin/plugins/jenkins_reproducible_builds @@ -0,0 +1,56 @@ +#!/bin/sh +# -*- sh -*- + +: << =cut + +=head1 NAME + +jenkins_reproducible_builds - Plugin to measure number of reproducible builds running + +=head1 AUTHOR + +Contributed by Holger Levsen + +=head1 LICENSE + +GPLv2 + +=head1 MAGIC MARKERS + + #%# family=auto + #%# capabilities=autoconf + +=cut + +. $MUNIN_LIBDIR/plugins/plugin.sh + +if [ "$1" = "autoconf" ]; then + echo yes + exit 0 +fi + +JOB_PREFIXES="first second" +if [ "$1" = "config" ]; then + echo 'graph_title Concurrent reproducible builds running' + echo 'graph_args --base 1000 -l 0 ' + echo 'graph_scale no' + echo 'graph_total total' + echo 'graph_vlabel Concurrent reproducible builds running' + echo 'graph_category jenkins' + draw=AREA + for PREFIX in $JOB_PREFIXES ; do + echo "jenkins_reproducible_${PREFIX}_build.label $PREFIX build" + echo "jenkins_reproducible_${PREFIX}_build.draw $draw" + if [ "$draw" = "AREA" ] ; then draw=STACK ; fi + done + exit 0 +fi + +for PREFIX in $JOB_PREFIXES ; do + if [ "$PREFIX" = "first" ] ; then + NR=$(pgrep -fc "bin/bash /srv/jenkins/bin/reproducible_build.sh 1") + else + NR=$(pgrep -fc "bin/bash /srv/jenkins/bin/reproducible_build.sh 2") + fi + echo "jenkins_reproducible_${PREFIX}_build.value $NR" + done diff --git a/hosts/opi2a-armhf-rb/etc/pbuilder/rebuild-hooks/B01_cleanup b/hosts/opi2a-armhf-rb/etc/pbuilder/rebuild-hooks/B01_cleanup new file mode 100755 index 00000000..bbea69fa --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/pbuilder/rebuild-hooks/B01_cleanup @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +BUILDDIR="${BUILDDIR:-/tmp/buildd}" + +# exit if we are in the same UTS namespace as init ( != 2nd build ) +[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0 + +# cease using disorderfs +if [ -d /tmp/disorderfs ] ; then + echo -n "Unmounting /tmp/disorderfs…" + fusermount -u "$BUILDDIR" + rmdir "$BUILDDIR" + mv /tmp/disorderfs "$BUILDDIR" + echo " done." +fi diff --git a/hosts/opi2a-armhf-rb/etc/pbuilder/rebuild-hooks/C01_cleanup b/hosts/opi2a-armhf-rb/etc/pbuilder/rebuild-hooks/C01_cleanup new file mode 100755 index 00000000..73f4083c --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/pbuilder/rebuild-hooks/C01_cleanup @@ -0,0 +1,21 @@ +#!/bin/sh + +set -e + +BUILDDIR="${BUILDDIR:-/tmp/buildd}" + +echo "debug output: disk usage on $(hostname) at $(date -u)" +df -h +echo + +# exit if we are in the same UTS namespace as init ( != 2nd build ) +[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0 + +# cease using disorderfs +if [ -d /tmp/disorderfs ] ; then + echo -n "Unmounting /tmp/disorderfs…" + fusermount -u "$BUILDDIR" + rmdir "$BUILDDIR" + mv /tmp/disorderfs "$BUILDDIR" + echo " done." +fi diff --git a/hosts/opi2a-armhf-rb/etc/pbuilder/rebuild-hooks/D01_modify_environment b/hosts/opi2a-armhf-rb/etc/pbuilder/rebuild-hooks/D01_modify_environment new file mode 100755 index 00000000..27fd6445 --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/pbuilder/rebuild-hooks/D01_modify_environment @@ -0,0 +1,42 @@ +#!/bin/sh + +set -e + +BUILDDIR="${BUILDDIR:-/tmp/buildd}" + +# exit if we are in the same UTS namespace as init ( != 2nd build ) +[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0 + +echo "I: Changing host+domainname to test build reproducibility" >&2 +sed -e '/^127.0.0.1/s/$/ i-capture-the-hostname i-capture-the-hostname.i-capture-the-domain/' -i /etc/hosts +hostname i-capture-the-hostname +domainname i-capture-the-domain +echo "I: Adding a custom variable just for the fun of it..." >&2 +export CAPTURE_ENVIRONMENT="I capture the environment" + +echo "I: Changing /bin/sh to bash" >&2 +echo "dash dash/sh boolean false" | debconf-set-selections +DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash +if [ -n "$BUILDUSERNAME" ] ; then + echo "I: Setting $BUILDUSERNAME's login shell to /bin/bash" >&2 + usermod -s /bin/bash $BUILDUSERNAME + if [ -n "$BUILDUSERGECOS" ] ; then + echo "I: Setting $BUILDUSERNAME's GECOS to $BUILDUSERGECOS" >&2 + usermod -c "$BUILDUSERGECOS" $BUILDUSERNAME + fi +fi + +# temporarily disable disorderfs as we have problems properly unmounting it +exit 0 + +# use disorderfs +if [ -x /usr/bin/disorderfs ] ; then + echo -n "Moving $BUILDDIR to /tmp/disorderfs and mounting this as $BUILDDIR via the fuse disorderfs…" + mknod -m 666 /dev/fuse c 10 229 + mv "$BUILDDIR" /tmp/disorderfs + mkdir "$BUILDDIR" + disorderfs --multi-user=yes /tmp/disorderfs "$BUILDDIR" + echo " done." +else + echo "Warning: disorderfs not available." +fi diff --git a/hosts/opi2a-armhf-rb/etc/pbuilderrc b/hosts/opi2a-armhf-rb/etc/pbuilderrc new file mode 100644 index 00000000..127b42ff --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/pbuilderrc @@ -0,0 +1,48 @@ +# this is your configuration file for pbuilder. +# the file in /usr/share/pbuilder/pbuilderrc is the default template. +# /etc/pbuilderrc is the one meant for overwriting defaults in +# the default template +# +# read pbuilderrc.5 document for notes on specific options. +# +# cater for different locations +case $HOSTNAME in + jenkins|jenkins-test-vm|profitbricks-build*) MIRRORSITE=http://ftp.de.debian.org/debian ;; + bbx15|bpi0|cbxi4*|cb3*|hb0|wbq0|odxu4*|odu3*|wbd0|rpi2*|ff2*|ff4*|opi2*|jtk1*) MIRRORSITE=http://ftp.us.debian.org/debian ;; + *) echo "unsupported host, exiting." ; exit 1 ;; +esac +# only use eatmydata on armhf+i386 - on amd64 we build in tmpfs anyway +case $HOSTNAME in + profitbricks-build*i386) + EATMYDATA=yes ;; + bbx15|bpi0|cbxi4*|cb3*|hb0|wbq0|odxu4*|odu3*|wbd0|rpi2*|ff2*|ff4*|opi2*|jtk1*) + EATMYDATA=yes ;; + *) ;; +esac +EXTRAPACKAGES="" # better list them in bin/reproducible_setup_pbuilder.sh +APTCACHE="" +COMPRESSPROG="pigz" +BUILDPLACE=/srv/workspace/pbuilder # build in /srv/workspace, which is in tmpfs + +# there are software wanting /dev/shm writable, so give them so. There are tons +# of bug shm related in pbuilder, that's just a workaround for all of them. +# /run/shm is already mounted as a tmpfs by default. +BINDMOUNTS="/dev/shm" +# set PATH to predictable values, see #780729 and #780725 +PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" + +# used for reproducible builds tests, when doing the 2nd build +if [ "$(readlink /proc/1/ns/uts)" != "$(readlink /proc/self/ns/uts)" ]; then + PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" +fi + +# needed to ignore failures due to running 398 days in the future… +# (only on those 2 nodes running in the future…) +if [ "$HOSTNAME" = "profitbricks-build5-amd64" ] || [ "$HOSTNAME" = "profitbricks-build6-i386" ] ; then + case "$PBUILDER_OPERATION" in + create) + APTGETOPT=(-o Acquire::Check-Valid-Until="false") + ;; + *) ;; + esac +fi diff --git a/hosts/opi2a-armhf-rb/etc/postfix/main.cf b/hosts/opi2a-armhf-rb/etc/postfix/main.cf new file mode 100644 index 00000000..7f09cc7e --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/postfix/main.cf @@ -0,0 +1,41 @@ +# See /usr/share/postfix/main.cf.dist for a commented, more complete version + + +# Debian specific: Specifying a file name will cause the first +# line of that file to be used as the name. The Debian default +# is /etc/mailname. +#myorigin = /etc/mailname + +smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) +biff = no + +# appending .domain is the MUA's job. +append_dot_mydomain = no + +# Uncomment the next line to generate "delayed mail" warnings +#delay_warning_time = 4h + +readme_directory = no + +# TLS parameters +smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem +smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key +smtpd_use_tls=yes +smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache +smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache + +# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for +# information on enabling SSL in the smtp client. + +myhostname = opi2a-armhf-rb.debian.net +alias_maps = hash:/etc/aliases +alias_database = hash:/etc/aliases +myorigin = /etc/mailname +mydestination = opi2a-armhf-rb.debian.net, localhost +relayhost = mail.holgerlevsen.de +#mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 +mynetworks = 127.0.0.0/8 +mailbox_size_limit = 0 +recipient_delimiter = + +inet_interfaces = all +mailbox_command = /usr/bin/procmail -a "$EXTENSION" diff --git a/hosts/opi2a-armhf-rb/etc/rc.local b/hosts/opi2a-armhf-rb/etc/rc.local new file mode 100755 index 00000000..1c4ece10 --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/rc.local @@ -0,0 +1,16 @@ +#!/bin/sh -e +# +# rc.local +# +# This script is executed at the end of each multiuser runlevel. +# Make sure that the script will "exit 0" on success or any other +# value on error. +# +# In order to enable or disable this script just change the execution +# bits. +# +# By default this script does nothing. + +echo "$(date -u) - system was rebooted" | mail -s "$(hostname -f) rebooted" root + +exit 0 diff --git a/hosts/opi2a-armhf-rb/etc/schroot/default/fstab b/hosts/opi2a-armhf-rb/etc/schroot/default/fstab new file mode 100644 index 00000000..74468dd2 --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/schroot/default/fstab @@ -0,0 +1,24 @@ +# fstab: static file system information for chroots. +# Note that the mount point will be prefixed by the chroot path +# (CHROOT_PATH) +# +# <file system> <mount point> <type> <options> <dump> <pass> +/proc /proc none rw,bind 0 0 +/sys /sys none rw,bind 0 0 +/dev /dev none rw,bind 0 0 +/dev/pts /dev/pts none rw,bind 0 0 +/home /home none rw,bind 0 0 +/tmp /tmp none rw,bind 0 0 +/srv/reproducible-results /srv/reproducible-results none rw,bind 0 0 +/srv/d-i /srv/d-i none rw,bind 0 0 +/srv/jenkins /srv/jenkins none rw,bind 0 0 +/srv/live-build /srv/live-build none rw,bind 0 0 + +# It may be desirable to have access to /run, especially if you wish +# to run additional services in the chroot. However, note that this +# may potentially cause undesirable behaviour on upgrades, such as +# killing services on the host. +#/run /run none rw,bind 0 0 +#/run/lock /run/lock none rw,bind 0 0 +/dev/shm /dev/shm none rw,bind 0 0 +/run/shm /run/shm none rw,bind 0 0 diff --git a/hosts/opi2a-armhf-rb/etc/schroot/default/nssdatabases b/hosts/opi2a-armhf-rb/etc/schroot/default/nssdatabases new file mode 100644 index 00000000..72615e5d --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/schroot/default/nssdatabases @@ -0,0 +1,11 @@ +# System databases to copy into the chroot from the host system. +# +# <database name> +passwd +shadow +group +gshadow +#services +protocols +networks +hosts diff --git a/hosts/opi2a-armhf-rb/etc/sudoers.d/jenkins b/hosts/opi2a-armhf-rb/etc/sudoers.d/jenkins new file mode 100644 index 00000000..96d501ef --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/sudoers.d/jenkins @@ -0,0 +1,51 @@ +jenkins ALL= \ + NOPASSWD: /usr/sbin/debootstrap *, \ + /usr/bin/tee /schroots/*, \ + /usr/bin/tee -a /schroots/*, \ + /usr/bin/tee /etc/schroot/chroot.d/jenkins*, \ + /bin/chmod +x /schroots/*, \ + /usr/sbin/chroot /schroots/*, \ + /usr/sbin/chroot /chroots/*, \ + /usr/sbin/chroot /media/*, \ + /bin/ls -la /media/*, \ + /bin/rm -rf --one-file-system /chroots/*, \ + /bin/rm -rf --one-file-system /schroots/*, \ + /bin/rm -rf --one-file-system /srv/live-build/*, \ + /bin/rm -rf --one-file-system /srv/workspace/pbuilder/*, \ + /bin/cp -v *.iso /srv/live-build/results/*, \ + /bin/mv /chroots/* /schroots/*, \ + /bin/mv /schroots/* /schroots/*, \ + /bin/umount -l /chroots/*, \ + /bin/umount -l /schroots/*, \ + /bin/umount -l /media/*, \ + /bin/rmdir /media/*, \ + /bin/mount -o loop*, \ + /bin/mount --bind *, \ + /usr/bin/du *, \ + /bin/kill *, \ + /usr/bin/file *, \ + /bin/dd if=/dev/zero of=/dev/jenkins*, \ + /usr/bin/qemu-system-x86_64 *, \ + /usr/bin/qemu-img *, \ + /sbin/lvcreate *, /sbin/lvremove *, \ + /bin/mkdir -p /media/*, \ + /usr/bin/guestmount *, \ + /bin/cp -rv /media/*, \ + /bin/chown -R jenkins\:jenkins /var/lib/jenkins/jobs/*,\ + SETENV: NOPASSWD: /usr/sbin/pbuilder *, \ + SETENV: NOPASSWD: /usr/bin/timeout -k ??.?h ??h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder *, \ + SETENV: NOPASSWD: /usr/bin/timeout -k ??.?h ??h /usr/bin/ionice -c 3 /usr/bin/nice /usr/bin/unshare --uts -- /usr/sbin/pbuilder *, \ + /bin/mv /var/cache/pbuilder/*base*.tgz /var/cache/pbuilder/*base*.tgz, \ + /bin/rm /var/cache/pbuilder/*base*.tgz, \ + /bin/rm -v /var/cache/pbuilder/*base*.tgz, \ + /bin/rm /var/cache/pbuilder/result/*, \ + /usr/bin/dcmd rm *.changes, \ + /usr/bin/dcmd rm *.dsc, \ + /usr/bin/apt-get update, \ + /usr/bin/killall timeout, \ + /usr/sbin/slay 1111, \ + /usr/sbin/slay 2222, \ + /usr/sbin/slay jenkins + +# keep these environment variables +Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/opi2a-armhf-rb/etc/sudoers.d/jenkins-adm b/hosts/opi2a-armhf-rb/etc/sudoers.d/jenkins-adm new file mode 100644 index 00000000..3c357be2 --- /dev/null +++ b/hosts/opi2a-armhf-rb/etc/sudoers.d/jenkins-adm @@ -0,0 +1,7 @@ +# allow member of the jenkins-adm group to sudo-to the jenkins-adm user (owner +# of jenkins script) and the jenkins user itself +%jenkins-adm ALL=(jenkins:jenkins) NOPASSWD: ALL +%jenkins-adm ALL=(jenkins-adm:jenkins-adm) NOPASSWD: ALL +# allow jenkins-adm to run everything as root +%jenkins-adm ALL= NOPASSWD: ALL + diff --git a/hosts/opi2a-armhf-rb/usr/local/bin/dsa-check-packages b/hosts/opi2a-armhf-rb/usr/local/bin/dsa-check-packages new file mode 100755 index 00000000..28844e5a --- /dev/null +++ b/hosts/opi2a-armhf-rb/usr/local/bin/dsa-check-packages @@ -0,0 +1,362 @@ +#!/usr/bin/perl + +# dsa-check-packages + +# checks for obsolete/local and upgradeable packages. +# +# packages for the obsolete/local check can be ignored, by +# listing their full name in /etc/nagios/obsolete-packages-ignore +# or by having a regex (starting a line with "/") that matches +# the packagename in said file. +# +# Takes one optional argument, the location of the ignore file. + + +# Copyright (C) 2008, 2009 Peter Palfrader <peter@palfrader.org> +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +use strict; +use warnings; +use English; + +my $IGNORE = "/etc/nagios/obsolete-packages-ignore"; +my $IGNORED = "/etc/nagios/obsolete-packages-ignore.d"; + +my %CODE = ( + 'OK' => 0, + 'WARNING' => 1, + 'CRITICAL' => 2, + 'UNKNOWN' => 3 +); +my $EXITCODE = 'OK'; +sub record($) { + my ($newexit) = @_; + die "code $newexit not defined\n" unless defined $CODE{$newexit}; + + if ($CODE{$newexit} > $CODE{$EXITCODE}) { + $EXITCODE = $newexit; + }; +} + + + +sub get_packages { + $ENV{'COLUMNS'} = 1000; + $ENV{'LC_ALL'} = 'C'; + open(F, "dpkg -l|") or die ("Cannot run dpkg: $!\n"); + my @lines = <F>; + close(F); + chomp(@lines); + + my $line; + my $has_arch = 0; + while (defined($line = shift @lines) && ($line !~ /\+\+\+/)) { + if ($line =~ /Architecture/) { $has_arch = 1; } + } + + my %pkgs; + for $line (@lines) { + my ($state, $pkg, $version, $arch, undef) = split(/ */, $line); + $arch = '' unless $has_arch; + $pkgs{$state}{$pkg} = { 'installed' => $version, arch => $arch } + } + + my $installed = $pkgs{'ii'}; + delete $pkgs{'ii'}; + + my @installed_packages = keys(%$installed); + my @cmd = ("apt-cache", "policy", @installed_packages); + + open my $olderr, ">&STDERR" or die "Can't dup STDERR: $!"; + open STDERR, ">/dev/null" or die "Can't dup STDOUT: $!"; + open (F, "-|", @cmd) or die ("Cannot run apt-cache policy: $!\n"); + @lines = <F>; + close(F); + open STDERR, ">&", $olderr or die "Can't dup OLDERR: $!"; + chomp(@lines); + + my $pkgname = undef; + my $candidate_found = 0; + while (defined($line = shift @lines)) { + if ($line =~ /^([^ ]*):$/) { + # when we have multi-arch capable fu, we require that + # apt-cache policy output is in the same order as its + # arguments. + # + # We needs thi, because the output block in apt-cache + # policy does not show the arch: + # + # | weasel@stanley:~$ apt-cache policy libedit2:amd64 + # | libedit2: + # | Installed: 2.11-20080614-5 + # | Candidate: 2.11-20080614-5 + # + # We replace the package name in the output with the + # one we asked for ($pkg:$arch) - but to match this up + # sanely we need the order to be correct. + # + # For squeeze systems (no m-a), apt-cache policy output + # is all different. + $pkgname = $1; + $candidate_found = 0; + if ($has_arch) { + my $from_list = shift @installed_packages; + next if ($pkgname eq $from_list); # no :$arch in pkgname we asked for + + my $ma_fix_pkgname = $pkgname.':'.$installed->{$from_list}->{'arch'}; + my $ma_fix_from_list = $from_list.':'.$installed->{$from_list}->{'arch'}; + + if ($pkgname eq $ma_fix_from_list || # e.g. ia32-libs-i386. dpkg -l: ia32-libs-i386, apt-cache policy: ia32-libs-i386:i386 + $ma_fix_pkgname eq $from_list) { + $pkgname = $from_list; + } else { + die "Unexpected order mismatch in apt-cache policy output (apt-cache policy name: $pkgname - dpkg -l name: $from_list)\n"; + } + } + } elsif ($line =~ /^ +Installed: (.*)$/) { + # etch dpkg -l does not print epochs, so use this info, it's better + $installed->{$pkgname}{'installed'} = $1; + # initialize security-update + $installed->{$pkgname}{'security-update'} = 0; + } elsif ($line =~ /^ +Candidate: (.*)$/) { + $installed->{$pkgname}{'candidate'} = $1; + } elsif ($line =~ / ([^ ]+) [0-9]+/) { + # check if the next lines show the sources of our candidate + if ($1 eq $installed->{$pkgname}{'candidate'}) { + $candidate_found = 1; + } + } elsif (($line =~ / +[0-9]+ [^ ]+\/(security\.([^ ]+\.)?debian\.org|debian-security).*\/updates\//) && $candidate_found ) { + $installed->{$pkgname}{'security-update'} = 1; + } elsif ($line =~ /^ +\*\*\*/) { + $line = shift @lines; + my @l = split(/ +/, $line); + $installed->{$pkgname}{'origin'} = $l[2]; + $candidate_found = 0; + } + } + + my (%current, %obsolete, %outofdate, %security_outofdate); + for my $pkgname (keys %$installed) { + my $pkg = $installed->{$pkgname}; + + unless (defined($pkg->{'candidate'}) && defined($pkg->{'origin'})) { + $obsolete{$pkgname} = $pkg; + next; + } + + if ($pkg->{'candidate'} ne $pkg->{'installed'}) { + if ($pkg->{'security-update'}) { + $security_outofdate{$pkgname} = $pkg; + } else { + $outofdate{$pkgname} = $pkg; + } + next; + }; + if ($pkg->{'origin'} eq '/var/lib/dpkg/status') { + $obsolete{$pkgname} = $pkg; + next; + } + $current{$pkgname} = $pkg; + } + + $pkgs{'current'} = \%current; + $pkgs{'outofdate'} = \%outofdate; + $pkgs{'security_outofdate'} = \%security_outofdate; + $pkgs{'obsolete'} = \%obsolete; + return \%pkgs; +} + +sub load_ignores { + my ($ignorefiles, $require_file) = @_; + + my @ignores; + + for my $ignoreitem (@$ignorefiles) { + next if (!$require_file and ! -e $ignoreitem); + + my @filestoopen; + if (-d $ignoreitem) { + opendir(DIR, $ignoreitem) or die ("Cannot open dir $ignoreitem: $!\n"); + @filestoopen = readdir(DIR); + closedir(DIR); + + @filestoopen = grep { -f ($ignoreitem.'/'.$_) } @filestoopen; + @filestoopen = grep { /^([a-z0-9_.-]+)+[a-z0-9]+$/i } @filestoopen; + @filestoopen = grep { !/dpkg-(old|dist|new|tmp)$/ } @filestoopen; + @filestoopen = map { ($ignoreitem.'/'.$_) } @filestoopen; + } else { + push @filestoopen, $ignoreitem; + } + + for my $f (@filestoopen) { + open (F, "< $f") or die ("Cannot open $f: $!\n"); + push @ignores, <F>; + close F; + } + } + chomp(@ignores); + return \@ignores; +} + +sub check_ignore { + my ($pkg, $ignores) = @_; + + my $ignore_this = 0; + for my $ignore (@$ignores) { + my $ig = $ignore; + return 1 if ($ig eq $pkg); + if (substr($ig,0,1) eq '/') { + substr($ig, 0, 1, ''); + $ig =~ s,/$,,; + return 1 if ($pkg =~ /$ig/); + } + } + return 0 +} + +sub filter_ignored { + my ($packages, $ignores) = @_; + + my $obs = $packages->{'obsolete'}; + + my (%ignored, %bad); + for my $pkg (keys %$obs) { + if (check_ignore($pkg, $ignores)) { + $ignored{$pkg} = $obs->{$pkg}; + } else { + $bad{$pkg} = $obs->{$pkg}; + }; + } + delete $packages->{'obsolete'}; + $packages->{'obsolete'} = \%bad; + $packages->{'obsolete-ignored'} = \%ignored; +}; + +sub usage { + my ($fd, $exit) = @_; + print $fd "Usage: $PROGRAM_NAME [<ignorefile|dir> [<ignorefile|dir> ...]]\n"; + exit $exit; +} + +my $ignorefiles = [$IGNORE, $IGNORED]; +my $ignorefile_userset = 0; +if (@ARGV >= 1) { + usage(\*STDOUT, 0) if ($ARGV[0] eq "-h"); + usage(\*STDOUT, 0) if ($ARGV[0] eq "--help"); + $ignorefile_userset = 1; + $ignorefiles = \@ARGV; +}; + +my $ignores = load_ignores($ignorefiles, $ignorefile_userset); +my $packages = get_packages(); + +filter_ignored($packages, $ignores); + + + +my @reportform = ( + { 'key' => 'obsolete', + 'listpackages' => 1, + 'long' => "%d local or obsolete packages: %s", + 'short' => "%d obs/loc", + 'perf' => "obs_loc=%d;1;5;0", + 'status' => 'WARNING' }, + { 'key' => 'outofdate', + 'listpackages' => 1, + 'long' => "%d out of date packages: %s", + 'short' => "%d updates", + 'perf' => "outdated=%d;1;5;0", + 'status' => 'WARNING' }, + { 'key' => 'current', + 'listpackages' => 0, + 'long' => "%d packages current.", + 'short' => "%d ok", + 'perf' => "current=%d;;;0", + 'status' => 'OK' }, + { 'key' => 'obsolete-ignored', + 'listpackages' => 1, + 'long' => "%d whitelisted local or obsolete packages: %s", + 'short' => "%d obs/loc(ignored)", + 'perf' => "obs_ign=%d;;;0", + 'status' => 'OK' }, + { 'key' => 'rc', + 'listpackages' => 1, + 'long' => "%d packages removed but not purged: %s", + 'short' => "%d rc", + 'perf' => "rm_unprg=%d;;;0", + 'status' => 'OK' }, + { 'key' => 'hi', + 'listpackages' => 1, + 'long' => "%d packages on hold: %s", + 'short' => "%d hi", + 'perf' => "hold=%d;;;0", + 'status' => 'OK' }, + { 'key' => 'pc', + 'listpackages' => 1, + 'long' => "%d packages requested to be purged but conffiles still installed: %s", + 'short' => "%d pc", + 'perf' => "prg_conf=%d;1;;0", + 'status' => 'WARNING' }, + { 'key' => 'security_outofdate', + 'listpackages' => 1, + 'long' => "%d packages with outstanding security updates: %s", + 'short' => "%d security-updates", + 'perf' => "security_outdated=%d;;1;0", + 'status' => 'CRITICAL' }, + ); + +my @longout; +my @perfout; +my @shortout; +for my $form (@reportform) { + my $pkgs = $packages->{$form->{'key'}}; + delete $packages->{$form->{'key'}}; + my $num = scalar keys %$pkgs; + push @perfout, sprintf($form->{'perf'}, $num); + next unless ($num > 0); + if ($form->{'listpackages'}) { + my $list = join(", ", keys %$pkgs); + push @longout, sprintf($form->{'long'}, $num, $list); + } else { + push @longout, sprintf($form->{'long'}, $num); + }; + push @shortout, sprintf($form->{'short'}, $num); + record($form->{'status'}); +}; +if (scalar keys %$packages) { + record('WARNING'); + unshift @shortout, "unk: ".join(", ", keys %$packages); + for my $status (sort {$b cmp $a} keys %$packages) { + my $pkgs = $packages->{$status}; + my $list = join(", ", keys %$pkgs); + unshift @longout, "Unknown package status $status: $list"; + }; +} + +my $shortout = $EXITCODE.": ".join(", ", @shortout); +my $longout = join("\n", @longout); +my $perfout = "|".join(" ", @perfout); + +print $shortout,"\n"; +print $longout,"\n"; +print $perfout,"\n"; + +exit $CODE{$EXITCODE}; diff --git a/hosts/opi2a-armhf-rb/usr/local/bin/dsa-check-running-kernel b/hosts/opi2a-armhf-rb/usr/local/bin/dsa-check-running-kernel new file mode 100755 index 00000000..80f45bfb --- /dev/null +++ b/hosts/opi2a-armhf-rb/usr/local/bin/dsa-check-running-kernel @@ -0,0 +1,252 @@ +#!/bin/bash + +# Check if the running kernel has the same version string as the on-disk +# kernel image. + +# Copyright 2008,2009,2011,2012,2013,2014 Peter Palfrader +# Copyright 2009 Stephen Gran +# Copyright 2010,2012,2013 Uli Martens +# Copyright 2011 Alexander Reichle-Schmehl +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +OK=0; +WARNING=1; +CRITICAL=2; +UNKNOWN=3; + +get_offset() { + local file needle + + file="$1" + needle="$2" + + perl -e ' + undef $/; + $i = 0; $k=<>; + while (($i = index($k, "'"$needle"'", $i)) >= 0) { + print $i++,"\n"; + }; ' < "$file" +} + +get_avail() { + # This is wrong, but leaves room for when we have to care for machines running + # myfirstunix-image-0.1-dsa-arm + local prefix="$1"; shift + + local kervers=$(uname -r) + + local metavers='' + + # DSA uses kernel versions of the form 2.6.29.3-dsa-dl380-oldxeon, where + # Debian uses versions of the form 2.6.29-2-amd64 + if [ "${kervers#3}" != "$kervers" ]; then + metavers=$(echo $kervers | sed -r -e 's/^3\.[0-9]+(\.[0-9])?+-[A-Za-z0-9\.]+-(.*)/\2/') + elif [ "${kervers//dsa}" != "$kervers" ]; then + metavers=$(echo $kervers | sed -r -e 's/^2\.(4|6)\.[0-9]+([\.0-9]+?)-(.*)/2.\1-\3/') + else + metavers=$(echo $kervers | sed -r -e 's/^2\.(4|6)\.[0-9]+-[A-Za-z0-9\.]+-(.*)/2.\1-\2/') + fi + + # Attempt to track back to a metapackage failed. bail + if [ "$metavers" = "$kervers" ]; then + return 2 + fi + + # We're just going to give up if we can't find a matching metapackage + # I tried being strict once, and it just caused a lot of headaches. We'll see how + # being lax does for us + + local output=$(apt-cache policy ${prefix}-image-${metavers} 2>/dev/null) + local metaavailvers=$(echo "$output" | grep '^ Candidate:' | awk '{print $2}') + local metainstavers=$(echo "$output" | grep '^ Installed:' | awk '{print $2}') + + if [ -z "$metaavailvers" ] || [ "$metaavailvers" = '(none)' ]; then + return 2 + fi + if [ -z "$metainstavers" ] || [ "$metainstavers" = '(none)' ]; then + return 2 + fi + + if [ "$metaavailvers" != "$metainstavers" ] ; then + echo "${prefix}-image-${metavers} $metaavailvers available but $metainstavers installed" + return 1 + fi + + local imagename=0 + # --no-all-versions show shows only the candidate + for vers in $(apt-cache --no-all-versions show ${prefix}-image-${metavers} | sed -n 's/^Depends: //p' | tr ',' '\n' | tr -d ' ' | grep ${prefix}-image | awk '{print $1}' | sort -u); do + if dpkg --compare-versions "1.$vers" gt "1.$imagename"; then + imagename=$vers + fi + done + + if [ -z "$imagename" ] || [ "$imagename" = 0 ]; then + return 2 + fi + + if [ "$imagename" != "${prefix}-image-${kervers}" ]; then + if dpkg --compare-versions 1."$imagename" lt 1."${prefix}-image-${kervers}"; then + return 2 + fi + echo "$imagename" != "${prefix}-image-${kervers}" + return 1 + fi + + local availvrs=$(apt-cache policy ${imagename} 2>/dev/null | grep '^ Candidate' | awk '{print $2}') + local kernelversion=$(apt-cache policy ${prefix}-image-${kervers} 2>/dev/null | grep '^ Installed:' | awk '{print $2}') + + if [ "$availvrs" = "$kernelversion" ]; then + return 0 + fi + + echo "$kernelversion != $availvrs" + return 1 +} + +cat_vmlinux() { + local image header filter hdroff + + image="$1" + header="$2" + filter="$3" + hdroff="$4" + + get_offset "$image" $header | head -n 5 | while read off; do + (if [ "$off" != 0 ]; then + dd ibs="$((off+hdroff))" skip=1 count=0 + fi && + dd bs=512k) < "$image" 2>/dev/null | $filter 2>/dev/null + done +} + +get_image_linux() { + local image + + image="$1" + + # gzip compressed image + cat_vmlinux "$image" "\x1f\x8b\x08\x00" "zcat" 0 + cat_vmlinux "$image" "\x1f\x8b\x08\x08" "zcat" 0 + # lzma compressed image + cat_vmlinux "$image" "\x00\x00\x00\x02\xff" "xzcat" -1 + cat_vmlinux "$image" "\x00\x00\x00\x04\xff" "xzcat" -1 + # xz compressed image + cat_vmlinux "$image" "\xfd\x37\x7a\x58\x5a " "xzcat" 0 + + echo "ERROR: Unable to extract kernel image." 2>&1 + exit 1 +} + + +freebsd_check_running_version() { + local imagefile="$1"; shift + + local r="$(uname -r)" + local v="$(uname -v| sed -e 's/^#[0-9]*/&:/')" + + local q='@(#)FreeBSD '"$r $v" + + if zcat "$imagefile" | $STRINGS | grep -F -q "$q"; then + echo "OK" + else + echo "not OK" + fi +} + +STRINGS=""; +if [ -x "$(which strings)" ]; then + STRINGS="$(which strings)" +elif [ -x "$(which busybox)" -a "$( echo foobar | $(which busybox) strings 2>/dev/null)" = "foobar" ]; then + STRINGS="$(which busybox) strings" +fi + +searched="" +for on_disk in \ + "/boot/vmlinuz-`uname -r`"\ + "/boot/vmlinux-`uname -r`"\ + "/boot/kfreebsd-`uname -r`.gz"; do + + if [ -e "$on_disk" ]; then + if [ -z "$STRINGS" ]; then + echo "UNKNOWN: 'strings' command missing, perhaps install binutils or busybox?" + exit $UNKNOWN + fi + if [ "${on_disk/vmlinu}" != "$on_disk" ]; then + on_disk_version="`get_image_linux "$on_disk" | $STRINGS | grep 'Linux version' | head -n1`" + if [ -x /usr/bin/lsb_release ] ; then + vendor=$(lsb_release -i -s) + if [ -n "$vendor" ] && [ "xDebian" != "x$vendor" ] ; then + on_disk_version=$( echo $on_disk_version|sed -e "s/ ($vendor [[:alnum:]\.-]\+ [[:alnum:]\.]\+)//") + fi + fi + [ -z "$on_disk_version" ] || break + on_disk_version="`cat "$on_disk" | $STRINGS | grep 'Linux version' | head -n1`" + [ -z "$on_disk_version" ] || break + + echo "UNKNOWN: Failed to get a version string from image $on_disk" + exit $UNKNOWN + else + on_disk_version="$(zcat $on_disk | $STRINGS | grep Debian | head -n 1 | sed -e 's/Debian [[:alnum:]]\+ (\(.*\))/\1/')" + fi + fi + searched="$searched $on_disk" +done + +if ! [ -e "$on_disk" ]; then + echo "WARNING: Did not find a kernel image (checked$searched) - I have no idea which kernel I am running" + exit $WARNING +fi + +if [ "$(uname -s)" = "Linux" ]; then + running_version="`cat /proc/version`" + if [ -z "$running_version" ] ; then + echo "UNKNOWN: Failed to get a version string from running system" + exit $UNKNOWN + fi + + if [ "$running_version" != "$on_disk_version" ]; then + echo "WARNING: Running kernel does not match on-disk kernel image: [$running_version != $on_disk_version]" + exit $WARNING + fi + + ret="$(get_avail linux)" + if [ $? = 1 ]; then + echo "WARNING: Kernel needs upgrade [$ret]" + exit $WARNING + fi +else + image_current=$(freebsd_check_running_version $on_disk) + running_version="`uname -s` `uname -r` `uname -v`" + if [ "$image_current" != "OK" ]; then + approx_time="$(date -d "@`stat -c '%Y' "$on_disk"`" +"%Y-%m-%d %H:%M:%S")" + echo "WARNING: Currently running kernel ($running_version) does not match on disk image (~ $approx_time)" + exit $WARNING; + fi + + ret="$(get_avail linux)" + if [ $? = 1 ]; then + echo "WARNING: Kernel needs upgrade [$ret]" + exit $WARNING + fi +fi + +echo "OK: Running kernel matches on disk image: [$running_version]" +exit $OK diff --git a/hosts/opi2a-armhf-rb/usr/local/sbin/nagios-check-libs b/hosts/opi2a-armhf-rb/usr/local/sbin/nagios-check-libs new file mode 100755 index 00000000..77b37805 --- /dev/null +++ b/hosts/opi2a-armhf-rb/usr/local/sbin/nagios-check-libs @@ -0,0 +1,204 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2005, 2006, 2007, 2008, 2012, 2015 Peter Palfrader <peter@palfrader.org> +# 2012 Uli Martens <uli@youam.net> +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +use strict; +use English; +use Getopt::Long; + +$ENV{'PATH'} = '/bin:/sbin:/usr/bin:/usr/sbin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; + +my $LSOF = '/usr/bin/lsof -F0'; +my $VERSION = '0.2015012901'; + +# nagios exit codes +my $OK = 0; +my $WARNING = 1; +my $CRITICAL = 2; +my $UNKNOWN = 3; + +my $params; +my $config; + +Getopt::Long::config('bundling'); + +sub dief { + print STDERR @_; + exit $UNKNOWN; +} + +if (!GetOptions ( + '--help' => \$params->{'help'}, + '--version' => \$params->{'version'}, + '--quiet' => \$params->{'quiet'}, + '--verbose' => \$params->{'verbose'}, + '-v' => \$params->{'verbose'}, + '--config=s' => \$params->{'config'}, + )) { + dief ("$PROGRAM_NAME: Usage: $PROGRAM_NAME [--help|--version] [--verbose] [--quiet] [--config=<CONFIGFILE>]\n"); +}; +if ($params->{'help'}) { + print "$PROGRAM_NAME: Usage: $PROGRAM_NAME [--help|--version] [--verbose] [--quiet] [--config=<CONFIGFILE>]\n"; + print "Reports processes that are linked against libraries that no longer exist.\n"; + print "The optional config file can specify ignore rules - see the sample config file.\n"; + exit (0); +}; +if ($params->{'version'}) { + print "nagios-check-libs $VERSION\n"; + print "nagios check for availability of debian (security) updates\n"; + print "Copyright (c) 2005, 2006, 2007, 2008, 2012 Peter Palfrader <peter\@palfrader.org>\n"; + exit (0); +}; + +if (! defined $params->{'config'}) { + $params->{'config'} = '/etc/nagios/check-libs.conf'; +} elsif (! -e $params->{'config'}) { + dief("Config file $params->{'config'} does not exist.\n"); +} + +if (-e $params->{'config'}) { + eval "use YAML::Syck; 1" or dief "you need YAML::Syck (libyaml-syck-perl) to load a config file"; + open(my $fh, '<', $params->{'config'}) or dief "Cannot open config file $params->{'config'}: $!"; + $config = LoadFile($fh); + close($fh); + if (!(ref($config) eq "HASH")) { + dief("Loaded config is not a hash!\n"); + } +} else { + $config = { + 'ignorelist' => [ + '$path =~ m#^/proc/#', + '$path =~ m#^/var/tmp/#', + '$path =~ m#^/SYS#', + '$path =~ m#^/drm$# # xserver stuff', + '$path =~ m#^/dev/zero#', + '$path =~ m#^/dev/shm/#', + ] + }; +} + +if (! exists $config->{'ignorelist'}) { + $config->{'ignorelist'} = []; +} elsif (! (ref($config->{'ignorelist'}) eq 'ARRAY')) { + dief("Config->ignorelist is not an array!\n"); +} + + +my %processes; + +sub getPIDs($$) { + my ($user, $process) = @_; + return join(', ', sort keys %{ $processes{$user}->{$process} }); +}; +sub getProcs($) { + my ($user) = @_; + + return join(', ', map { $_.' ('.getPIDs($user, $_).')' } (sort {$a cmp $b} keys %{ $processes{$user} })); +}; +sub getUsers() { + return join('; ', (map { $_.': '.getProcs($_) } (sort {$a cmp $b} keys %processes))); +}; +sub inVserver() { + my ($f, $key); + if (-e "/proc/self/vinfo" ) { + $f = "/proc/self/vinfo"; + $key = "XID"; + } else { + $f = "/proc/self/status"; + $key = "s_context"; + }; + open(F, "< $f") or return 0; + while (<F>) { + my ($k, $v) = split(/: */, $_, 2); + if ($k eq $key) { + close F; + return ($v > 0); + }; + }; + close F; + return 0; +} + +my $INVSERVER = inVserver(); + +print STDERR "Running $LSOF -n\n" if $params->{'verbose'}; +open (LSOF, "$LSOF -n|") or dief ("Cannot run $LSOF -n: $!\n"); +my @lsof=<LSOF>; +close LSOF; +if ($CHILD_ERROR) { # program failed + dief("$LSOF -n returned with non-zero exit code: ".($CHILD_ERROR / 256)."\n"); +}; + +my ($process, $pid, $user); +LINE: for my $line (@lsof) { + if ( $line =~ /^p/ ) { + my %fields = map { m/^(.)(.*)$/ ; $1 => $2 } grep { defined $_ and length $_ >1} split /\0/, $line; + $process = $fields{c}; + $pid = $fields{p}; + $user = $fields{L}; + next; + } + + unless ( $line =~ /^f/ ) { + dief("UNKNOWN strange line read from lsof\n"); + # don't print it because it contains NULL characters... + } + + my %fields = map { m/^(.)(.*)$/ ; $1 => $2 } grep { defined $_ and length $_ >1} split /\0/, $line; + + my $fd = $fields{f}; + my $inode = $fields{i}; + my $path = $fields{n}; + if ($path =~ m/\.dpkg-/ || $path =~ m/\(deleted\)/ || $path =~ /path inode=/ || $path =~ m#/\.nfs# || $fd eq 'DEL') { + my $deleted_in_path = ($path =~ m/\(deleted\)/ || $path =~ m/\.nfs/); + next if ($deleted_in_path && $fd =~ /^[0-9]*$/); # Ignore deleted files that are open via normal file handles. + next if ($deleted_in_path && $fd eq 'cwd'); # Ignore deleted directories that we happen to be in. + + $path =~ s/^\(deleted\)//; # in some cases "(deleted)" is at the beginning of the string + for my $i (@{$config->{'ignorelist'}}) { + my $ignore = eval($i); + next LINE if $ignore; + } + next if ($INVSERVER && ($process eq 'init') && ($pid == 1) && ($user eq 'root')); + if ( $params->{'verbose'} ) { + print STDERR "adding $process($pid) because of [$path]:\n"; + print STDERR $line; + } + $processes{$user}->{$process}->{$pid} = 1; + }; +}; + + + +my $message=''; +my $exit = $OK; +if (keys %processes) { + $exit = $WARNING; + $message = 'The following processes have libs linked that were upgraded: '. getUsers()."\n"; +} else { + $message = "No upgraded libs linked in running processes\n" unless $params->{'quiet'}; +}; + +print $message; +exit $exit; diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index d4890533..36f2d36b 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -328,6 +328,7 @@ - 'odxu4b': { my_arch: 'armhf' } - 'odxu4c': { my_arch: 'armhf' } - 'odu3a': { my_arch: 'armhf' } + - 'opi2a': { my_arch: 'armhf' } - 'opi2b': { my_arch: 'armhf' } - 'opi2c': { my_arch: 'armhf' } - 'wbd0': { my_arch: 'armhf' } @@ -378,6 +379,7 @@ - 'odxu4b': { my_arch: 'armhf' } - 'odxu4c': { my_arch: 'armhf' } - 'odu3a': { my_arch: 'armhf' } + - 'opi2a': { my_arch: 'armhf' } - 'opi2b': { my_arch: 'armhf' } - 'opi2c': { my_arch: 'armhf' } - 'wbd0': { my_arch: 'armhf' } @@ -563,7 +565,7 @@ # could have all archs in one block, but it seems cleaner this way # to choose new jobs: - # for i in bbx15 bpi0 cbxi4a cbxi4b cbxi4pro0 cb3a ff2a ff2b ff4a hb0 jtk1a odxu4 odxu4b odxu4c odu3a opi2b opi2c rpi2b rpi2c wbd0 wbq0 ; do echo "$i: " ; grep my_node1 job-cfg/reproducible.yaml|grep armhf|grep $i-armhf ; echo ; done + # for i in bbx15 bpi0 cbxi4a cbxi4b cbxi4pro0 cb3a ff2a ff2b ff4a hb0 jtk1a odxu4 odxu4b odxu4c odu3a opi2a opi2b opi2c rpi2b rpi2c wbd0 wbq0 ; do echo "$i: " ; grep my_node1 job-cfg/reproducible.yaml|grep armhf|grep $i-armhf ; echo ; done # 8 jobs for quad-cores with 4 gb ram # 6 jobs for octo-cores with 2 gb ram # 6 jobs for quad-cores with 2 gb ram |