diff options
| author | Vagrant Cascadian <vagrant@debian.org> | 2017-08-02 17:03:03 -0400 |
|---|---|---|
| committer | Holger Levsen <holger@layer-acht.org> | 2017-08-03 16:22:39 -0400 |
| commit | 4c5ea6af7385064575531b1572c9bebf999b005d (patch) | |
| tree | c8528a53c4157d3e5b945149e887f5eaf117a247 | |
| parent | 34c1dd358e6879664f12703ccdcd1e88f11854b1 (diff) | |
| download | jenkins.debian.net-4c5ea6af7385064575531b1572c9bebf999b005d.tar.xz | |
reproducible: Add two new armhf builders (jtk1b, jtx1b).
Temporarily disable two armhf build nodes (jtk1a down, ff64a too
slow).
Add an additional build job, as the new machines have more capacity
than the old.
Signed-off-by: Holger Levsen <holger@layer-acht.org>
58 files changed, 3049 insertions, 25 deletions
@@ -135,7 +135,7 @@ Installation tests inside chroot environments. * The (current) purpose of https://tests.reproducible-builds.org is to show the potential of reproducible builds for Debian - and six other projects currently. This is research, showing what could (and should) be done... check https://wiki.debian.org/ReproducibleBuilds for the real status of the project for Debian! -* For Debian, four suites, 'stretch', 'buster', 'unstable' and 'experimental', are tested on four architectures: 'amd64', 'i386', 'arm64' and 'armhf'. The tests are done using 'pbuilder' through several concurrent workers: 40 for 'amd64', 24 for 'i386', 32 for 'arm64' and 51 for 'armhf', which are each constantly testing packages and saving the results of these tests. There's a single link:https://anonscm.debian.org/git/qa/jenkins.debian.net.git/tree/bin/reproducible_build_service.sh[systemd service] starting all of these link:https://anonscm.debian.org/git/qa/jenkins.debian.net.git/tree/bin/reproducible_worker.sh[workers] which in turn launch the actual link:https://anonscm.debian.org/git/qa/jenkins.debian.net.git/tree/bin/reproducible_build.sh[build script]. (So the actual builds and tests are happening outside the jenkins service.) +* For Debian, four suites, 'stretch', 'buster', 'unstable' and 'experimental', are tested on four architectures: 'amd64', 'i386', 'arm64' and 'armhf'. The tests are done using 'pbuilder' through several concurrent workers: 40 for 'amd64', 24 for 'i386', 32 for 'arm64' and 52 for 'armhf', which are each constantly testing packages and saving the results of these tests. There's a single link:https://anonscm.debian.org/git/qa/jenkins.debian.net.git/tree/bin/reproducible_build_service.sh[systemd service] starting all of these link:https://anonscm.debian.org/git/qa/jenkins.debian.net.git/tree/bin/reproducible_worker.sh[workers] which in turn launch the actual link:https://anonscm.debian.org/git/qa/jenkins.debian.net.git/tree/bin/reproducible_build.sh[build script]. (So the actual builds and tests are happening outside the jenkins service.) ** To shutdown all the workers use: `sudo systemctl stop reproducible_build@startup.service ; /srv/jenkins/bin/reproducible_cleanup_nodes.sh` ** To start all the workers use: `sudo systemctl start reproducible_build@startup.service` @@ -144,10 +144,9 @@ Installation tests inside chroot environments. ** for 'i386' we are also using four virtual machines, profitbricks-build(2+6+12+16)-i386, which have 10 or 9 cores and 36gb ram each. pb2+12 run emulated AMD Opteron CPUs and pb6+16 Intel Xeon CPUs. These nodes are also sponsored by link:https://jenkins.debian.net/userContent/thanks.html[Profitbricks]. ** for 'arm64' we are using eight "moonshot" sleds, codethink-sled9-15-arm64, which have 8 cores and 64gb ram each. These nodes are sponsored by link:https://jenkins.debian.net/userContent/thanks.html[Codethink]. ** To test 'armhf' we are using 27 small boards donated by vagrant@d.o: -*** four quad-cores (cbxi4a, cbxi4b, ff4a and jtx1a) with 4gb ram, +*** four quad-cores (cbxi4a, cbxi4b, ff4a, jtx1a, jtx1b) with 4gb ram, *** three octo-cores (odxu4, odxu4b and odxu4c) with 2gb ram, -*** one hexa-core (ff64a) with 2gb ram -*** twelve quad-cores (wbq0, cbxi4pro0, ff2a, ff2b, odc2a, odu3a, opi2a, opi2b, opi2c, jtk1a, p64b and p64c) with 2gb ram, +*** twelve quad-cores (wbq0, cbxi4pro0, ff2a, ff2b, odc2a, odu3a, opi2a, opi2b, opi2c, jtk1b, p64b and p64c) with 2gb ram, *** two dual-core (bbx15 and cb3a) with 2gb ram and, *** two quad-cores (rpi2b and rpi2c) with 1gb ram and *** three dual-cores (bpi0, hb0 and wbd0) with 1gb ram, each. diff --git a/bin/jenkins_node_definitions.sh b/bin/jenkins_node_definitions.sh index 46848fbb..7e10695a 100755 --- a/bin/jenkins_node_definitions.sh +++ b/bin/jenkins_node_definitions.sh @@ -28,10 +28,10 @@ codethink-sled16-arm64.debian.net ff2a-armhf-rb.debian.net ff2b-armhf-rb.debian.net ff4a-armhf-rb.debian.net -ff64a-armhf-rb.debian.net hb0-armhf-rb.debian.net -jtk1a-armhf-rb.debian.net +jtk1b-armhf-rb.debian.net jtx1a-armhf-rb.debian.net +jtx1b-armhf-rb.debian.net odc2a-armhf-rb.debian.net odu3a-armhf-rb.debian.net odxu4-armhf-rb.debian.net @@ -131,9 +131,15 @@ get_node_ssh_port() { jtk1a-armhf-rb.debian.net) PORT=2246 ;; + jtk1b-armhf-rb.debian.net) + PORT=2252 + ;; jtx1a-armhf-rb.debian.net) PORT=2249 ;; + jtx1b-armhf-rb.debian.net) + PORT=2253 + ;; p64b-armhf-rb.debian.net) PORT=2247 ;; diff --git a/bin/reproducible_build_service.sh b/bin/reproducible_build_service.sh index 0b148214..abcda366 100755 --- a/bin/reproducible_build_service.sh +++ b/bin/reproducible_build_service.sh @@ -116,7 +116,7 @@ choose_nodes() { arm64_31) NODE1=codethink-sled14-arm64 NODE2=codethink-sled11-arm64 ;; arm64_32) NODE1=codethink-sled13-arm64 NODE2=codethink-sled14-arm64 ;; # to choose new armhf jobs: - # for i in cb3a hb0 rpi2b rpi2c wbd0 bpi0 bbx15 cbxi4pro0 ff2a ff2b ff64a jtk1a odxu4 odxu4b odxu4c odu3a opi2a opi2b opi2c p64b p64c wbq0 cbxi4a cbxi4b ff4a jtx1a ; do echo "$i: " ; grep NODE1 bin/reproducible_build_service.sh|grep armhf|grep $i-armhf ; done + # for i in cb3a hb0 rpi2b rpi2c wbd0 bpi0 bbx15 cbxi4pro0 ff2a ff2b ff64a jtk1a jtk1b odxu4 odxu4b odxu4c odu3a opi2a opi2b opi2c p64b p64c wbq0 cbxi4a cbxi4b ff4a jtx1a jtx1b ; do echo "$i: " ; grep NODE1 bin/reproducible_build_service.sh|grep armhf|grep $i-armhf ; done # 6 jobs for quad-cores with 4 gb ram # 4 jobs for octo-cores with 2 gb ram # 4 jobs for hexa-cores with 2 gb ram @@ -142,22 +142,22 @@ choose_nodes() { armhf_13) NODE1=ff2a-armhf-rb NODE2=p64c-armhf-rb ;; armhf_14) NODE1=ff2b-armhf-rb NODE2=p64b-armhf-rb ;; armhf_15) NODE1=ff2b-armhf-rb NODE2=opi2c-armhf-rb ;; - armhf_16) NODE1=ff64a-armhf-rb NODE2=p64b-armhf-rb ;; - armhf_17) NODE1=ff64a-armhf-rb NODE2=opi2c-armhf-rb ;; - armhf_18) NODE1=jtk1a-armhf-rb NODE2=opi2b-armhf-rb ;; - armhf_19) NODE1=jtk1a-armhf-rb NODE2=opi2a-armhf-rb ;; + armhf_16) NODE1=jtx1b-armhf-rb NODE2=p64b-armhf-rb ;; + armhf_17) NODE1=jtx1b-armhf-rb NODE2=opi2c-armhf-rb ;; + armhf_18) NODE1=jtk1b-armhf-rb NODE2=opi2b-armhf-rb ;; + armhf_19) NODE1=jtk1b-armhf-rb NODE2=opi2a-armhf-rb ;; armhf_20) NODE1=odxu4-armhf-rb NODE2=opi2b-armhf-rb ;; armhf_21) NODE1=odxu4-armhf-rb NODE2=opi2a-armhf-rb ;; armhf_22) NODE1=odxu4b-armhf-rb NODE2=odu3a-armhf-rb ;; - armhf_23) NODE1=odxu4b-armhf-rb NODE2=jtk1a-armhf-rb ;; + armhf_23) NODE1=odxu4b-armhf-rb NODE2=jtk1b-armhf-rb ;; armhf_24) NODE1=odxu4c-armhf-rb NODE2=odu3a-armhf-rb ;; - armhf_25) NODE1=odxu4c-armhf-rb NODE2=jtk1a-armhf-rb ;; + armhf_25) NODE1=odxu4c-armhf-rb NODE2=jtk1b-armhf-rb ;; armhf_26) NODE1=odu3a-armhf-rb NODE2=odxu4-armhf-rb ;; - armhf_27) NODE1=odu3a-armhf-rb NODE2=ff64a-armhf-rb ;; + armhf_27) NODE1=odu3a-armhf-rb NODE2=jtx1b-armhf-rb ;; armhf_28) NODE1=opi2a-armhf-rb NODE2=odxu4c-armhf-rb ;; armhf_29) NODE1=opi2a-armhf-rb NODE2=cbxi4a-armhf-rb ;; armhf_30) NODE1=opi2b-armhf-rb NODE2=odxu4c-armhf-rb ;; - armhf_31) NODE1=opi2b-armhf-rb NODE2=ff64a-armhf-rb ;; + armhf_31) NODE1=opi2b-armhf-rb NODE2=jtx1b-armhf-rb ;; armhf_32) NODE1=opi2c-armhf-rb NODE2=odxu4b-armhf-rb ;; armhf_33) NODE1=opi2c-armhf-rb NODE2=ff2a-armhf-rb ;; armhf_34) NODE1=p64b-armhf-rb NODE2=odxu4b-armhf-rb ;; @@ -166,7 +166,7 @@ choose_nodes() { armhf_37) NODE1=p64c-armhf-rb NODE2=cbxi4pro0-armhf-rb ;; armhf_38) NODE1=wbq0-armhf-rb NODE2=ff2b-armhf-rb ;; armhf_39) NODE1=wbq0-armhf-rb NODE2=cbxi4pro0-armhf-rb ;; - armhf_40) NODE1=cbxi4a-armhf-rb NODE2=bbx15-armhf-rb ;; + armhf_40) NODE1=cbxi4a-armhf-rb NODE2=jtx1b-armhf-rb ;; armhf_41) NODE1=cbxi4a-armhf-rb NODE2=cb3a-armhf-rb ;; armhf_42) NODE1=cbxi4a-armhf-rb NODE2=rpi2c-armhf-rb ;; armhf_43) NODE1=cbxi4b-armhf-rb NODE2=bbx15-armhf-rb ;; @@ -178,6 +178,7 @@ choose_nodes() { armhf_49) NODE1=jtx1a-armhf-rb NODE2=cbxi4a-armhf-rb ;; armhf_50) NODE1=jtx1a-armhf-rb NODE2=rpi2b-armhf-rb ;; armhf_51) NODE1=jtx1a-armhf-rb NODE2=odxu4-armhf-rb ;; + armhf_52) NODE1=jtx1b-armhf-rb NODE2=bbx15-armhf-rb ;; *) NODE1=undefined ;; esac @@ -59,10 +59,10 @@ cbxi4pro0-armhf-rb.debian.net ff2a-armhf-rb.debian.net ff2b-armhf-rb.debian.net ff4a-armhf-rb.debian.net -ff64a-armhf-rb.debian.net hb0-armhf-rb.debian.net -jtk1a-armhf-rb.debian.net +jtk1b-armhf-rb.debian.net jtx1a-armhf-rb.debian.net +jtx1b-armhf-rb.debian.net odc2a-armhf-rb.debian.net odu3a-armhf-rb.debian.net odxu4-armhf-rb.debian.net diff --git a/hosts/jenkins/etc/munin/munin.conf b/hosts/jenkins/etc/munin/munin.conf index 8719cf9c..f74eb562 100644 --- a/hosts/jenkins/etc/munin/munin.conf +++ b/hosts/jenkins/etc/munin/munin.conf @@ -516,6 +516,16 @@ contact.me.command mail -s "Munin notification ${var:host}" root diskstats_utilization.graph no diskstats_iops.graph no +[jtk1b-armhf-rb.debian.net] + address ssh://jenkins@jtk1b-armhf-rb.debian.net:2252/bin/nc localhost 4949 + use_node_name yes + diskstats_latency.sda.avgwrwait.warning 0:20 + diskstats_latency.sda.avgrdwait.warning 0:10 + diskstats_latency.graph no + diskstats_throughput.graph no + diskstats_utilization.graph no + diskstats_iops.graph no + [jtx1a-armhf-rb.debian.net] address ssh://jenkins@jtx1a-armhf-rb.debian.net:2249/bin/nc localhost 4949 use_node_name yes @@ -526,6 +536,16 @@ contact.me.command mail -s "Munin notification ${var:host}" root diskstats_utilization.graph no diskstats_iops.graph no +[jtx1b-armhf-rb.debian.net] + address ssh://jenkins@jtx1b-armhf-rb.debian.net:2253/bin/nc localhost 4949 + use_node_name yes + diskstats_latency.sda.avgwrwait.warning 0:20 + diskstats_latency.sda.avgrdwait.warning 0:10 + diskstats_latency.graph no + diskstats_throughput.graph no + diskstats_utilization.graph no + diskstats_iops.graph no + [odc2a-armhf-rb.debian.net] address ssh://jenkins@odc2a-armhf-rb.debian.net:2251/bin/nc localhost 4949 use_node_name yes @@ -709,5 +729,5 @@ contact.me.command mail -s "Munin notification ${var:host}" root # node_order Totals fii.foo.com fay.foo.com # [debian.net;] - node_order jenkins.debian.net profitbricks-build1-amd64.debian.net profitbricks-build2-i386.debian.net profitbricks-build3-amd64.debian.net profitbricks-build4-amd64.debian.net profitbricks-build5-amd64.debian.net profitbricks-build6-i386.debian.net profitbricks-build9-amd64.debian.net profitbricks-build10-amd64.debian.net profitbricks-build11-amd64.debian.net profitbricks-build12-i386.debian.net profitbricks-build15-amd64.debian.net profitbricks-build16-i386.debian.net codethink-sled9-arm64.debian.net codethink-sled10-arm64.debian.net codethink-sled11-arm64.debian.net codethink-sled12-arm64.debian.net codethink-sled13-arm64.debian.net codethink-sled14-arm64.debian.net codethink-sled15-arm64.debian.net codethink-sled16-arm64.debian.net bbx15-armhf-rb.debian.net bpi0-armhf-rb.debian.net cbxi4a-armhf-rb.debian.net cbxi4b-armhf-rb.debian.net cbxi4pro0-armhf-rb.debian.net cb3a-armhf-rb.debian.net ff2a-armhf-rb.debian.net ff2b-armhf-rb.debian.net ff4a-armhf-rb.debian.net ff64a-armhf-rb.debian.net hb0-armhf-rb.debian.net odxu4-armhf-rb.debian.net odxu4b-armhf-rb.debian.net odxu4c-armhf-rb.debian.net odu3a-armhf-rb.debian.net jtk1a-armhf-rb.debian.net jtx1a-armhf-rb.debian.net odc2a-armhf-rb.debian.net opi2a-armhf-rb.debian.net opi2b-armhf-rb.debian.net opi2c-armhf-rb.debian.net p64b-armhf-rb.debian.net p64c-armhf-rb.debian.net rpi2b-armhf-rb.debian.net rpi2c-armhf-rb.debian.net wbd0-armhf-rb.debian.net wbq0-armhf-rb.debian.net freebsd-jenkins.debian.net jenkins-test-vm.debian.net + node_order jenkins.debian.net profitbricks-build1-amd64.debian.net profitbricks-build2-i386.debian.net profitbricks-build3-amd64.debian.net profitbricks-build4-amd64.debian.net profitbricks-build5-amd64.debian.net profitbricks-build6-i386.debian.net profitbricks-build9-amd64.debian.net profitbricks-build10-amd64.debian.net profitbricks-build11-amd64.debian.net profitbricks-build12-i386.debian.net profitbricks-build15-amd64.debian.net profitbricks-build16-i386.debian.net codethink-sled9-arm64.debian.net codethink-sled10-arm64.debian.net codethink-sled11-arm64.debian.net codethink-sled12-arm64.debian.net codethink-sled13-arm64.debian.net codethink-sled14-arm64.debian.net codethink-sled15-arm64.debian.net codethink-sled16-arm64.debian.net bbx15-armhf-rb.debian.net bpi0-armhf-rb.debian.net cbxi4a-armhf-rb.debian.net cbxi4b-armhf-rb.debian.net cbxi4pro0-armhf-rb.debian.net cb3a-armhf-rb.debian.net ff2a-armhf-rb.debian.net ff2b-armhf-rb.debian.net ff4a-armhf-rb.debian.net ff64a-armhf-rb.debian.net hb0-armhf-rb.debian.net odxu4-armhf-rb.debian.net odxu4b-armhf-rb.debian.net odxu4c-armhf-rb.debian.net odu3a-armhf-rb.debian.net jtk1a-armhf-rb.debian.net jtk1b-armhf-rb.debian.net jtx1a-armhf-rb.debian.net jtx1b-armhf-rb.debian.net odc2a-armhf-rb.debian.net opi2a-armhf-rb.debian.net opi2b-armhf-rb.debian.net opi2c-armhf-rb.debian.net p64b-armhf-rb.debian.net p64c-armhf-rb.debian.net rpi2b-armhf-rb.debian.net rpi2c-armhf-rb.debian.net wbd0-armhf-rb.debian.net wbq0-armhf-rb.debian.net freebsd-jenkins.debian.net jenkins-test-vm.debian.net diff --git a/hosts/jtk1b b/hosts/jtk1b new file mode 120000 index 00000000..37aff3c8 --- /dev/null +++ b/hosts/jtk1b @@ -0,0 +1 @@ +jtk1b-armhf-rb/
\ No newline at end of file diff --git a/hosts/jtk1b-armhf-rb/etc/apt/apt.conf.d/10no-package-descriptions b/hosts/jtk1b-armhf-rb/etc/apt/apt.conf.d/10no-package-descriptions new file mode 100644 index 00000000..2318f84e --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/apt/apt.conf.d/10no-package-descriptions @@ -0,0 +1 @@ +Acquire::Languages "none"; diff --git a/hosts/jtk1b-armhf-rb/etc/apt/apt.conf.d/80proxy b/hosts/jtk1b-armhf-rb/etc/apt/apt.conf.d/80proxy new file mode 100644 index 00000000..9e738254 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/apt/apt.conf.d/80proxy @@ -0,0 +1,2 @@ +Acquire::http::Proxy "http://10.0.0.15:8000/"; + diff --git a/hosts/jtk1b-armhf-rb/etc/apt/listchanges.conf b/hosts/jtk1b-armhf-rb/etc/apt/listchanges.conf new file mode 100644 index 00000000..8b598c0a --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/apt/listchanges.conf @@ -0,0 +1,6 @@ +[apt] +frontend=mail +email_address=root +confirm=0 +save_seen=/var/lib/apt/listchanges.db +which=both diff --git a/hosts/jtk1b-armhf-rb/etc/apt/sources.list b/hosts/jtk1b-armhf-rb/etc/apt/sources.list new file mode 100644 index 00000000..a26fb33d --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/apt/sources.list @@ -0,0 +1,11 @@ +deb http://deb.debian.org/debian/ stretch main contrib non-free +#deb-src http://deb.debian.org/debian/ stretch main contrib non-free + +deb http://deb.debian.org/debian/ stretch-updates main contrib non-free +#deb-src http://deb.debian.org/debian/ stretch-updates main contrib non-free + +deb http://security.debian.org/ stretch/updates main contrib non-free +#deb-src http://security.debian.org/ stretch/updates main contrib non-free + +deb http://deb.debian.org/debian/ stretch-backports main contrib non-free +#deb-src http://deb.debian.org/debian/ stretch-backports main contrib non-free diff --git a/hosts/jtk1b-armhf-rb/etc/cron.d/dsa b/hosts/jtk1b-armhf-rb/etc/cron.d/dsa new file mode 100755 index 00000000..9be64c36 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/cron.d/dsa @@ -0,0 +1,10 @@ +# m h dom mon dow (0|7=sun,1=mon) command + +# +# cron-jobs for jenkins.debian.net and nodes +# + +MAILTO=root + +0 1,13 * * * nobody /usr/bin/chronic /usr/local/bin/dsa-check-running-kernel +2 1,13 * * * nobody /usr/bin/chronic /usr/local/bin/dsa-check-packages diff --git a/hosts/jtk1b-armhf-rb/etc/logrotate.d/jenkins.debian.net-update_jdn b/hosts/jtk1b-armhf-rb/etc/logrotate.d/jenkins.debian.net-update_jdn new file mode 100644 index 00000000..c1f207a7 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/logrotate.d/jenkins.debian.net-update_jdn @@ -0,0 +1,9 @@ +/var/log/jenkins/update_jdn.log { + daily + copytruncate + missingok + rotate 7 + compress + delaycompress + notifempty +} diff --git a/hosts/jtk1b-armhf-rb/etc/munin/munin-node.conf b/hosts/jtk1b-armhf-rb/etc/munin/munin-node.conf new file mode 100644 index 00000000..c986ea07 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/munin/munin-node.conf @@ -0,0 +1,66 @@ +# +# Example config-file for munin-node +# + +log_level 4 +log_file /var/log/munin/munin-node.log +pid_file /var/run/munin/munin-node.pid + +background 1 +setsid 1 + +user root +group root + +# This is the timeout for the whole transaction. +# Units are in sec. Default is 15 min +# +# global_timeout 900 + +# This is the timeout for each plugin. +# Units are in sec. Default is 1 min +# +# timeout 60 + +# Regexps for files to ignore +ignore_file [\#~]$ +ignore_file DEADJOE$ +ignore_file \.bak$ +ignore_file %$ +ignore_file \.dpkg-(tmp|new|old|dist)$ +ignore_file \.rpm(save|new)$ +ignore_file \.pod$ + +# Set this if the client doesn't report the correct hostname when +# telnetting to localhost, port 4949 +# +#host_name localhost.localdomain + +# A list of addresses that are allowed to connect. This must be a +# regular expression, since Net::Server does not understand CIDR-style +# network notation unless the perl module Net::CIDR is installed. You +# may repeat the allow line as many times as you'd like + +allow ^127\.0\.0\.1$ +allow ^::1$ + +# If you have installed the Net::CIDR perl module, you can use one or more +# cidr_allow and cidr_deny address/mask patterns. A connecting client must +# match any cidr_allow, and not match any cidr_deny. Note that a netmask +# *must* be provided, even if it's /32 +# +# Example: +# +# cidr_allow 127.0.0.1/32 +# cidr_allow 192.0.2.0/24 +# cidr_deny 192.0.2.42/32 + +# Which address to bind to; +host * +# host 127.0.0.1 + +# And which port +port 4949 + +allow ^78\.137\.96\.196 +hostname jtk1b-armhf-rb.debian.net diff --git a/hosts/jtk1b-armhf-rb/etc/munin/plugin-conf.d/df b/hosts/jtk1b-armhf-rb/etc/munin/plugin-conf.d/df new file mode 100644 index 00000000..b3fdadcb --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/munin/plugin-conf.d/df @@ -0,0 +1,6 @@ +[df*] +env.exclude none unknown iso9660 squashfs udf romfs ramfs debugfs devtmpfs sysfs +env.exclude_re /srv/workspace/pbuilder /run /dev/disk/by /var/lib/schroot/mount /srv/workspace/varlibschroot /dev/shm /sys/fs/cgroup +env.warning 92 +env.critical 98 + diff --git a/hosts/jtk1b-armhf-rb/etc/munin/plugin-conf.d/diskstats b/hosts/jtk1b-armhf-rb/etc/munin/plugin-conf.d/diskstats new file mode 100644 index 00000000..2d11f397 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/munin/plugin-conf.d/diskstats @@ -0,0 +1,4 @@ +[diskstats] +env.trim_labels yes +env.include_only /dev/sda + diff --git a/hosts/jtk1b-armhf-rb/etc/munin/plugin-conf.d/munin-node b/hosts/jtk1b-armhf-rb/etc/munin/plugin-conf.d/munin-node new file mode 100644 index 00000000..e766928f --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/munin/plugin-conf.d/munin-node @@ -0,0 +1,117 @@ +# This file is used to configure how the plugins are invoked. +# Place in /etc/munin/plugin-conf.d/ or corresponding directory. +# +# PLEASE NOTE: Changes in the plugin-conf.d directory are only +# read at munin-node startup, so restart at any changes. +# +# user <user> # Set the user to run the plugin as. +# group <group> # Set the group to run the plugin as. +# command <command> # Run <command> instead of the plugin. %c expands to +# what would normally be run. +# env.<variable> <value> # Sets <variable> in the plugin's environment, see the +# individual plugins to find out which variables they +# care about. + + +[amavis] +group adm +env.MUNIN_MKTEMP /bin/mktemp -p /tmp/ $1 +env.amavislog /var/log/mail.info + +[apt] +user root + +[courier_mta_mailqueue] +group daemon + +[courier_mta_mailstats] +group adm + +[courier_mta_mailvolume] +group adm + +[cps*] +user root + +[exim_mailqueue] +group adm, (Debian-exim) + +[exim_mailstats] +group adm, (Debian-exim) +env.logdir /var/log/exim4/ +env.logname mainlog + +[fw_conntrack] +user root + +[fw_forwarded_local] +user root + +[hddtemp_smartctl] +user root + +[hddtemp2] +user root + +[if_*] +user root + +[if_err_*] +user nobody + +[ip_*] +user root + +[ipmi_*] +user root + +[mysql*] +user root +env.mysqlopts --defaults-file=/etc/mysql/debian.cnf +env.mysqluser debian-sys-maint +env.mysqlconnection DBI:mysql:mysql;mysql_read_default_file=/etc/mysql/debian.cnf + +[postfix_mailqueue] +user postfix + +[postfix_mailstats] +group adm + +[postfix_mailvolume] +group adm +env.logfile mail.log + +[smart_*] +user root + +[vlan*] +user root + +[ejabberd*] +user ejabberd +env.statuses available away chat xa +env.days 1 7 30 + +[dhcpd3] +user root +env.leasefile /var/lib/dhcp3/dhcpd.leases +env.configfile /etc/dhcp3/dhcpd.conf + +[jmx_*] +env.ip 127.0.0.1 +env.port 5400 + +[samba] +user root + +[munin_stats] +user munin +group munin + +[postgres_*] +user postgres +env.PGUSER postgres +env.PGPORT 5432 + +[fail2ban] +user root diff --git a/hosts/jtk1b-armhf-rb/etc/munin/plugins/jenkins_reproducible_builds b/hosts/jtk1b-armhf-rb/etc/munin/plugins/jenkins_reproducible_builds new file mode 100755 index 00000000..e990c127 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/munin/plugins/jenkins_reproducible_builds @@ -0,0 +1,56 @@ +#!/bin/sh +# -*- sh -*- + +: << =cut + +=head1 NAME + +jenkins_reproducible_builds - Plugin to measure number of reproducible builds running + +=head1 AUTHOR + +Contributed by Holger Levsen + +=head1 LICENSE + +GPLv2 + +=head1 MAGIC MARKERS + + #%# family=auto + #%# capabilities=autoconf + +=cut + +. $MUNIN_LIBDIR/plugins/plugin.sh + +if [ "$1" = "autoconf" ]; then + echo yes + exit 0 +fi + +JOB_PREFIXES="first second" +if [ "$1" = "config" ]; then + echo 'graph_title Concurrent reproducible builds running' + echo 'graph_args --base 1000 -l 0 ' + echo 'graph_scale no' + echo 'graph_total total' + echo 'graph_vlabel Concurrent reproducible builds running' + echo 'graph_category jenkins' + draw=AREA + for PREFIX in $JOB_PREFIXES ; do + echo "jenkins_reproducible_${PREFIX}_build.label $PREFIX build" + echo "jenkins_reproducible_${PREFIX}_build.draw $draw" + if [ "$draw" = "AREA" ] ; then draw=STACK ; fi + done + exit 0 +fi + +for PREFIX in $JOB_PREFIXES ; do + if [ "$PREFIX" = "first" ] ; then + NR=$(pgrep -fc "bin/bash /srv/jenkins/bin/reproducible_build.sh 1") + else + NR=$(pgrep -fc "bin/bash /srv/jenkins/bin/reproducible_build.sh 2") + fi + echo "jenkins_reproducible_${PREFIX}_build.value $NR" + done diff --git a/hosts/jtk1b-armhf-rb/etc/pbuilder/rebuild-hooks/B01_cleanup b/hosts/jtk1b-armhf-rb/etc/pbuilder/rebuild-hooks/B01_cleanup new file mode 100755 index 00000000..334e03a2 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/pbuilder/rebuild-hooks/B01_cleanup @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +BUILDDIR="${BUILDDIR:-/tmp/buildd}" + +# exit if we are in the same UTS namespace as init ( != 2nd build ) +[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0 + +# cease using disorderfs +if [ -d /tmp/disorderfs ] ; then + echo -n "Unmounting /tmp/disorderfs…" + fusermount -z -u "$BUILDDIR" + rmdir "$BUILDDIR" + mv /tmp/disorderfs "$BUILDDIR" + echo " done." +fi diff --git a/hosts/jtk1b-armhf-rb/etc/pbuilder/rebuild-hooks/C01_cleanup b/hosts/jtk1b-armhf-rb/etc/pbuilder/rebuild-hooks/C01_cleanup new file mode 100755 index 00000000..47719ae7 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/pbuilder/rebuild-hooks/C01_cleanup @@ -0,0 +1,21 @@ +#!/bin/sh + +set -e + +BUILDDIR="${BUILDDIR:-/tmp/buildd}" + +echo "debug output: disk usage on $(hostname) at $(date -u)" +df -h +echo + +# exit if we are in the same UTS namespace as init ( != 2nd build ) +[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0 + +# cease using disorderfs +if [ -d /tmp/disorderfs ] ; then + echo -n "Unmounting /tmp/disorderfs…" + fusermount -z -u "$BUILDDIR" + rmdir "$BUILDDIR" + mv /tmp/disorderfs "$BUILDDIR" + echo " done." +fi diff --git a/hosts/jtk1b-armhf-rb/etc/pbuilder/rebuild-hooks/D01_modify_environment b/hosts/jtk1b-armhf-rb/etc/pbuilder/rebuild-hooks/D01_modify_environment new file mode 100755 index 00000000..4b4c9ab3 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/pbuilder/rebuild-hooks/D01_modify_environment @@ -0,0 +1,56 @@ +#!/bin/sh + +set -e + +BUILDDIR="${BUILDDIR:-/tmp/buildd}" + +# exit if we are in the same UTS namespace as init ( != 2nd build ) +[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0 + +# +# doing variations for the 2nd builds: +# +REAL_HOSTNAME=$(hostname) +echo "disorder-fs debug: Running on $REAL_HOSTNAME." + +echo "I: Changing host+domainname to test build reproducibility" >&2 +sed -e '/^127.0.0.1/s/$/ i-capture-the-hostname i-capture-the-hostname.i-capture-the-domain/' -i /etc/hosts +hostname i-capture-the-hostname +domainname i-capture-the-domain +echo "I: Adding a custom variable just for the fun of it..." >&2 +export CAPTURE_ENVIRONMENT="I capture the environment" + +echo "I: Changing /bin/sh to bash" >&2 +echo "dash dash/sh boolean false" | debconf-set-selections +DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash +if [ -n "$BUILDUSERNAME" ] ; then + echo "I: Setting $BUILDUSERNAME's login shell to /bin/bash" >&2 + usermod -s /bin/bash $BUILDUSERNAME + if [ -n "$BUILDUSERGECOS" ] ; then + echo "I: Setting $BUILDUSERNAME's GECOS to $BUILDUSERGECOS" >&2 + usermod -c "$BUILDUSERGECOS" $BUILDUSERNAME + fi +fi + +# disable disorderfs due to #844498 +#exit 0 +# use disorderfs on armhf only for now +if [ "$(dpkg --print-architecture)" != "armhf" ] ; then + exit 0 +elif [ "$REAL_HOSTNAME" != "bpi0" ] ; then + exit 0 +else + echo "disorder-fs debug: bpi0, yay, enabling disorderfs now…" +fi + +# use disorderfs +if [ -x /usr/bin/disorderfs ] ; then + echo -n "Moving $BUILDDIR to /tmp/disorderfs and mounting this as $BUILDDIR via the fuse disorderfs…" + mknod -m 666 /dev/fuse c 10 229 + mv "$BUILDDIR" /tmp/disorderfs + mkdir "$BUILDDIR" + disorderfs --multi-user=yes /tmp/disorderfs "$BUILDDIR" + echo " done." +else + echo "Warning: disorderfs not available." +fi diff --git a/hosts/jtk1b-armhf-rb/etc/pbuilderrc b/hosts/jtk1b-armhf-rb/etc/pbuilderrc new file mode 100644 index 00000000..d97c552b --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/pbuilderrc @@ -0,0 +1,53 @@ +# this is your configuration file for pbuilder. +# the file in /usr/share/pbuilder/pbuilderrc is the default template. +# /etc/pbuilderrc is the one meant for overwriting defaults in +# the default template +# +# read pbuilderrc.5 document for notes on specific options. +# +# cater for different locations +case $HOSTNAME in + jenkins|jenkins-test-vm|profitbricks-build*) + MIRRORSITE=http://deb.debian.org/debian ;; + bbx15|bpi0|cb3*|cbxi4*|hb0|wbq0|odxu4*|odu3*|odc*|wbd0|rpi2*|ff*|ff4*|opi2*|jt?1*|p64*) + + MIRRORSITE=http://deb.debian.org/debian ;; + codethink*) + MIRRORSITE=http://deb.debian.org/debian ;; + *) + echo "unsupported host, exiting." ; exit 1 ;; +esac +# only use eatmydata on armhf+i386 - on amd64+arm64 we build in tmpfs anyway +case $HOSTNAME in + profitbricks-build*i386) + EATMYDATA=yes ;; + bbx15|bpi0|cb3*|cbxi4*|hb0|wbq0|odxu4*|odu3*|odc*|wbd0|rpi2*|ff*|ff4*|opi2*|jt?1*|p64*) + EATMYDATA=yes ;; + *) ;; +esac +EXTRAPACKAGES="" # better list them in bin/reproducible_setup_pbuilder.sh +APTCACHE="" +COMPRESSPROG="pigz" +BUILDPLACE=/srv/workspace/pbuilder # build in /srv/workspace, which is in tmpfs + +# set PATH to predictable values, see #780729 and #780725 +PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" + +# used for reproducible builds tests, when doing the 2nd build +if [ "$(readlink /proc/1/ns/uts)" != "$(readlink /proc/self/ns/uts)" ]; then + PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" +fi + +# needed to ignore failures due to running 398 days in the future… +# (only on those 4 nodes running in the future…) +if [ "$HOSTNAME" = "profitbricks-build5-amd64" ] || [ "$HOSTNAME" = "profitbricks-build6-i386" ] || \ + [ "$HOSTNAME" = "profitbricks-build15-amd64" ] || [ "$HOSTNAME" = "profitbricks-build16-i386" ] || \ + [ "$HOSTNAME" = "codethink-sled9-arm64" ] || [ "$HOSTNAME" = "codethink-sled11-arm64" ] || \ + [ "$HOSTNAME" = "codethink-sled13-arm64" ] || [ "$HOSTNAME" = "codethink-sled15-arm64" ] ; then + case "$PBUILDER_OPERATION" in + create) + APTGETOPT=(-o Acquire::Check-Valid-Until="false") + ;; + *) ;; + esac +fi diff --git a/hosts/jtk1b-armhf-rb/etc/postfix/main.cf b/hosts/jtk1b-armhf-rb/etc/postfix/main.cf new file mode 100644 index 00000000..5434f975 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/postfix/main.cf @@ -0,0 +1,41 @@ +# See /usr/share/postfix/main.cf.dist for a commented, more complete version + + +# Debian specific: Specifying a file name will cause the first +# line of that file to be used as the name. The Debian default +# is /etc/mailname. +#myorigin = /etc/mailname + +smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) +biff = no + +# appending .domain is the MUA's job. +append_dot_mydomain = no + +# Uncomment the next line to generate "delayed mail" warnings +#delay_warning_time = 4h + +readme_directory = no + +# TLS parameters +smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem +smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key +smtpd_use_tls=yes +smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache +smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache + +# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for +# information on enabling SSL in the smtp client. + +myhostname = jtk1b-armhf-rb.debian.net +alias_maps = hash:/etc/aliases +alias_database = hash:/etc/aliases +myorigin = /etc/mailname +mydestination = jtk1b-armhf-rb.debian.net, localhost +relayhost = mail.holgerlevsen.de +#mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 +mynetworks = 127.0.0.0/8 +mailbox_size_limit = 0 +recipient_delimiter = + +inet_interfaces = all +mailbox_command = /usr/bin/procmail -a "$EXTENSION" diff --git a/hosts/jtk1b-armhf-rb/etc/rc.local b/hosts/jtk1b-armhf-rb/etc/rc.local new file mode 100755 index 00000000..aa6bfefc --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/rc.local @@ -0,0 +1,105 @@ +#!/bin/sh -e +# +# rc.local +# +# This script is executed at the end of each multiuser runlevel. +# Make sure that the script will "exit 0" on success or any other +# value on error. + +set -x + +send_back_to_the_future() { + # stop ntp + service ntp stop || true + # disable systemd date services (and don't fail if systemd ain't running) + systemctl disable systemd-timesyncd || true + systemctl disable systemd-timedated || true + systemctl disable ntp.service || true + systemctl stop systemd-timesyncd || true + systemctl stop systemd-timedated || true + systemctl stop ntp.service || true + # set correct date + ntpdate -b de.pool.ntp.org + # set fake date + date --set="+398 days +6 hours + 23 minutes" + # finally report success + echo "$(date -u) - system is running in the future now" | mail -s "$(hostname -f) in the future" root +} + +put_schroots_on_tmpfs() { + # keep schroot sessions on tmpfs + [ -L /var/lib/schroot ] || echo "$(date -u) - /var/lib/schroot is not a link (to /srv/workspace/varlibschroot/) as it should, please fix manually" + mkdir -p /srv/workspace/varlibschroot + cd /srv/workspace/varlibschroot || exit 1 + mkdir -p mount session union unpack + mkdir -p union/overlay union/underlay +} + +fixup_shm() { + # this is always harmless + chmod 1777 /dev/shm +} + +fixup_mtu() { + # only act on systems which have eth0 + if ip link show eth0 >/dev/null 2>&1 ; then + # if MTU != 1500 set it to 1500 + if [ -z "$(ip link show eth0 | grep 'eth0:' | grep 'mtu 1500 ' || true)" ] ; then + ip link set dev eth0 mtu 1500 + fi + ip link show eth0 + fi +} + +# +# init, notify about reboots +# +MESSAGE="$(hostname -f) rebooted" +echo "$(date -u) - system was rebooted" | mail -s "$MESSAGE" root + +# +# notify jenkins reboots on irc +# +if [ "$(hostname)" = "jenkins" ] ; then + for channel in debian-qa debian-reproducible ; do + kgb-client --conf /srv/jenkins/kgb/$channel.conf --relay-msg "$MESSAGE" + done +fi + +# +# fixup /(dev|run)/shm if needed +# +fixup_shm + +# +# fixup eth0's MTU if needed +fixup_mtu + +# +# put schroots on tmpfs for non debian hosts +# +case $(hostname) in + profitbricks-build3*) put_schroots_on_tmpfs ;; + profitbricks-build4*) put_schroots_on_tmpfs ;; + profitbricks-build7*) put_schroots_on_tmpfs ;; + *) ;; +esac + +# +# run some hosts in the future +# +case $(hostname) in + codethink-sled9*) send_back_to_the_future ;; + codethink-sled11*) send_back_to_the_future ;; + codethink-sled13*) send_back_to_the_future ;; + codethink-sled15*) send_back_to_the_future ;; + profitbricks-build4*) send_back_to_the_future ;; + profitbricks-build5*) send_back_to_the_future ;; + profitbricks-build6*) send_back_to_the_future ;; + profitbricks-build15*) send_back_to_the_future ;; + profitbricks-build16*) send_back_to_the_future ;; + *) ;; +esac + +echo "$(date -u) - system booted up." +exit 0 diff --git a/hosts/jtk1b-armhf-rb/etc/schroot/default/fstab b/hosts/jtk1b-armhf-rb/etc/schroot/default/fstab new file mode 100644 index 00000000..74468dd2 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/schroot/default/fstab @@ -0,0 +1,24 @@ +# fstab: static file system information for chroots. +# Note that the mount point will be prefixed by the chroot path +# (CHROOT_PATH) +# +# <file system> <mount point> <type> <options> <dump> <pass> +/proc /proc none rw,bind 0 0 +/sys /sys none rw,bind 0 0 +/dev /dev none rw,bind 0 0 +/dev/pts /dev/pts none rw,bind 0 0 +/home /home none rw,bind 0 0 +/tmp /tmp none rw,bind 0 0 +/srv/reproducible-results /srv/reproducible-results none rw,bind 0 0 +/srv/d-i /srv/d-i none rw,bind 0 0 +/srv/jenkins /srv/jenkins none rw,bind 0 0 +/srv/live-build /srv/live-build none rw,bind 0 0 + +# It may be desirable to have access to /run, especially if you wish +# to run additional services in the chroot. However, note that this +# may potentially cause undesirable behaviour on upgrades, such as +# killing services on the host. +#/run /run none rw,bind 0 0 +#/run/lock /run/lock none rw,bind 0 0 +/dev/shm /dev/shm none rw,bind 0 0 +/run/shm /run/shm none rw,bind 0 0 diff --git a/hosts/jtk1b-armhf-rb/etc/schroot/default/nssdatabases b/hosts/jtk1b-armhf-rb/etc/schroot/default/nssdatabases new file mode 100644 index 00000000..72615e5d --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/schroot/default/nssdatabases @@ -0,0 +1,11 @@ +# System databases to copy into the chroot from the host system. +# +# <database name> +passwd +shadow +group +gshadow +#services +protocols +networks +hosts diff --git a/hosts/jtk1b-armhf-rb/etc/sudoers.d/jenkins b/hosts/jtk1b-armhf-rb/etc/sudoers.d/jenkins new file mode 100644 index 00000000..d249be94 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/sudoers.d/jenkins @@ -0,0 +1,51 @@ +jenkins ALL= \ + NOPASSWD: /usr/sbin/debootstrap *, \ + /usr/bin/tee /schroots/*, \ + /usr/bin/tee -a /schroots/*, \ + /usr/bin/tee /etc/schroot/chroot.d/jenkins*, \ + /bin/chmod +x /schroots/*, \ + /usr/sbin/chroot /schroots/*, \ + /usr/sbin/chroot /chroots/*, \ + /usr/sbin/chroot /media/*, \ + /bin/ls -la /media/*, \ + /bin/rm -rf --one-file-system /chroots/*, \ + /bin/rm -rf --one-file-system /schroots/*, \ + /bin/rm -rf --one-file-system /srv/live-build/*, \ + /bin/rm -rf --one-file-system /srv/workspace/pbuilder/*, \ + /bin/cp -v *.iso /srv/live-build/results/*, \ + /bin/mv /chroots/* /schroots/*, \ + /bin/mv /schroots/* /schroots/*, \ + /bin/umount -l /chroots/*, \ + /bin/umount -l /schroots/*, \ + /bin/umount -l /media/*, \ + /bin/rmdir /media/*, \ + /bin/mount -o loop*, \ + /bin/mount --bind *, \ + /usr/bin/du *, \ + /bin/kill *, \ + /usr/bin/file *, \ + /bin/dd if=/dev/zero of=/dev/jenkins*, \ + /usr/bin/qemu-system-x86_64 *, \ + /usr/bin/qemu-img *, \ + /sbin/lvcreate *, /sbin/lvremove *, \ + /bin/mkdir -p /media/*, \ + /usr/bin/guestmount *, \ + /bin/cp -rv /media/*, \ + /bin/chown -R jenkins\:jenkins /var/lib/jenkins/jobs/*,\ + SETENV: NOPASSWD: /usr/sbin/pbuilder *, \ + SETENV: NOPASSWD: /usr/bin/timeout -k ??.?h ??h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder *, \ + SETENV: NOPASSWD: /usr/bin/timeout -k ??.?h ??h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder *, \ + /bin/mv /var/cache/pbuilder/*base*.tgz /var/cache/pbuilder/*base*.tgz, \ + /bin/rm /var/cache/pbuilder/*base*.tgz, \ + /bin/rm -v /var/cache/pbuilder/*base*.tgz, \ + /bin/rm /var/cache/pbuilder/result/*, \ + /usr/bin/dcmd rm *.changes, \ + /usr/bin/dcmd rm *.dsc, \ + /usr/bin/apt-get update, \ + /usr/bin/killall timeout, \ + /usr/sbin/slay 1111, \ + /usr/sbin/slay 2222, \ + /usr/sbin/slay jenkins + +# keep these environment variables +Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/jtk1b-armhf-rb/etc/sudoers.d/jenkins-adm b/hosts/jtk1b-armhf-rb/etc/sudoers.d/jenkins-adm new file mode 100644 index 00000000..3c357be2 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/sudoers.d/jenkins-adm @@ -0,0 +1,7 @@ +# allow member of the jenkins-adm group to sudo-to the jenkins-adm user (owner +# of jenkins script) and the jenkins user itself +%jenkins-adm ALL=(jenkins:jenkins) NOPASSWD: ALL +%jenkins-adm ALL=(jenkins-adm:jenkins-adm) NOPASSWD: ALL +# allow jenkins-adm to run everything as root +%jenkins-adm ALL= NOPASSWD: ALL + diff --git a/hosts/jtk1b-armhf-rb/etc/systemd/system/rc-local.service.d/local.conf b/hosts/jtk1b-armhf-rb/etc/systemd/system/rc-local.service.d/local.conf new file mode 100644 index 00000000..91ed832a --- /dev/null +++ b/hosts/jtk1b-armhf-rb/etc/systemd/system/rc-local.service.d/local.conf @@ -0,0 +1,4 @@ +[Unit] +After=network-online.target +Wants=network-online.target + diff --git a/hosts/jtk1b-armhf-rb/usr/local/bin/dsa-check-packages b/hosts/jtk1b-armhf-rb/usr/local/bin/dsa-check-packages new file mode 100755 index 00000000..28844e5a --- /dev/null +++ b/hosts/jtk1b-armhf-rb/usr/local/bin/dsa-check-packages @@ -0,0 +1,362 @@ +#!/usr/bin/perl + +# dsa-check-packages + +# checks for obsolete/local and upgradeable packages. +# +# packages for the obsolete/local check can be ignored, by +# listing their full name in /etc/nagios/obsolete-packages-ignore +# or by having a regex (starting a line with "/") that matches +# the packagename in said file. +# +# Takes one optional argument, the location of the ignore file. + + +# Copyright (C) 2008, 2009 Peter Palfrader <peter@palfrader.org> +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +use strict; +use warnings; +use English; + +my $IGNORE = "/etc/nagios/obsolete-packages-ignore"; +my $IGNORED = "/etc/nagios/obsolete-packages-ignore.d"; + +my %CODE = ( + 'OK' => 0, + 'WARNING' => 1, + 'CRITICAL' => 2, + 'UNKNOWN' => 3 +); +my $EXITCODE = 'OK'; +sub record($) { + my ($newexit) = @_; + die "code $newexit not defined\n" unless defined $CODE{$newexit}; + + if ($CODE{$newexit} > $CODE{$EXITCODE}) { + $EXITCODE = $newexit; + }; +} + + + +sub get_packages { + $ENV{'COLUMNS'} = 1000; + $ENV{'LC_ALL'} = 'C'; + open(F, "dpkg -l|") or die ("Cannot run dpkg: $!\n"); + my @lines = <F>; + close(F); + chomp(@lines); + + my $line; + my $has_arch = 0; + while (defined($line = shift @lines) && ($line !~ /\+\+\+/)) { + if ($line =~ /Architecture/) { $has_arch = 1; } + } + + my %pkgs; + for $line (@lines) { + my ($state, $pkg, $version, $arch, undef) = split(/ */, $line); + $arch = '' unless $has_arch; + $pkgs{$state}{$pkg} = { 'installed' => $version, arch => $arch } + } + + my $installed = $pkgs{'ii'}; + delete $pkgs{'ii'}; + + my @installed_packages = keys(%$installed); + my @cmd = ("apt-cache", "policy", @installed_packages); + + open my $olderr, ">&STDERR" or die "Can't dup STDERR: $!"; + open STDERR, ">/dev/null" or die "Can't dup STDOUT: $!"; + open (F, "-|", @cmd) or die ("Cannot run apt-cache policy: $!\n"); + @lines = <F>; + close(F); + open STDERR, ">&", $olderr or die "Can't dup OLDERR: $!"; + chomp(@lines); + + my $pkgname = undef; + my $candidate_found = 0; + while (defined($line = shift @lines)) { + if ($line =~ /^([^ ]*):$/) { + # when we have multi-arch capable fu, we require that + # apt-cache policy output is in the same order as its + # arguments. + # + # We needs thi, because the output block in apt-cache + # policy does not show the arch: + # + # | weasel@stanley:~$ apt-cache policy libedit2:amd64 + # | libedit2: + # | Installed: 2.11-20080614-5 + # | Candidate: 2.11-20080614-5 + # + # We replace the package name in the output with the + # one we asked for ($pkg:$arch) - but to match this up + # sanely we need the order to be correct. + # + # For squeeze systems (no m-a), apt-cache policy output + # is all different. + $pkgname = $1; + $candidate_found = 0; + if ($has_arch) { + my $from_list = shift @installed_packages; + next if ($pkgname eq $from_list); # no :$arch in pkgname we asked for + + my $ma_fix_pkgname = $pkgname.':'.$installed->{$from_list}->{'arch'}; + my $ma_fix_from_list = $from_list.':'.$installed->{$from_list}->{'arch'}; + + if ($pkgname eq $ma_fix_from_list || # e.g. ia32-libs-i386. dpkg -l: ia32-libs-i386, apt-cache policy: ia32-libs-i386:i386 + $ma_fix_pkgname eq $from_list) { + $pkgname = $from_list; + } else { + die "Unexpected order mismatch in apt-cache policy output (apt-cache policy name: $pkgname - dpkg -l name: $from_list)\n"; + } + } + } elsif ($line =~ /^ +Installed: (.*)$/) { + # etch dpkg -l does not print epochs, so use this info, it's better + $installed->{$pkgname}{'installed'} = $1; + # initialize security-update + $installed->{$pkgname}{'security-update'} = 0; + } elsif ($line =~ /^ +Candidate: (.*)$/) { + $installed->{$pkgname}{'candidate'} = $1; + } elsif ($line =~ / ([^ ]+) [0-9]+/) { + # check if the next lines show the sources of our candidate + if ($1 eq $installed->{$pkgname}{'candidate'}) { + $candidate_found = 1; + } + } elsif (($line =~ / +[0-9]+ [^ ]+\/(security\.([^ ]+\.)?debian\.org|debian-security).*\/updates\//) && $candidate_found ) { + $installed->{$pkgname}{'security-update'} = 1; + } elsif ($line =~ /^ +\*\*\*/) { + $line = shift @lines; + my @l = split(/ +/, $line); + $installed->{$pkgname}{'origin'} = $l[2]; + $candidate_found = 0; + } + } + + my (%current, %obsolete, %outofdate, %security_outofdate); + for my $pkgname (keys %$installed) { + my $pkg = $installed->{$pkgname}; + + unless (defined($pkg->{'candidate'}) && defined($pkg->{'origin'})) { + $obsolete{$pkgname} = $pkg; + next; + } + + if ($pkg->{'candidate'} ne $pkg->{'installed'}) { + if ($pkg->{'security-update'}) { + $security_outofdate{$pkgname} = $pkg; + } else { + $outofdate{$pkgname} = $pkg; + } + next; + }; + if ($pkg->{'origin'} eq '/var/lib/dpkg/status') { + $obsolete{$pkgname} = $pkg; + next; + } + $current{$pkgname} = $pkg; + } + + $pkgs{'current'} = \%current; + $pkgs{'outofdate'} = \%outofdate; + $pkgs{'security_outofdate'} = \%security_outofdate; + $pkgs{'obsolete'} = \%obsolete; + return \%pkgs; +} + +sub load_ignores { + my ($ignorefiles, $require_file) = @_; + + my @ignores; + + for my $ignoreitem (@$ignorefiles) { + next if (!$require_file and ! -e $ignoreitem); + + my @filestoopen; + if (-d $ignoreitem) { + opendir(DIR, $ignoreitem) or die ("Cannot open dir $ignoreitem: $!\n"); + @filestoopen = readdir(DIR); + closedir(DIR); + + @filestoopen = grep { -f ($ignoreitem.'/'.$_) } @filestoopen; + @filestoopen = grep { /^([a-z0-9_.-]+)+[a-z0-9]+$/i } @filestoopen; + @filestoopen = grep { !/dpkg-(old|dist|new|tmp)$/ } @filestoopen; + @filestoopen = map { ($ignoreitem.'/'.$_) } @filestoopen; + } else { + push @filestoopen, $ignoreitem; + } + + for my $f (@filestoopen) { + open (F, "< $f") or die ("Cannot open $f: $!\n"); + push @ignores, <F>; + close F; + } + } + chomp(@ignores); + return \@ignores; +} + +sub check_ignore { + my ($pkg, $ignores) = @_; + + my $ignore_this = 0; + for my $ignore (@$ignores) { + my $ig = $ignore; + return 1 if ($ig eq $pkg); + if (substr($ig,0,1) eq '/') { + substr($ig, 0, 1, ''); + $ig =~ s,/$,,; + return 1 if ($pkg =~ /$ig/); + } + } + return 0 +} + +sub filter_ignored { + my ($packages, $ignores) = @_; + + my $obs = $packages->{'obsolete'}; + + my (%ignored, %bad); + for my $pkg (keys %$obs) { + if (check_ignore($pkg, $ignores)) { + $ignored{$pkg} = $obs->{$pkg}; + } else { + $bad{$pkg} = $obs->{$pkg}; + }; + } + delete $packages->{'obsolete'}; + $packages->{'obsolete'} = \%bad; + $packages->{'obsolete-ignored'} = \%ignored; +}; + +sub usage { + my ($fd, $exit) = @_; + print $fd "Usage: $PROGRAM_NAME [<ignorefile|dir> [<ignorefile|dir> ...]]\n"; + exit $exit; +} + +my $ignorefiles = [$IGNORE, $IGNORED]; +my $ignorefile_userset = 0; +if (@ARGV >= 1) { + usage(\*STDOUT, 0) if ($ARGV[0] eq "-h"); + usage(\*STDOUT, 0) if ($ARGV[0] eq "--help"); + $ignorefile_userset = 1; + $ignorefiles = \@ARGV; +}; + +my $ignores = load_ignores($ignorefiles, $ignorefile_userset); +my $packages = get_packages(); + +filter_ignored($packages, $ignores); + + + +my @reportform = ( + { 'key' => 'obsolete', + 'listpackages' => 1, + 'long' => "%d local or obsolete packages: %s", + 'short' => "%d obs/loc", + 'perf' => "obs_loc=%d;1;5;0", + 'status' => 'WARNING' }, + { 'key' => 'outofdate', + 'listpackages' => 1, + 'long' => "%d out of date packages: %s", + 'short' => "%d updates", + 'perf' => "outdated=%d;1;5;0", + 'status' => 'WARNING' }, + { 'key' => 'current', + 'listpackages' => 0, + 'long' => "%d packages current.", + 'short' => "%d ok", + 'perf' => "current=%d;;;0", + 'status' => 'OK' }, + { 'key' => 'obsolete-ignored', + 'listpackages' => 1, + 'long' => "%d whitelisted local or obsolete packages: %s", + 'short' => "%d obs/loc(ignored)", + 'perf' => "obs_ign=%d;;;0", + 'status' => 'OK' }, + { 'key' => 'rc', + 'listpackages' => 1, + 'long' => "%d packages removed but not purged: %s", + 'short' => "%d rc", + 'perf' => "rm_unprg=%d;;;0", + 'status' => 'OK' }, + { 'key' => 'hi', + 'listpackages' => 1, + 'long' => "%d packages on hold: %s", + 'short' => "%d hi", + 'perf' => "hold=%d;;;0", + 'status' => 'OK' }, + { 'key' => 'pc', + 'listpackages' => 1, + 'long' => "%d packages requested to be purged but conffiles still installed: %s", + 'short' => "%d pc", + 'perf' => "prg_conf=%d;1;;0", + 'status' => 'WARNING' }, + { 'key' => 'security_outofdate', + 'listpackages' => 1, + 'long' => "%d packages with outstanding security updates: %s", + 'short' => "%d security-updates", + 'perf' => "security_outdated=%d;;1;0", + 'status' => 'CRITICAL' }, + ); + +my @longout; +my @perfout; +my @shortout; +for my $form (@reportform) { + my $pkgs = $packages->{$form->{'key'}}; + delete $packages->{$form->{'key'}}; + my $num = scalar keys %$pkgs; + push @perfout, sprintf($form->{'perf'}, $num); + next unless ($num > 0); + if ($form->{'listpackages'}) { + my $list = join(", ", keys %$pkgs); + push @longout, sprintf($form->{'long'}, $num, $list); + } else { + push @longout, sprintf($form->{'long'}, $num); + }; + push @shortout, sprintf($form->{'short'}, $num); + record($form->{'status'}); +}; +if (scalar keys %$packages) { + record('WARNING'); + unshift @shortout, "unk: ".join(", ", keys %$packages); + for my $status (sort {$b cmp $a} keys %$packages) { + my $pkgs = $packages->{$status}; + my $list = join(", ", keys %$pkgs); + unshift @longout, "Unknown package status $status: $list"; + }; +} + +my $shortout = $EXITCODE.": ".join(", ", @shortout); +my $longout = join("\n", @longout); +my $perfout = "|".join(" ", @perfout); + +print $shortout,"\n"; +print $longout,"\n"; +print $perfout,"\n"; + +exit $CODE{$EXITCODE}; diff --git a/hosts/jtk1b-armhf-rb/usr/local/bin/dsa-check-running-kernel b/hosts/jtk1b-armhf-rb/usr/local/bin/dsa-check-running-kernel new file mode 100755 index 00000000..aa6e35bd --- /dev/null +++ b/hosts/jtk1b-armhf-rb/usr/local/bin/dsa-check-running-kernel @@ -0,0 +1,254 @@ +#!/bin/bash + +# Check if the running kernel has the same version string as the on-disk +# kernel image. + +# Copyright 2008,2009,2011,2012,2013,2014 Peter Palfrader +# Copyright 2009 Stephen Gran +# Copyright 2010,2012,2013 Uli Martens +# Copyright 2011 Alexander Reichle-Schmehl +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +OK=0; +WARNING=1; +CRITICAL=2; +UNKNOWN=3; + +get_offset() { + local file needle + + file="$1" + needle="$2" + + perl -e ' + undef $/; + $i = 0; $k=<>; + while (($i = index($k, "'"$needle"'", $i)) >= 0) { + print $i++,"\n"; + }; ' < "$file" +} + +get_avail() { + # This is wrong, but leaves room for when we have to care for machines running + # myfirstunix-image-0.1-dsa-arm + local prefix="$1"; shift + + local kervers=$(uname -r) + + local metavers='' + + # DSA uses kernel versions of the form 2.6.29.3-dsa-dl380-oldxeon, where + # Debian uses versions of the form 2.6.29-2-amd64 + if [ "${kervers#2}" != "$kervers" ]; then + if [ "${kervers//dsa}" != "$kervers" ]; then + metavers=$(echo $kervers | sed -r -e 's/^2\.(4|6)\.[0-9]+([\.0-9]+?)-(.*)/2.\1-\3/') + else + metavers=$(echo $kervers | sed -r -e 's/^2\.(4|6)\.[0-9]+-[A-Za-z0-9\.]+-(.*)/2.\1-\2/') + fi + else + metavers=$(echo $kervers | sed -r -e 's/^[0-9]+\.[0-9]+(\.[0-9])?+-[A-Za-z0-9\.]+-(.*)/\2/') + fi + + # Attempt to track back to a metapackage failed. bail + if [ "$metavers" = "$kervers" ]; then + return 2 + fi + + # We're just going to give up if we can't find a matching metapackage + # I tried being strict once, and it just caused a lot of headaches. We'll see how + # being lax does for us + + local output=$(apt-cache policy ${prefix}-image-${metavers} 2>/dev/null) + local metaavailvers=$(echo "$output" | grep '^ Candidate:' | awk '{print $2}') + local metainstavers=$(echo "$output" | grep '^ Installed:' | awk '{print $2}') + + if [ -z "$metaavailvers" ] || [ "$metaavailvers" = '(none)' ]; then + return 2 + fi + if [ -z "$metainstavers" ] || [ "$metainstavers" = '(none)' ]; then + return 2 + fi + + if [ "$metaavailvers" != "$metainstavers" ] ; then + echo "${prefix}-image-${metavers} $metaavailvers available but $metainstavers installed" + return 1 + fi + + local imagename=0 + # --no-all-versions show shows only the candidate + for vers in $(apt-cache --no-all-versions show ${prefix}-image-${metavers} | sed -n 's/^Depends: //p' | tr ',' '\n' | tr -d ' ' | grep ${prefix}-image | awk '{print $1}' | sort -u); do + if dpkg --compare-versions "1.$vers" gt "1.$imagename"; then + imagename=$vers + fi + done + + if [ -z "$imagename" ] || [ "$imagename" = 0 ]; then + return 2 + fi + + if [ "$imagename" != "${prefix}-image-${kervers}" ]; then + if dpkg --compare-versions 1."$imagename" lt 1."${prefix}-image-${kervers}"; then + return 2 + fi + echo "$imagename" != "${prefix}-image-${kervers}" + return 1 + fi + + local availvrs=$(apt-cache policy ${imagename} 2>/dev/null | grep '^ Candidate' | awk '{print $2}') + local kernelversion=$(apt-cache policy ${prefix}-image-${kervers} 2>/dev/null | grep '^ Installed:' | awk '{print $2}') + + if [ "$availvrs" = "$kernelversion" ]; then + return 0 + fi + + echo "$kernelversion != $availvrs" + return 1 +} + +cat_vmlinux() { + local image header filter hdroff + + image="$1" + header="$2" + filter="$3" + hdroff="$4" + + get_offset "$image" $header | head -n 5 | while read off; do + (if [ "$off" != 0 ]; then + dd ibs="$((off+hdroff))" skip=1 count=0 + fi && + dd bs=512k) < "$image" 2>/dev/null | $filter 2>/dev/null + done +} + +get_image_linux() { + local image + + image="$1" + + # gzip compressed image + cat_vmlinux "$image" "\x1f\x8b\x08\x00" "zcat" 0 + cat_vmlinux "$image" "\x1f\x8b\x08\x08" "zcat" 0 + # lzma compressed image + cat_vmlinux "$image" "\x00\x00\x00\x02\xff" "xzcat" -1 + cat_vmlinux "$image" "\x00\x00\x00\x04\xff" "xzcat" -1 + # xz compressed image + cat_vmlinux "$image" "\xfd\x37\x7a\x58\x5a " "xzcat" 0 + + echo "ERROR: Unable to extract kernel image." 2>&1 + exit 1 +} + + +freebsd_check_running_version() { + local imagefile="$1"; shift + + local r="$(uname -r)" + local v="$(uname -v| sed -e 's/^#[0-9]*/&:/')" + + local q='@(#)FreeBSD '"$r $v" + + if zcat "$imagefile" | $STRINGS | grep -F -q "$q"; then + echo "OK" + else + echo "not OK" + fi +} + +STRINGS=""; +if [ -x "$(which strings)" ]; then + STRINGS="$(which strings)" +elif [ -x "$(which busybox)" -a "$( echo foobar | $(which busybox) strings 2>/dev/null)" = "foobar" ]; then + STRINGS="$(which busybox) strings" +fi + +searched="" +for on_disk in \ + "/boot/vmlinuz-`uname -r`"\ + "/boot/vmlinux-`uname -r`"\ + "/boot/kfreebsd-`uname -r`.gz"; do + + if [ -e "$on_disk" ]; then + if [ -z "$STRINGS" ]; then + echo "UNKNOWN: 'strings' command missing, perhaps install binutils or busybox?" + exit $UNKNOWN + fi + if [ "${on_disk/vmlinu}" != "$on_disk" ]; then + on_disk_version="`get_image_linux "$on_disk" | $STRINGS | grep 'Linux version' | head -n1`" + if [ -x /usr/bin/lsb_release ] ; then + vendor=$(lsb_release -i -s) + if [ -n "$vendor" ] && [ "xDebian" != "x$vendor" ] ; then + on_disk_version=$( echo $on_disk_version|sed -e "s/ ($vendor [[:alnum:]\.-]\+ [[:alnum:]\.]\+)//") + fi + fi + [ -z "$on_disk_version" ] || break + on_disk_version="`cat "$on_disk" | $STRINGS | grep 'Linux version' | head -n1`" + [ -z "$on_disk_version" ] || break + + echo "UNKNOWN: Failed to get a version string from image $on_disk" + exit $UNKNOWN + else + on_disk_version="$(zcat $on_disk | $STRINGS | grep Debian | head -n 1 | sed -e 's/Debian [[:alnum:]]\+ (\(.*\))/\1/')" + fi + fi + searched="$searched $on_disk" +done + +if ! [ -e "$on_disk" ]; then + echo "WARNING: Did not find a kernel image (checked$searched) - I have no idea which kernel I am running" + exit $WARNING +fi + +if [ "$(uname -s)" = "Linux" ]; then + running_version="`cat /proc/version`" + if [ -z "$running_version" ] ; then + echo "UNKNOWN: Failed to get a version string from running system" + exit $UNKNOWN + fi + + if [ "$running_version" != "$on_disk_version" ]; then + echo "WARNING: Running kernel does not match on-disk kernel image: [$running_version != $on_disk_version]" + exit $WARNING + fi + + ret="$(get_avail linux)" + if [ $? = 1 ]; then + echo "WARNING: Kernel needs upgrade [$ret]" + exit $WARNING + fi +else + image_current=$(freebsd_check_running_version $on_disk) + running_version="`uname -s` `uname -r` `uname -v`" + if [ "$image_current" != "OK" ]; then + approx_time="$(date -d "@`stat -c '%Y' "$on_disk"`" +"%Y-%m-%d %H:%M:%S")" + echo "WARNING: Currently running kernel ($running_version) does not match on disk image (~ $approx_time)" + exit $WARNING; + fi + + ret="$(get_avail linux)" + if [ $? = 1 ]; then + echo "WARNING: Kernel needs upgrade [$ret]" + exit $WARNING + fi +fi + +echo "OK: Running kernel matches on disk image: [$running_version]" +exit $OK diff --git a/hosts/jtk1b-armhf-rb/usr/local/sbin/nagios-check-libs b/hosts/jtk1b-armhf-rb/usr/local/sbin/nagios-check-libs new file mode 100755 index 00000000..77b37805 --- /dev/null +++ b/hosts/jtk1b-armhf-rb/usr/local/sbin/nagios-check-libs @@ -0,0 +1,204 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2005, 2006, 2007, 2008, 2012, 2015 Peter Palfrader <peter@palfrader.org> +# 2012 Uli Martens <uli@youam.net> +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +use strict; +use English; +use Getopt::Long; + +$ENV{'PATH'} = '/bin:/sbin:/usr/bin:/usr/sbin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; + +my $LSOF = '/usr/bin/lsof -F0'; +my $VERSION = '0.2015012901'; + +# nagios exit codes +my $OK = 0; +my $WARNING = 1; +my $CRITICAL = 2; +my $UNKNOWN = 3; + +my $params; +my $config; + +Getopt::Long::config('bundling'); + +sub dief { + print STDERR @_; + exit $UNKNOWN; +} + +if (!GetOptions ( + '--help' => \$params->{'help'}, + '--version' => \$params->{'version'}, + '--quiet' => \$params->{'quiet'}, + '--verbose' => \$params->{'verbose'}, + '-v' => \$params->{'verbose'}, + '--config=s' => \$params->{'config'}, + )) { + dief ("$PROGRAM_NAME: Usage: $PROGRAM_NAME [--help|--version] [--verbose] [--quiet] [--config=<CONFIGFILE>]\n"); +}; +if ($params->{'help'}) { + print "$PROGRAM_NAME: Usage: $PROGRAM_NAME [--help|--version] [--verbose] [--quiet] [--config=<CONFIGFILE>]\n"; + print "Reports processes that are linked against libraries that no longer exist.\n"; + print "The optional config file can specify ignore rules - see the sample config file.\n"; + exit (0); +}; +if ($params->{'version'}) { + print "nagios-check-libs $VERSION\n"; + print "nagios check for availability of debian (security) updates\n"; + print "Copyright (c) 2005, 2006, 2007, 2008, 2012 Peter Palfrader <peter\@palfrader.org>\n"; + exit (0); +}; + +if (! defined $params->{'config'}) { + $params->{'config'} = '/etc/nagios/check-libs.conf'; +} elsif (! -e $params->{'config'}) { + dief("Config file $params->{'config'} does not exist.\n"); +} + +if (-e $params->{'config'}) { + eval "use YAML::Syck; 1" or dief "you need YAML::Syck (libyaml-syck-perl) to load a config file"; + open(my $fh, '<', $params->{'config'}) or dief "Cannot open config file $params->{'config'}: $!"; + $config = LoadFile($fh); + close($fh); + if (!(ref($config) eq "HASH")) { + dief("Loaded config is not a hash!\n"); + } +} else { + $config = { + 'ignorelist' => [ + '$path =~ m#^/proc/#', + '$path =~ m#^/var/tmp/#', + '$path =~ m#^/SYS#', + '$path =~ m#^/drm$# # xserver stuff', + '$path =~ m#^/dev/zero#', + '$path =~ m#^/dev/shm/#', + ] + }; +} + +if (! exists $config->{'ignorelist'}) { + $config->{'ignorelist'} = []; +} elsif (! (ref($config->{'ignorelist'}) eq 'ARRAY')) { + dief("Config->ignorelist is not an array!\n"); +} + + +my %processes; + +sub getPIDs($$) { + my ($user, $process) = @_; + return join(', ', sort keys %{ $processes{$user}->{$process} }); +}; +sub getProcs($) { + my ($user) = @_; + + return join(', ', map { $_.' ('.getPIDs($user, $_).')' } (sort {$a cmp $b} keys %{ $processes{$user} })); +}; +sub getUsers() { + return join('; ', (map { $_.': '.getProcs($_) } (sort {$a cmp $b} keys %processes))); +}; +sub inVserver() { + my ($f, $key); + if (-e "/proc/self/vinfo" ) { + $f = "/proc/self/vinfo"; + $key = "XID"; + } else { + $f = "/proc/self/status"; + $key = "s_context"; + }; + open(F, "< $f") or return 0; + while (<F>) { + my ($k, $v) = split(/: */, $_, 2); + if ($k eq $key) { + close F; + return ($v > 0); + }; + }; + close F; + return 0; +} + +my $INVSERVER = inVserver(); + +print STDERR "Running $LSOF -n\n" if $params->{'verbose'}; +open (LSOF, "$LSOF -n|") or dief ("Cannot run $LSOF -n: $!\n"); +my @lsof=<LSOF>; +close LSOF; +if ($CHILD_ERROR) { # program failed + dief("$LSOF -n returned with non-zero exit code: ".($CHILD_ERROR / 256)."\n"); +}; + +my ($process, $pid, $user); +LINE: for my $line (@lsof) { + if ( $line =~ /^p/ ) { + my %fields = map { m/^(.)(.*)$/ ; $1 => $2 } grep { defined $_ and length $_ >1} split /\0/, $line; + $process = $fields{c}; + $pid = $fields{p}; + $user = $fields{L}; + next; + } + + unless ( $line =~ /^f/ ) { + dief("UNKNOWN strange line read from lsof\n"); + # don't print it because it contains NULL characters... + } + + my %fields = map { m/^(.)(.*)$/ ; $1 => $2 } grep { defined $_ and length $_ >1} split /\0/, $line; + + my $fd = $fields{f}; + my $inode = $fields{i}; + my $path = $fields{n}; + if ($path =~ m/\.dpkg-/ || $path =~ m/\(deleted\)/ || $path =~ /path inode=/ || $path =~ m#/\.nfs# || $fd eq 'DEL') { + my $deleted_in_path = ($path =~ m/\(deleted\)/ || $path =~ m/\.nfs/); + next if ($deleted_in_path && $fd =~ /^[0-9]*$/); # Ignore deleted files that are open via normal file handles. + next if ($deleted_in_path && $fd eq 'cwd'); # Ignore deleted directories that we happen to be in. + + $path =~ s/^\(deleted\)//; # in some cases "(deleted)" is at the beginning of the string + for my $i (@{$config->{'ignorelist'}}) { + my $ignore = eval($i); + next LINE if $ignore; + } + next if ($INVSERVER && ($process eq 'init') && ($pid == 1) && ($user eq 'root')); + if ( $params->{'verbose'} ) { + print STDERR "adding $process($pid) because of [$path]:\n"; + print STDERR $line; + } + $processes{$user}->{$process}->{$pid} = 1; + }; +}; + + + +my $message=''; +my $exit = $OK; +if (keys %processes) { + $exit = $WARNING; + $message = 'The following processes have libs linked that were upgraded: '. getUsers()."\n"; +} else { + $message = "No upgraded libs linked in running processes\n" unless $params->{'quiet'}; +}; + +print $message; +exit $exit; diff --git a/hosts/jtx1b b/hosts/jtx1b new file mode 120000 index 00000000..a78c377c --- /dev/null +++ b/hosts/jtx1b @@ -0,0 +1 @@ +jtx1b-armhf-rb/
\ No newline at end of file diff --git a/hosts/jtx1b-armhf-rb/etc/apt/apt.conf.d/10no-package-descriptions b/hosts/jtx1b-armhf-rb/etc/apt/apt.conf.d/10no-package-descriptions new file mode 100644 index 00000000..2318f84e --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/apt/apt.conf.d/10no-package-descriptions @@ -0,0 +1 @@ +Acquire::Languages "none"; diff --git a/hosts/jtx1b-armhf-rb/etc/apt/apt.conf.d/80proxy b/hosts/jtx1b-armhf-rb/etc/apt/apt.conf.d/80proxy new file mode 100644 index 00000000..9e738254 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/apt/apt.conf.d/80proxy @@ -0,0 +1,2 @@ +Acquire::http::Proxy "http://10.0.0.15:8000/"; + diff --git a/hosts/jtx1b-armhf-rb/etc/apt/listchanges.conf b/hosts/jtx1b-armhf-rb/etc/apt/listchanges.conf new file mode 100644 index 00000000..8b598c0a --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/apt/listchanges.conf @@ -0,0 +1,6 @@ +[apt] +frontend=mail +email_address=root +confirm=0 +save_seen=/var/lib/apt/listchanges.db +which=both diff --git a/hosts/jtx1b-armhf-rb/etc/apt/sources.list b/hosts/jtx1b-armhf-rb/etc/apt/sources.list new file mode 100644 index 00000000..a26fb33d --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/apt/sources.list @@ -0,0 +1,11 @@ +deb http://deb.debian.org/debian/ stretch main contrib non-free +#deb-src http://deb.debian.org/debian/ stretch main contrib non-free + +deb http://deb.debian.org/debian/ stretch-updates main contrib non-free +#deb-src http://deb.debian.org/debian/ stretch-updates main contrib non-free + +deb http://security.debian.org/ stretch/updates main contrib non-free +#deb-src http://security.debian.org/ stretch/updates main contrib non-free + +deb http://deb.debian.org/debian/ stretch-backports main contrib non-free +#deb-src http://deb.debian.org/debian/ stretch-backports main contrib non-free diff --git a/hosts/jtx1b-armhf-rb/etc/cron.d/dsa b/hosts/jtx1b-armhf-rb/etc/cron.d/dsa new file mode 100755 index 00000000..9be64c36 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/cron.d/dsa @@ -0,0 +1,10 @@ +# m h dom mon dow (0|7=sun,1=mon) command + +# +# cron-jobs for jenkins.debian.net and nodes +# + +MAILTO=root + +0 1,13 * * * nobody /usr/bin/chronic /usr/local/bin/dsa-check-running-kernel +2 1,13 * * * nobody /usr/bin/chronic /usr/local/bin/dsa-check-packages diff --git a/hosts/jtx1b-armhf-rb/etc/logrotate.d/jenkins.debian.net-update_jdn b/hosts/jtx1b-armhf-rb/etc/logrotate.d/jenkins.debian.net-update_jdn new file mode 100644 index 00000000..c1f207a7 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/logrotate.d/jenkins.debian.net-update_jdn @@ -0,0 +1,9 @@ +/var/log/jenkins/update_jdn.log { + daily + copytruncate + missingok + rotate 7 + compress + delaycompress + notifempty +} diff --git a/hosts/jtx1b-armhf-rb/etc/munin/munin-node.conf b/hosts/jtx1b-armhf-rb/etc/munin/munin-node.conf new file mode 100644 index 00000000..2ebe8e40 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/munin/munin-node.conf @@ -0,0 +1,66 @@ +# +# Example config-file for munin-node +# + +log_level 4 +log_file /var/log/munin/munin-node.log +pid_file /var/run/munin/munin-node.pid + +background 1 +setsid 1 + +user root +group root + +# This is the timeout for the whole transaction. +# Units are in sec. Default is 15 min +# +# global_timeout 900 + +# This is the timeout for each plugin. +# Units are in sec. Default is 1 min +# +# timeout 60 + +# Regexps for files to ignore +ignore_file [\#~]$ +ignore_file DEADJOE$ +ignore_file \.bak$ +ignore_file %$ +ignore_file \.dpkg-(tmp|new|old|dist)$ +ignore_file \.rpm(save|new)$ +ignore_file \.pod$ + +# Set this if the client doesn't report the correct hostname when +# telnetting to localhost, port 4949 +# +#host_name localhost.localdomain + +# A list of addresses that are allowed to connect. This must be a +# regular expression, since Net::Server does not understand CIDR-style +# network notation unless the perl module Net::CIDR is installed. You +# may repeat the allow line as many times as you'd like + +allow ^127\.0\.0\.1$ +allow ^::1$ + +# If you have installed the Net::CIDR perl module, you can use one or more +# cidr_allow and cidr_deny address/mask patterns. A connecting client must +# match any cidr_allow, and not match any cidr_deny. Note that a netmask +# *must* be provided, even if it's /32 +# +# Example: +# +# cidr_allow 127.0.0.1/32 +# cidr_allow 192.0.2.0/24 +# cidr_deny 192.0.2.42/32 + +# Which address to bind to; +host * +# host 127.0.0.1 + +# And which port +port 4949 + +allow ^78\.137\.96\.196 +hostname jtx1b-armhf-rb.debian.net diff --git a/hosts/jtx1b-armhf-rb/etc/munin/plugin-conf.d/df b/hosts/jtx1b-armhf-rb/etc/munin/plugin-conf.d/df new file mode 100644 index 00000000..b3fdadcb --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/munin/plugin-conf.d/df @@ -0,0 +1,6 @@ +[df*] +env.exclude none unknown iso9660 squashfs udf romfs ramfs debugfs devtmpfs sysfs +env.exclude_re /srv/workspace/pbuilder /run /dev/disk/by /var/lib/schroot/mount /srv/workspace/varlibschroot /dev/shm /sys/fs/cgroup +env.warning 92 +env.critical 98 + diff --git a/hosts/jtx1b-armhf-rb/etc/munin/plugin-conf.d/diskstats b/hosts/jtx1b-armhf-rb/etc/munin/plugin-conf.d/diskstats new file mode 100644 index 00000000..2d11f397 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/munin/plugin-conf.d/diskstats @@ -0,0 +1,4 @@ +[diskstats] +env.trim_labels yes +env.include_only /dev/sda + diff --git a/hosts/jtx1b-armhf-rb/etc/munin/plugin-conf.d/munin-node b/hosts/jtx1b-armhf-rb/etc/munin/plugin-conf.d/munin-node new file mode 100644 index 00000000..e766928f --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/munin/plugin-conf.d/munin-node @@ -0,0 +1,117 @@ +# This file is used to configure how the plugins are invoked. +# Place in /etc/munin/plugin-conf.d/ or corresponding directory. +# +# PLEASE NOTE: Changes in the plugin-conf.d directory are only +# read at munin-node startup, so restart at any changes. +# +# user <user> # Set the user to run the plugin as. +# group <group> # Set the group to run the plugin as. +# command <command> # Run <command> instead of the plugin. %c expands to +# what would normally be run. +# env.<variable> <value> # Sets <variable> in the plugin's environment, see the +# individual plugins to find out which variables they +# care about. + + +[amavis] +group adm +env.MUNIN_MKTEMP /bin/mktemp -p /tmp/ $1 +env.amavislog /var/log/mail.info + +[apt] +user root + +[courier_mta_mailqueue] +group daemon + +[courier_mta_mailstats] +group adm + +[courier_mta_mailvolume] +group adm + +[cps*] +user root + +[exim_mailqueue] +group adm, (Debian-exim) + +[exim_mailstats] +group adm, (Debian-exim) +env.logdir /var/log/exim4/ +env.logname mainlog + +[fw_conntrack] +user root + +[fw_forwarded_local] +user root + +[hddtemp_smartctl] +user root + +[hddtemp2] +user root + +[if_*] +user root + +[if_err_*] +user nobody + +[ip_*] +user root + +[ipmi_*] +user root + +[mysql*] +user root +env.mysqlopts --defaults-file=/etc/mysql/debian.cnf +env.mysqluser debian-sys-maint +env.mysqlconnection DBI:mysql:mysql;mysql_read_default_file=/etc/mysql/debian.cnf + +[postfix_mailqueue] +user postfix + +[postfix_mailstats] +group adm + +[postfix_mailvolume] +group adm +env.logfile mail.log + +[smart_*] +user root + +[vlan*] +user root + +[ejabberd*] +user ejabberd +env.statuses available away chat xa +env.days 1 7 30 + +[dhcpd3] +user root +env.leasefile /var/lib/dhcp3/dhcpd.leases +env.configfile /etc/dhcp3/dhcpd.conf + +[jmx_*] +env.ip 127.0.0.1 +env.port 5400 + +[samba] +user root + +[munin_stats] +user munin +group munin + +[postgres_*] +user postgres +env.PGUSER postgres +env.PGPORT 5432 + +[fail2ban] +user root diff --git a/hosts/jtx1b-armhf-rb/etc/munin/plugins/jenkins_reproducible_builds b/hosts/jtx1b-armhf-rb/etc/munin/plugins/jenkins_reproducible_builds new file mode 100755 index 00000000..e990c127 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/munin/plugins/jenkins_reproducible_builds @@ -0,0 +1,56 @@ +#!/bin/sh +# -*- sh -*- + +: << =cut + +=head1 NAME + +jenkins_reproducible_builds - Plugin to measure number of reproducible builds running + +=head1 AUTHOR + +Contributed by Holger Levsen + +=head1 LICENSE + +GPLv2 + +=head1 MAGIC MARKERS + + #%# family=auto + #%# capabilities=autoconf + +=cut + +. $MUNIN_LIBDIR/plugins/plugin.sh + +if [ "$1" = "autoconf" ]; then + echo yes + exit 0 +fi + +JOB_PREFIXES="first second" +if [ "$1" = "config" ]; then + echo 'graph_title Concurrent reproducible builds running' + echo 'graph_args --base 1000 -l 0 ' + echo 'graph_scale no' + echo 'graph_total total' + echo 'graph_vlabel Concurrent reproducible builds running' + echo 'graph_category jenkins' + draw=AREA + for PREFIX in $JOB_PREFIXES ; do + echo "jenkins_reproducible_${PREFIX}_build.label $PREFIX build" + echo "jenkins_reproducible_${PREFIX}_build.draw $draw" + if [ "$draw" = "AREA" ] ; then draw=STACK ; fi + done + exit 0 +fi + +for PREFIX in $JOB_PREFIXES ; do + if [ "$PREFIX" = "first" ] ; then + NR=$(pgrep -fc "bin/bash /srv/jenkins/bin/reproducible_build.sh 1") + else + NR=$(pgrep -fc "bin/bash /srv/jenkins/bin/reproducible_build.sh 2") + fi + echo "jenkins_reproducible_${PREFIX}_build.value $NR" + done diff --git a/hosts/jtx1b-armhf-rb/etc/pbuilder/rebuild-hooks/B01_cleanup b/hosts/jtx1b-armhf-rb/etc/pbuilder/rebuild-hooks/B01_cleanup new file mode 100755 index 00000000..334e03a2 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/pbuilder/rebuild-hooks/B01_cleanup @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +BUILDDIR="${BUILDDIR:-/tmp/buildd}" + +# exit if we are in the same UTS namespace as init ( != 2nd build ) +[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0 + +# cease using disorderfs +if [ -d /tmp/disorderfs ] ; then + echo -n "Unmounting /tmp/disorderfs…" + fusermount -z -u "$BUILDDIR" + rmdir "$BUILDDIR" + mv /tmp/disorderfs "$BUILDDIR" + echo " done." +fi diff --git a/hosts/jtx1b-armhf-rb/etc/pbuilder/rebuild-hooks/C01_cleanup b/hosts/jtx1b-armhf-rb/etc/pbuilder/rebuild-hooks/C01_cleanup new file mode 100755 index 00000000..47719ae7 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/pbuilder/rebuild-hooks/C01_cleanup @@ -0,0 +1,21 @@ +#!/bin/sh + +set -e + +BUILDDIR="${BUILDDIR:-/tmp/buildd}" + +echo "debug output: disk usage on $(hostname) at $(date -u)" +df -h +echo + +# exit if we are in the same UTS namespace as init ( != 2nd build ) +[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0 + +# cease using disorderfs +if [ -d /tmp/disorderfs ] ; then + echo -n "Unmounting /tmp/disorderfs…" + fusermount -z -u "$BUILDDIR" + rmdir "$BUILDDIR" + mv /tmp/disorderfs "$BUILDDIR" + echo " done." +fi diff --git a/hosts/jtx1b-armhf-rb/etc/pbuilder/rebuild-hooks/D01_modify_environment b/hosts/jtx1b-armhf-rb/etc/pbuilder/rebuild-hooks/D01_modify_environment new file mode 100755 index 00000000..4b4c9ab3 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/pbuilder/rebuild-hooks/D01_modify_environment @@ -0,0 +1,56 @@ +#!/bin/sh + +set -e + +BUILDDIR="${BUILDDIR:-/tmp/buildd}" + +# exit if we are in the same UTS namespace as init ( != 2nd build ) +[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0 + +# +# doing variations for the 2nd builds: +# +REAL_HOSTNAME=$(hostname) +echo "disorder-fs debug: Running on $REAL_HOSTNAME." + +echo "I: Changing host+domainname to test build reproducibility" >&2 +sed -e '/^127.0.0.1/s/$/ i-capture-the-hostname i-capture-the-hostname.i-capture-the-domain/' -i /etc/hosts +hostname i-capture-the-hostname +domainname i-capture-the-domain +echo "I: Adding a custom variable just for the fun of it..." >&2 +export CAPTURE_ENVIRONMENT="I capture the environment" + +echo "I: Changing /bin/sh to bash" >&2 +echo "dash dash/sh boolean false" | debconf-set-selections +DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash +if [ -n "$BUILDUSERNAME" ] ; then + echo "I: Setting $BUILDUSERNAME's login shell to /bin/bash" >&2 + usermod -s /bin/bash $BUILDUSERNAME + if [ -n "$BUILDUSERGECOS" ] ; then + echo "I: Setting $BUILDUSERNAME's GECOS to $BUILDUSERGECOS" >&2 + usermod -c "$BUILDUSERGECOS" $BUILDUSERNAME + fi +fi + +# disable disorderfs due to #844498 +#exit 0 +# use disorderfs on armhf only for now +if [ "$(dpkg --print-architecture)" != "armhf" ] ; then + exit 0 +elif [ "$REAL_HOSTNAME" != "bpi0" ] ; then + exit 0 +else + echo "disorder-fs debug: bpi0, yay, enabling disorderfs now…" +fi + +# use disorderfs +if [ -x /usr/bin/disorderfs ] ; then + echo -n "Moving $BUILDDIR to /tmp/disorderfs and mounting this as $BUILDDIR via the fuse disorderfs…" + mknod -m 666 /dev/fuse c 10 229 + mv "$BUILDDIR" /tmp/disorderfs + mkdir "$BUILDDIR" + disorderfs --multi-user=yes /tmp/disorderfs "$BUILDDIR" + echo " done." +else + echo "Warning: disorderfs not available." +fi diff --git a/hosts/jtx1b-armhf-rb/etc/pbuilderrc b/hosts/jtx1b-armhf-rb/etc/pbuilderrc new file mode 100644 index 00000000..d97c552b --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/pbuilderrc @@ -0,0 +1,53 @@ +# this is your configuration file for pbuilder. +# the file in /usr/share/pbuilder/pbuilderrc is the default template. +# /etc/pbuilderrc is the one meant for overwriting defaults in +# the default template +# +# read pbuilderrc.5 document for notes on specific options. +# +# cater for different locations +case $HOSTNAME in + jenkins|jenkins-test-vm|profitbricks-build*) + MIRRORSITE=http://deb.debian.org/debian ;; + bbx15|bpi0|cb3*|cbxi4*|hb0|wbq0|odxu4*|odu3*|odc*|wbd0|rpi2*|ff*|ff4*|opi2*|jt?1*|p64*) + + MIRRORSITE=http://deb.debian.org/debian ;; + codethink*) + MIRRORSITE=http://deb.debian.org/debian ;; + *) + echo "unsupported host, exiting." ; exit 1 ;; +esac +# only use eatmydata on armhf+i386 - on amd64+arm64 we build in tmpfs anyway +case $HOSTNAME in + profitbricks-build*i386) + EATMYDATA=yes ;; + bbx15|bpi0|cb3*|cbxi4*|hb0|wbq0|odxu4*|odu3*|odc*|wbd0|rpi2*|ff*|ff4*|opi2*|jt?1*|p64*) + EATMYDATA=yes ;; + *) ;; +esac +EXTRAPACKAGES="" # better list them in bin/reproducible_setup_pbuilder.sh +APTCACHE="" +COMPRESSPROG="pigz" +BUILDPLACE=/srv/workspace/pbuilder # build in /srv/workspace, which is in tmpfs + +# set PATH to predictable values, see #780729 and #780725 +PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" + +# used for reproducible builds tests, when doing the 2nd build +if [ "$(readlink /proc/1/ns/uts)" != "$(readlink /proc/self/ns/uts)" ]; then + PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" +fi + +# needed to ignore failures due to running 398 days in the future… +# (only on those 4 nodes running in the future…) +if [ "$HOSTNAME" = "profitbricks-build5-amd64" ] || [ "$HOSTNAME" = "profitbricks-build6-i386" ] || \ + [ "$HOSTNAME" = "profitbricks-build15-amd64" ] || [ "$HOSTNAME" = "profitbricks-build16-i386" ] || \ + [ "$HOSTNAME" = "codethink-sled9-arm64" ] || [ "$HOSTNAME" = "codethink-sled11-arm64" ] || \ + [ "$HOSTNAME" = "codethink-sled13-arm64" ] || [ "$HOSTNAME" = "codethink-sled15-arm64" ] ; then + case "$PBUILDER_OPERATION" in + create) + APTGETOPT=(-o Acquire::Check-Valid-Until="false") + ;; + *) ;; + esac +fi diff --git a/hosts/jtx1b-armhf-rb/etc/postfix/main.cf b/hosts/jtx1b-armhf-rb/etc/postfix/main.cf new file mode 100644 index 00000000..406fcfe9 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/postfix/main.cf @@ -0,0 +1,41 @@ +# See /usr/share/postfix/main.cf.dist for a commented, more complete version + + +# Debian specific: Specifying a file name will cause the first +# line of that file to be used as the name. The Debian default +# is /etc/mailname. +#myorigin = /etc/mailname + +smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) +biff = no + +# appending .domain is the MUA's job. +append_dot_mydomain = no + +# Uncomment the next line to generate "delayed mail" warnings +#delay_warning_time = 4h + +readme_directory = no + +# TLS parameters +smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem +smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key +smtpd_use_tls=yes +smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache +smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache + +# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for +# information on enabling SSL in the smtp client. + +myhostname = jtx1b-armhf-rb.debian.net +alias_maps = hash:/etc/aliases +alias_database = hash:/etc/aliases +myorigin = /etc/mailname +mydestination = jtx1b-armhf-rb.debian.net, localhost +relayhost = mail.holgerlevsen.de +#mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 +mynetworks = 127.0.0.0/8 +mailbox_size_limit = 0 +recipient_delimiter = + +inet_interfaces = all +mailbox_command = /usr/bin/procmail -a "$EXTENSION" diff --git a/hosts/jtx1b-armhf-rb/etc/rc.local b/hosts/jtx1b-armhf-rb/etc/rc.local new file mode 100755 index 00000000..aa6bfefc --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/rc.local @@ -0,0 +1,105 @@ +#!/bin/sh -e +# +# rc.local +# +# This script is executed at the end of each multiuser runlevel. +# Make sure that the script will "exit 0" on success or any other +# value on error. + +set -x + +send_back_to_the_future() { + # stop ntp + service ntp stop || true + # disable systemd date services (and don't fail if systemd ain't running) + systemctl disable systemd-timesyncd || true + systemctl disable systemd-timedated || true + systemctl disable ntp.service || true + systemctl stop systemd-timesyncd || true + systemctl stop systemd-timedated || true + systemctl stop ntp.service || true + # set correct date + ntpdate -b de.pool.ntp.org + # set fake date + date --set="+398 days +6 hours + 23 minutes" + # finally report success + echo "$(date -u) - system is running in the future now" | mail -s "$(hostname -f) in the future" root +} + +put_schroots_on_tmpfs() { + # keep schroot sessions on tmpfs + [ -L /var/lib/schroot ] || echo "$(date -u) - /var/lib/schroot is not a link (to /srv/workspace/varlibschroot/) as it should, please fix manually" + mkdir -p /srv/workspace/varlibschroot + cd /srv/workspace/varlibschroot || exit 1 + mkdir -p mount session union unpack + mkdir -p union/overlay union/underlay +} + +fixup_shm() { + # this is always harmless + chmod 1777 /dev/shm +} + +fixup_mtu() { + # only act on systems which have eth0 + if ip link show eth0 >/dev/null 2>&1 ; then + # if MTU != 1500 set it to 1500 + if [ -z "$(ip link show eth0 | grep 'eth0:' | grep 'mtu 1500 ' || true)" ] ; then + ip link set dev eth0 mtu 1500 + fi + ip link show eth0 + fi +} + +# +# init, notify about reboots +# +MESSAGE="$(hostname -f) rebooted" +echo "$(date -u) - system was rebooted" | mail -s "$MESSAGE" root + +# +# notify jenkins reboots on irc +# +if [ "$(hostname)" = "jenkins" ] ; then + for channel in debian-qa debian-reproducible ; do + kgb-client --conf /srv/jenkins/kgb/$channel.conf --relay-msg "$MESSAGE" + done +fi + +# +# fixup /(dev|run)/shm if needed +# +fixup_shm + +# +# fixup eth0's MTU if needed +fixup_mtu + +# +# put schroots on tmpfs for non debian hosts +# +case $(hostname) in + profitbricks-build3*) put_schroots_on_tmpfs ;; + profitbricks-build4*) put_schroots_on_tmpfs ;; + profitbricks-build7*) put_schroots_on_tmpfs ;; + *) ;; +esac + +# +# run some hosts in the future +# +case $(hostname) in + codethink-sled9*) send_back_to_the_future ;; + codethink-sled11*) send_back_to_the_future ;; + codethink-sled13*) send_back_to_the_future ;; + codethink-sled15*) send_back_to_the_future ;; + profitbricks-build4*) send_back_to_the_future ;; + profitbricks-build5*) send_back_to_the_future ;; + profitbricks-build6*) send_back_to_the_future ;; + profitbricks-build15*) send_back_to_the_future ;; + profitbricks-build16*) send_back_to_the_future ;; + *) ;; +esac + +echo "$(date -u) - system booted up." +exit 0 diff --git a/hosts/jtx1b-armhf-rb/etc/schroot/default/fstab b/hosts/jtx1b-armhf-rb/etc/schroot/default/fstab new file mode 100644 index 00000000..74468dd2 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/schroot/default/fstab @@ -0,0 +1,24 @@ +# fstab: static file system information for chroots. +# Note that the mount point will be prefixed by the chroot path +# (CHROOT_PATH) +# +# <file system> <mount point> <type> <options> <dump> <pass> +/proc /proc none rw,bind 0 0 +/sys /sys none rw,bind 0 0 +/dev /dev none rw,bind 0 0 +/dev/pts /dev/pts none rw,bind 0 0 +/home /home none rw,bind 0 0 +/tmp /tmp none rw,bind 0 0 +/srv/reproducible-results /srv/reproducible-results none rw,bind 0 0 +/srv/d-i /srv/d-i none rw,bind 0 0 +/srv/jenkins /srv/jenkins none rw,bind 0 0 +/srv/live-build /srv/live-build none rw,bind 0 0 + +# It may be desirable to have access to /run, especially if you wish +# to run additional services in the chroot. However, note that this +# may potentially cause undesirable behaviour on upgrades, such as +# killing services on the host. +#/run /run none rw,bind 0 0 +#/run/lock /run/lock none rw,bind 0 0 +/dev/shm /dev/shm none rw,bind 0 0 +/run/shm /run/shm none rw,bind 0 0 diff --git a/hosts/jtx1b-armhf-rb/etc/schroot/default/nssdatabases b/hosts/jtx1b-armhf-rb/etc/schroot/default/nssdatabases new file mode 100644 index 00000000..72615e5d --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/schroot/default/nssdatabases @@ -0,0 +1,11 @@ +# System databases to copy into the chroot from the host system. +# +# <database name> +passwd +shadow +group +gshadow +#services +protocols +networks +hosts diff --git a/hosts/jtx1b-armhf-rb/etc/sudoers.d/jenkins b/hosts/jtx1b-armhf-rb/etc/sudoers.d/jenkins new file mode 100644 index 00000000..d249be94 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/sudoers.d/jenkins @@ -0,0 +1,51 @@ +jenkins ALL= \ + NOPASSWD: /usr/sbin/debootstrap *, \ + /usr/bin/tee /schroots/*, \ + /usr/bin/tee -a /schroots/*, \ + /usr/bin/tee /etc/schroot/chroot.d/jenkins*, \ + /bin/chmod +x /schroots/*, \ + /usr/sbin/chroot /schroots/*, \ + /usr/sbin/chroot /chroots/*, \ + /usr/sbin/chroot /media/*, \ + /bin/ls -la /media/*, \ + /bin/rm -rf --one-file-system /chroots/*, \ + /bin/rm -rf --one-file-system /schroots/*, \ + /bin/rm -rf --one-file-system /srv/live-build/*, \ + /bin/rm -rf --one-file-system /srv/workspace/pbuilder/*, \ + /bin/cp -v *.iso /srv/live-build/results/*, \ + /bin/mv /chroots/* /schroots/*, \ + /bin/mv /schroots/* /schroots/*, \ + /bin/umount -l /chroots/*, \ + /bin/umount -l /schroots/*, \ + /bin/umount -l /media/*, \ + /bin/rmdir /media/*, \ + /bin/mount -o loop*, \ + /bin/mount --bind *, \ + /usr/bin/du *, \ + /bin/kill *, \ + /usr/bin/file *, \ + /bin/dd if=/dev/zero of=/dev/jenkins*, \ + /usr/bin/qemu-system-x86_64 *, \ + /usr/bin/qemu-img *, \ + /sbin/lvcreate *, /sbin/lvremove *, \ + /bin/mkdir -p /media/*, \ + /usr/bin/guestmount *, \ + /bin/cp -rv /media/*, \ + /bin/chown -R jenkins\:jenkins /var/lib/jenkins/jobs/*,\ + SETENV: NOPASSWD: /usr/sbin/pbuilder *, \ + SETENV: NOPASSWD: /usr/bin/timeout -k ??.?h ??h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder *, \ + SETENV: NOPASSWD: /usr/bin/timeout -k ??.?h ??h /usr/bin/ionice -c 3 /usr/bin/nice -n 11 /usr/bin/unshare --uts -- /usr/sbin/pbuilder *, \ + /bin/mv /var/cache/pbuilder/*base*.tgz /var/cache/pbuilder/*base*.tgz, \ + /bin/rm /var/cache/pbuilder/*base*.tgz, \ + /bin/rm -v /var/cache/pbuilder/*base*.tgz, \ + /bin/rm /var/cache/pbuilder/result/*, \ + /usr/bin/dcmd rm *.changes, \ + /usr/bin/dcmd rm *.dsc, \ + /usr/bin/apt-get update, \ + /usr/bin/killall timeout, \ + /usr/sbin/slay 1111, \ + /usr/sbin/slay 2222, \ + /usr/sbin/slay jenkins + +# keep these environment variables +Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/jtx1b-armhf-rb/etc/sudoers.d/jenkins-adm b/hosts/jtx1b-armhf-rb/etc/sudoers.d/jenkins-adm new file mode 100644 index 00000000..3c357be2 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/sudoers.d/jenkins-adm @@ -0,0 +1,7 @@ +# allow member of the jenkins-adm group to sudo-to the jenkins-adm user (owner +# of jenkins script) and the jenkins user itself +%jenkins-adm ALL=(jenkins:jenkins) NOPASSWD: ALL +%jenkins-adm ALL=(jenkins-adm:jenkins-adm) NOPASSWD: ALL +# allow jenkins-adm to run everything as root +%jenkins-adm ALL= NOPASSWD: ALL + diff --git a/hosts/jtx1b-armhf-rb/etc/systemd/system/rc-local.service.d/local.conf b/hosts/jtx1b-armhf-rb/etc/systemd/system/rc-local.service.d/local.conf new file mode 100644 index 00000000..91ed832a --- /dev/null +++ b/hosts/jtx1b-armhf-rb/etc/systemd/system/rc-local.service.d/local.conf @@ -0,0 +1,4 @@ +[Unit] +After=network-online.target +Wants=network-online.target + diff --git a/hosts/jtx1b-armhf-rb/usr/local/bin/dsa-check-packages b/hosts/jtx1b-armhf-rb/usr/local/bin/dsa-check-packages new file mode 100755 index 00000000..28844e5a --- /dev/null +++ b/hosts/jtx1b-armhf-rb/usr/local/bin/dsa-check-packages @@ -0,0 +1,362 @@ +#!/usr/bin/perl + +# dsa-check-packages + +# checks for obsolete/local and upgradeable packages. +# +# packages for the obsolete/local check can be ignored, by +# listing their full name in /etc/nagios/obsolete-packages-ignore +# or by having a regex (starting a line with "/") that matches +# the packagename in said file. +# +# Takes one optional argument, the location of the ignore file. + + +# Copyright (C) 2008, 2009 Peter Palfrader <peter@palfrader.org> +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +use strict; +use warnings; +use English; + +my $IGNORE = "/etc/nagios/obsolete-packages-ignore"; +my $IGNORED = "/etc/nagios/obsolete-packages-ignore.d"; + +my %CODE = ( + 'OK' => 0, + 'WARNING' => 1, + 'CRITICAL' => 2, + 'UNKNOWN' => 3 +); +my $EXITCODE = 'OK'; +sub record($) { + my ($newexit) = @_; + die "code $newexit not defined\n" unless defined $CODE{$newexit}; + + if ($CODE{$newexit} > $CODE{$EXITCODE}) { + $EXITCODE = $newexit; + }; +} + + + +sub get_packages { + $ENV{'COLUMNS'} = 1000; + $ENV{'LC_ALL'} = 'C'; + open(F, "dpkg -l|") or die ("Cannot run dpkg: $!\n"); + my @lines = <F>; + close(F); + chomp(@lines); + + my $line; + my $has_arch = 0; + while (defined($line = shift @lines) && ($line !~ /\+\+\+/)) { + if ($line =~ /Architecture/) { $has_arch = 1; } + } + + my %pkgs; + for $line (@lines) { + my ($state, $pkg, $version, $arch, undef) = split(/ */, $line); + $arch = '' unless $has_arch; + $pkgs{$state}{$pkg} = { 'installed' => $version, arch => $arch } + } + + my $installed = $pkgs{'ii'}; + delete $pkgs{'ii'}; + + my @installed_packages = keys(%$installed); + my @cmd = ("apt-cache", "policy", @installed_packages); + + open my $olderr, ">&STDERR" or die "Can't dup STDERR: $!"; + open STDERR, ">/dev/null" or die "Can't dup STDOUT: $!"; + open (F, "-|", @cmd) or die ("Cannot run apt-cache policy: $!\n"); + @lines = <F>; + close(F); + open STDERR, ">&", $olderr or die "Can't dup OLDERR: $!"; + chomp(@lines); + + my $pkgname = undef; + my $candidate_found = 0; + while (defined($line = shift @lines)) { + if ($line =~ /^([^ ]*):$/) { + # when we have multi-arch capable fu, we require that + # apt-cache policy output is in the same order as its + # arguments. + # + # We needs thi, because the output block in apt-cache + # policy does not show the arch: + # + # | weasel@stanley:~$ apt-cache policy libedit2:amd64 + # | libedit2: + # | Installed: 2.11-20080614-5 + # | Candidate: 2.11-20080614-5 + # + # We replace the package name in the output with the + # one we asked for ($pkg:$arch) - but to match this up + # sanely we need the order to be correct. + # + # For squeeze systems (no m-a), apt-cache policy output + # is all different. + $pkgname = $1; + $candidate_found = 0; + if ($has_arch) { + my $from_list = shift @installed_packages; + next if ($pkgname eq $from_list); # no :$arch in pkgname we asked for + + my $ma_fix_pkgname = $pkgname.':'.$installed->{$from_list}->{'arch'}; + my $ma_fix_from_list = $from_list.':'.$installed->{$from_list}->{'arch'}; + + if ($pkgname eq $ma_fix_from_list || # e.g. ia32-libs-i386. dpkg -l: ia32-libs-i386, apt-cache policy: ia32-libs-i386:i386 + $ma_fix_pkgname eq $from_list) { + $pkgname = $from_list; + } else { + die "Unexpected order mismatch in apt-cache policy output (apt-cache policy name: $pkgname - dpkg -l name: $from_list)\n"; + } + } + } elsif ($line =~ /^ +Installed: (.*)$/) { + # etch dpkg -l does not print epochs, so use this info, it's better + $installed->{$pkgname}{'installed'} = $1; + # initialize security-update + $installed->{$pkgname}{'security-update'} = 0; + } elsif ($line =~ /^ +Candidate: (.*)$/) { + $installed->{$pkgname}{'candidate'} = $1; + } elsif ($line =~ / ([^ ]+) [0-9]+/) { + # check if the next lines show the sources of our candidate + if ($1 eq $installed->{$pkgname}{'candidate'}) { + $candidate_found = 1; + } + } elsif (($line =~ / +[0-9]+ [^ ]+\/(security\.([^ ]+\.)?debian\.org|debian-security).*\/updates\//) && $candidate_found ) { + $installed->{$pkgname}{'security-update'} = 1; + } elsif ($line =~ /^ +\*\*\*/) { + $line = shift @lines; + my @l = split(/ +/, $line); + $installed->{$pkgname}{'origin'} = $l[2]; + $candidate_found = 0; + } + } + + my (%current, %obsolete, %outofdate, %security_outofdate); + for my $pkgname (keys %$installed) { + my $pkg = $installed->{$pkgname}; + + unless (defined($pkg->{'candidate'}) && defined($pkg->{'origin'})) { + $obsolete{$pkgname} = $pkg; + next; + } + + if ($pkg->{'candidate'} ne $pkg->{'installed'}) { + if ($pkg->{'security-update'}) { + $security_outofdate{$pkgname} = $pkg; + } else { + $outofdate{$pkgname} = $pkg; + } + next; + }; + if ($pkg->{'origin'} eq '/var/lib/dpkg/status') { + $obsolete{$pkgname} = $pkg; + next; + } + $current{$pkgname} = $pkg; + } + + $pkgs{'current'} = \%current; + $pkgs{'outofdate'} = \%outofdate; + $pkgs{'security_outofdate'} = \%security_outofdate; + $pkgs{'obsolete'} = \%obsolete; + return \%pkgs; +} + +sub load_ignores { + my ($ignorefiles, $require_file) = @_; + + my @ignores; + + for my $ignoreitem (@$ignorefiles) { + next if (!$require_file and ! -e $ignoreitem); + + my @filestoopen; + if (-d $ignoreitem) { + opendir(DIR, $ignoreitem) or die ("Cannot open dir $ignoreitem: $!\n"); + @filestoopen = readdir(DIR); + closedir(DIR); + + @filestoopen = grep { -f ($ignoreitem.'/'.$_) } @filestoopen; + @filestoopen = grep { /^([a-z0-9_.-]+)+[a-z0-9]+$/i } @filestoopen; + @filestoopen = grep { !/dpkg-(old|dist|new|tmp)$/ } @filestoopen; + @filestoopen = map { ($ignoreitem.'/'.$_) } @filestoopen; + } else { + push @filestoopen, $ignoreitem; + } + + for my $f (@filestoopen) { + open (F, "< $f") or die ("Cannot open $f: $!\n"); + push @ignores, <F>; + close F; + } + } + chomp(@ignores); + return \@ignores; +} + +sub check_ignore { + my ($pkg, $ignores) = @_; + + my $ignore_this = 0; + for my $ignore (@$ignores) { + my $ig = $ignore; + return 1 if ($ig eq $pkg); + if (substr($ig,0,1) eq '/') { + substr($ig, 0, 1, ''); + $ig =~ s,/$,,; + return 1 if ($pkg =~ /$ig/); + } + } + return 0 +} + +sub filter_ignored { + my ($packages, $ignores) = @_; + + my $obs = $packages->{'obsolete'}; + + my (%ignored, %bad); + for my $pkg (keys %$obs) { + if (check_ignore($pkg, $ignores)) { + $ignored{$pkg} = $obs->{$pkg}; + } else { + $bad{$pkg} = $obs->{$pkg}; + }; + } + delete $packages->{'obsolete'}; + $packages->{'obsolete'} = \%bad; + $packages->{'obsolete-ignored'} = \%ignored; +}; + +sub usage { + my ($fd, $exit) = @_; + print $fd "Usage: $PROGRAM_NAME [<ignorefile|dir> [<ignorefile|dir> ...]]\n"; + exit $exit; +} + +my $ignorefiles = [$IGNORE, $IGNORED]; +my $ignorefile_userset = 0; +if (@ARGV >= 1) { + usage(\*STDOUT, 0) if ($ARGV[0] eq "-h"); + usage(\*STDOUT, 0) if ($ARGV[0] eq "--help"); + $ignorefile_userset = 1; + $ignorefiles = \@ARGV; +}; + +my $ignores = load_ignores($ignorefiles, $ignorefile_userset); +my $packages = get_packages(); + +filter_ignored($packages, $ignores); + + + +my @reportform = ( + { 'key' => 'obsolete', + 'listpackages' => 1, + 'long' => "%d local or obsolete packages: %s", + 'short' => "%d obs/loc", + 'perf' => "obs_loc=%d;1;5;0", + 'status' => 'WARNING' }, + { 'key' => 'outofdate', + 'listpackages' => 1, + 'long' => "%d out of date packages: %s", + 'short' => "%d updates", + 'perf' => "outdated=%d;1;5;0", + 'status' => 'WARNING' }, + { 'key' => 'current', + 'listpackages' => 0, + 'long' => "%d packages current.", + 'short' => "%d ok", + 'perf' => "current=%d;;;0", + 'status' => 'OK' }, + { 'key' => 'obsolete-ignored', + 'listpackages' => 1, + 'long' => "%d whitelisted local or obsolete packages: %s", + 'short' => "%d obs/loc(ignored)", + 'perf' => "obs_ign=%d;;;0", + 'status' => 'OK' }, + { 'key' => 'rc', + 'listpackages' => 1, + 'long' => "%d packages removed but not purged: %s", + 'short' => "%d rc", + 'perf' => "rm_unprg=%d;;;0", + 'status' => 'OK' }, + { 'key' => 'hi', + 'listpackages' => 1, + 'long' => "%d packages on hold: %s", + 'short' => "%d hi", + 'perf' => "hold=%d;;;0", + 'status' => 'OK' }, + { 'key' => 'pc', + 'listpackages' => 1, + 'long' => "%d packages requested to be purged but conffiles still installed: %s", + 'short' => "%d pc", + 'perf' => "prg_conf=%d;1;;0", + 'status' => 'WARNING' }, + { 'key' => 'security_outofdate', + 'listpackages' => 1, + 'long' => "%d packages with outstanding security updates: %s", + 'short' => "%d security-updates", + 'perf' => "security_outdated=%d;;1;0", + 'status' => 'CRITICAL' }, + ); + +my @longout; +my @perfout; +my @shortout; +for my $form (@reportform) { + my $pkgs = $packages->{$form->{'key'}}; + delete $packages->{$form->{'key'}}; + my $num = scalar keys %$pkgs; + push @perfout, sprintf($form->{'perf'}, $num); + next unless ($num > 0); + if ($form->{'listpackages'}) { + my $list = join(", ", keys %$pkgs); + push @longout, sprintf($form->{'long'}, $num, $list); + } else { + push @longout, sprintf($form->{'long'}, $num); + }; + push @shortout, sprintf($form->{'short'}, $num); + record($form->{'status'}); +}; +if (scalar keys %$packages) { + record('WARNING'); + unshift @shortout, "unk: ".join(", ", keys %$packages); + for my $status (sort {$b cmp $a} keys %$packages) { + my $pkgs = $packages->{$status}; + my $list = join(", ", keys %$pkgs); + unshift @longout, "Unknown package status $status: $list"; + }; +} + +my $shortout = $EXITCODE.": ".join(", ", @shortout); +my $longout = join("\n", @longout); +my $perfout = "|".join(" ", @perfout); + +print $shortout,"\n"; +print $longout,"\n"; +print $perfout,"\n"; + +exit $CODE{$EXITCODE}; diff --git a/hosts/jtx1b-armhf-rb/usr/local/bin/dsa-check-running-kernel b/hosts/jtx1b-armhf-rb/usr/local/bin/dsa-check-running-kernel new file mode 100755 index 00000000..aa6e35bd --- /dev/null +++ b/hosts/jtx1b-armhf-rb/usr/local/bin/dsa-check-running-kernel @@ -0,0 +1,254 @@ +#!/bin/bash + +# Check if the running kernel has the same version string as the on-disk +# kernel image. + +# Copyright 2008,2009,2011,2012,2013,2014 Peter Palfrader +# Copyright 2009 Stephen Gran +# Copyright 2010,2012,2013 Uli Martens +# Copyright 2011 Alexander Reichle-Schmehl +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +OK=0; +WARNING=1; +CRITICAL=2; +UNKNOWN=3; + +get_offset() { + local file needle + + file="$1" + needle="$2" + + perl -e ' + undef $/; + $i = 0; $k=<>; + while (($i = index($k, "'"$needle"'", $i)) >= 0) { + print $i++,"\n"; + }; ' < "$file" +} + +get_avail() { + # This is wrong, but leaves room for when we have to care for machines running + # myfirstunix-image-0.1-dsa-arm + local prefix="$1"; shift + + local kervers=$(uname -r) + + local metavers='' + + # DSA uses kernel versions of the form 2.6.29.3-dsa-dl380-oldxeon, where + # Debian uses versions of the form 2.6.29-2-amd64 + if [ "${kervers#2}" != "$kervers" ]; then + if [ "${kervers//dsa}" != "$kervers" ]; then + metavers=$(echo $kervers | sed -r -e 's/^2\.(4|6)\.[0-9]+([\.0-9]+?)-(.*)/2.\1-\3/') + else + metavers=$(echo $kervers | sed -r -e 's/^2\.(4|6)\.[0-9]+-[A-Za-z0-9\.]+-(.*)/2.\1-\2/') + fi + else + metavers=$(echo $kervers | sed -r -e 's/^[0-9]+\.[0-9]+(\.[0-9])?+-[A-Za-z0-9\.]+-(.*)/\2/') + fi + + # Attempt to track back to a metapackage failed. bail + if [ "$metavers" = "$kervers" ]; then + return 2 + fi + + # We're just going to give up if we can't find a matching metapackage + # I tried being strict once, and it just caused a lot of headaches. We'll see how + # being lax does for us + + local output=$(apt-cache policy ${prefix}-image-${metavers} 2>/dev/null) + local metaavailvers=$(echo "$output" | grep '^ Candidate:' | awk '{print $2}') + local metainstavers=$(echo "$output" | grep '^ Installed:' | awk '{print $2}') + + if [ -z "$metaavailvers" ] || [ "$metaavailvers" = '(none)' ]; then + return 2 + fi + if [ -z "$metainstavers" ] || [ "$metainstavers" = '(none)' ]; then + return 2 + fi + + if [ "$metaavailvers" != "$metainstavers" ] ; then + echo "${prefix}-image-${metavers} $metaavailvers available but $metainstavers installed" + return 1 + fi + + local imagename=0 + # --no-all-versions show shows only the candidate + for vers in $(apt-cache --no-all-versions show ${prefix}-image-${metavers} | sed -n 's/^Depends: //p' | tr ',' '\n' | tr -d ' ' | grep ${prefix}-image | awk '{print $1}' | sort -u); do + if dpkg --compare-versions "1.$vers" gt "1.$imagename"; then + imagename=$vers + fi + done + + if [ -z "$imagename" ] || [ "$imagename" = 0 ]; then + return 2 + fi + + if [ "$imagename" != "${prefix}-image-${kervers}" ]; then + if dpkg --compare-versions 1."$imagename" lt 1."${prefix}-image-${kervers}"; then + return 2 + fi + echo "$imagename" != "${prefix}-image-${kervers}" + return 1 + fi + + local availvrs=$(apt-cache policy ${imagename} 2>/dev/null | grep '^ Candidate' | awk '{print $2}') + local kernelversion=$(apt-cache policy ${prefix}-image-${kervers} 2>/dev/null | grep '^ Installed:' | awk '{print $2}') + + if [ "$availvrs" = "$kernelversion" ]; then + return 0 + fi + + echo "$kernelversion != $availvrs" + return 1 +} + +cat_vmlinux() { + local image header filter hdroff + + image="$1" + header="$2" + filter="$3" + hdroff="$4" + + get_offset "$image" $header | head -n 5 | while read off; do + (if [ "$off" != 0 ]; then + dd ibs="$((off+hdroff))" skip=1 count=0 + fi && + dd bs=512k) < "$image" 2>/dev/null | $filter 2>/dev/null + done +} + +get_image_linux() { + local image + + image="$1" + + # gzip compressed image + cat_vmlinux "$image" "\x1f\x8b\x08\x00" "zcat" 0 + cat_vmlinux "$image" "\x1f\x8b\x08\x08" "zcat" 0 + # lzma compressed image + cat_vmlinux "$image" "\x00\x00\x00\x02\xff" "xzcat" -1 + cat_vmlinux "$image" "\x00\x00\x00\x04\xff" "xzcat" -1 + # xz compressed image + cat_vmlinux "$image" "\xfd\x37\x7a\x58\x5a " "xzcat" 0 + + echo "ERROR: Unable to extract kernel image." 2>&1 + exit 1 +} + + +freebsd_check_running_version() { + local imagefile="$1"; shift + + local r="$(uname -r)" + local v="$(uname -v| sed -e 's/^#[0-9]*/&:/')" + + local q='@(#)FreeBSD '"$r $v" + + if zcat "$imagefile" | $STRINGS | grep -F -q "$q"; then + echo "OK" + else + echo "not OK" + fi +} + +STRINGS=""; +if [ -x "$(which strings)" ]; then + STRINGS="$(which strings)" +elif [ -x "$(which busybox)" -a "$( echo foobar | $(which busybox) strings 2>/dev/null)" = "foobar" ]; then + STRINGS="$(which busybox) strings" +fi + +searched="" +for on_disk in \ + "/boot/vmlinuz-`uname -r`"\ + "/boot/vmlinux-`uname -r`"\ + "/boot/kfreebsd-`uname -r`.gz"; do + + if [ -e "$on_disk" ]; then + if [ -z "$STRINGS" ]; then + echo "UNKNOWN: 'strings' command missing, perhaps install binutils or busybox?" + exit $UNKNOWN + fi + if [ "${on_disk/vmlinu}" != "$on_disk" ]; then + on_disk_version="`get_image_linux "$on_disk" | $STRINGS | grep 'Linux version' | head -n1`" + if [ -x /usr/bin/lsb_release ] ; then + vendor=$(lsb_release -i -s) + if [ -n "$vendor" ] && [ "xDebian" != "x$vendor" ] ; then + on_disk_version=$( echo $on_disk_version|sed -e "s/ ($vendor [[:alnum:]\.-]\+ [[:alnum:]\.]\+)//") + fi + fi + [ -z "$on_disk_version" ] || break + on_disk_version="`cat "$on_disk" | $STRINGS | grep 'Linux version' | head -n1`" + [ -z "$on_disk_version" ] || break + + echo "UNKNOWN: Failed to get a version string from image $on_disk" + exit $UNKNOWN + else + on_disk_version="$(zcat $on_disk | $STRINGS | grep Debian | head -n 1 | sed -e 's/Debian [[:alnum:]]\+ (\(.*\))/\1/')" + fi + fi + searched="$searched $on_disk" +done + +if ! [ -e "$on_disk" ]; then + echo "WARNING: Did not find a kernel image (checked$searched) - I have no idea which kernel I am running" + exit $WARNING +fi + +if [ "$(uname -s)" = "Linux" ]; then + running_version="`cat /proc/version`" + if [ -z "$running_version" ] ; then + echo "UNKNOWN: Failed to get a version string from running system" + exit $UNKNOWN + fi + + if [ "$running_version" != "$on_disk_version" ]; then + echo "WARNING: Running kernel does not match on-disk kernel image: [$running_version != $on_disk_version]" + exit $WARNING + fi + + ret="$(get_avail linux)" + if [ $? = 1 ]; then + echo "WARNING: Kernel needs upgrade [$ret]" + exit $WARNING + fi +else + image_current=$(freebsd_check_running_version $on_disk) + running_version="`uname -s` `uname -r` `uname -v`" + if [ "$image_current" != "OK" ]; then + approx_time="$(date -d "@`stat -c '%Y' "$on_disk"`" +"%Y-%m-%d %H:%M:%S")" + echo "WARNING: Currently running kernel ($running_version) does not match on disk image (~ $approx_time)" + exit $WARNING; + fi + + ret="$(get_avail linux)" + if [ $? = 1 ]; then + echo "WARNING: Kernel needs upgrade [$ret]" + exit $WARNING + fi +fi + +echo "OK: Running kernel matches on disk image: [$running_version]" +exit $OK diff --git a/hosts/jtx1b-armhf-rb/usr/local/sbin/nagios-check-libs b/hosts/jtx1b-armhf-rb/usr/local/sbin/nagios-check-libs new file mode 100755 index 00000000..77b37805 --- /dev/null +++ b/hosts/jtx1b-armhf-rb/usr/local/sbin/nagios-check-libs @@ -0,0 +1,204 @@ +#!/usr/bin/perl -w + +# Copyright (C) 2005, 2006, 2007, 2008, 2012, 2015 Peter Palfrader <peter@palfrader.org> +# 2012 Uli Martens <uli@youam.net> +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +use strict; +use English; +use Getopt::Long; + +$ENV{'PATH'} = '/bin:/sbin:/usr/bin:/usr/sbin'; +delete @ENV{'IFS', 'CDPATH', 'ENV', 'BASH_ENV'}; + +my $LSOF = '/usr/bin/lsof -F0'; +my $VERSION = '0.2015012901'; + +# nagios exit codes +my $OK = 0; +my $WARNING = 1; +my $CRITICAL = 2; +my $UNKNOWN = 3; + +my $params; +my $config; + +Getopt::Long::config('bundling'); + +sub dief { + print STDERR @_; + exit $UNKNOWN; +} + +if (!GetOptions ( + '--help' => \$params->{'help'}, + '--version' => \$params->{'version'}, + '--quiet' => \$params->{'quiet'}, + '--verbose' => \$params->{'verbose'}, + '-v' => \$params->{'verbose'}, + '--config=s' => \$params->{'config'}, + )) { + dief ("$PROGRAM_NAME: Usage: $PROGRAM_NAME [--help|--version] [--verbose] [--quiet] [--config=<CONFIGFILE>]\n"); +}; +if ($params->{'help'}) { + print "$PROGRAM_NAME: Usage: $PROGRAM_NAME [--help|--version] [--verbose] [--quiet] [--config=<CONFIGFILE>]\n"; + print "Reports processes that are linked against libraries that no longer exist.\n"; + print "The optional config file can specify ignore rules - see the sample config file.\n"; + exit (0); +}; +if ($params->{'version'}) { + print "nagios-check-libs $VERSION\n"; + print "nagios check for availability of debian (security) updates\n"; + print "Copyright (c) 2005, 2006, 2007, 2008, 2012 Peter Palfrader <peter\@palfrader.org>\n"; + exit (0); +}; + +if (! defined $params->{'config'}) { + $params->{'config'} = '/etc/nagios/check-libs.conf'; +} elsif (! -e $params->{'config'}) { + dief("Config file $params->{'config'} does not exist.\n"); +} + +if (-e $params->{'config'}) { + eval "use YAML::Syck; 1" or dief "you need YAML::Syck (libyaml-syck-perl) to load a config file"; + open(my $fh, '<', $params->{'config'}) or dief "Cannot open config file $params->{'config'}: $!"; + $config = LoadFile($fh); + close($fh); + if (!(ref($config) eq "HASH")) { + dief("Loaded config is not a hash!\n"); + } +} else { + $config = { + 'ignorelist' => [ + '$path =~ m#^/proc/#', + '$path =~ m#^/var/tmp/#', + '$path =~ m#^/SYS#', + '$path =~ m#^/drm$# # xserver stuff', + '$path =~ m#^/dev/zero#', + '$path =~ m#^/dev/shm/#', + ] + }; +} + +if (! exists $config->{'ignorelist'}) { + $config->{'ignorelist'} = []; +} elsif (! (ref($config->{'ignorelist'}) eq 'ARRAY')) { + dief("Config->ignorelist is not an array!\n"); +} + + +my %processes; + +sub getPIDs($$) { + my ($user, $process) = @_; + return join(', ', sort keys %{ $processes{$user}->{$process} }); +}; +sub getProcs($) { + my ($user) = @_; + + return join(', ', map { $_.' ('.getPIDs($user, $_).')' } (sort {$a cmp $b} keys %{ $processes{$user} })); +}; +sub getUsers() { + return join('; ', (map { $_.': '.getProcs($_) } (sort {$a cmp $b} keys %processes))); +}; +sub inVserver() { + my ($f, $key); + if (-e "/proc/self/vinfo" ) { + $f = "/proc/self/vinfo"; + $key = "XID"; + } else { + $f = "/proc/self/status"; + $key = "s_context"; + }; + open(F, "< $f") or return 0; + while (<F>) { + my ($k, $v) = split(/: */, $_, 2); + if ($k eq $key) { + close F; + return ($v > 0); + }; + }; + close F; + return 0; +} + +my $INVSERVER = inVserver(); + +print STDERR "Running $LSOF -n\n" if $params->{'verbose'}; +open (LSOF, "$LSOF -n|") or dief ("Cannot run $LSOF -n: $!\n"); +my @lsof=<LSOF>; +close LSOF; +if ($CHILD_ERROR) { # program failed + dief("$LSOF -n returned with non-zero exit code: ".($CHILD_ERROR / 256)."\n"); +}; + +my ($process, $pid, $user); +LINE: for my $line (@lsof) { + if ( $line =~ /^p/ ) { + my %fields = map { m/^(.)(.*)$/ ; $1 => $2 } grep { defined $_ and length $_ >1} split /\0/, $line; + $process = $fields{c}; + $pid = $fields{p}; + $user = $fields{L}; + next; + } + + unless ( $line =~ /^f/ ) { + dief("UNKNOWN strange line read from lsof\n"); + # don't print it because it contains NULL characters... + } + + my %fields = map { m/^(.)(.*)$/ ; $1 => $2 } grep { defined $_ and length $_ >1} split /\0/, $line; + + my $fd = $fields{f}; + my $inode = $fields{i}; + my $path = $fields{n}; + if ($path =~ m/\.dpkg-/ || $path =~ m/\(deleted\)/ || $path =~ /path inode=/ || $path =~ m#/\.nfs# || $fd eq 'DEL') { + my $deleted_in_path = ($path =~ m/\(deleted\)/ || $path =~ m/\.nfs/); + next if ($deleted_in_path && $fd =~ /^[0-9]*$/); # Ignore deleted files that are open via normal file handles. + next if ($deleted_in_path && $fd eq 'cwd'); # Ignore deleted directories that we happen to be in. + + $path =~ s/^\(deleted\)//; # in some cases "(deleted)" is at the beginning of the string + for my $i (@{$config->{'ignorelist'}}) { + my $ignore = eval($i); + next LINE if $ignore; + } + next if ($INVSERVER && ($process eq 'init') && ($pid == 1) && ($user eq 'root')); + if ( $params->{'verbose'} ) { + print STDERR "adding $process($pid) because of [$path]:\n"; + print STDERR $line; + } + $processes{$user}->{$process}->{$pid} = 1; + }; +}; + + + +my $message=''; +my $exit = $OK; +if (keys %processes) { + $exit = $WARNING; + $message = 'The following processes have libs linked that were upgraded: '. getUsers()."\n"; +} else { + $message = "No upgraded libs linked in running processes\n" unless $params->{'quiet'}; +}; + +print $message; +exit $exit; diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index 2aef7f7e..66783572 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -290,10 +290,10 @@ - 'ff2a': { my_arch: 'armhf' } - 'ff2b': { my_arch: 'armhf' } - 'ff4a': { my_arch: 'armhf' } - - 'ff64a': { my_arch: 'armhf' } - 'hb0': { my_arch: 'armhf' } - - 'jtk1a': { my_arch: 'armhf' } + - 'jtk1b': { my_arch: 'armhf' } - 'jtx1a': { my_arch: 'armhf' } + - 'jtx1b': { my_arch: 'armhf' } - 'odc2a': { my_arch: 'armhf' } - 'odxu4': { my_arch: 'armhf' } - 'odxu4b': { my_arch: 'armhf' } @@ -344,10 +344,10 @@ - 'ff2a': { my_arch: 'armhf' } - 'ff2b': { my_arch: 'armhf' } - 'ff4a': { my_arch: 'armhf' } - - 'ff64a': { my_arch: 'armhf' } - 'hb0': { my_arch: 'armhf' } - - 'jtk1a': { my_arch: 'armhf' } + - 'jtk1b': { my_arch: 'armhf' } - 'jtx1a': { my_arch: 'armhf' } + - 'jtx1b': { my_arch: 'armhf' } - 'odc2a': { my_arch: 'armhf' } - 'odxu4': { my_arch: 'armhf' } - 'odxu4b': { my_arch: 'armhf' } @@ -419,10 +419,10 @@ - 'ff2a': { my_arch: 'armhf' } - 'ff2b': { my_arch: 'armhf' } - 'ff4a': { my_arch: 'armhf' } - - 'ff64a': { my_arch: 'armhf' } - 'hb0': { my_arch: 'armhf' } - - 'jtk1a': { my_arch: 'armhf' } + - 'jtk1b': { my_arch: 'armhf' } - 'jtx1a': { my_arch: 'armhf' } + - 'jtx1b': { my_arch: 'armhf' } - 'odc2a': { my_arch: 'armhf' } - 'odxu4': { my_arch: 'armhf' } - 'odxu4b': { my_arch: 'armhf' } |