diff options
| -rwxr-xr-x | bin/common-functions.sh | 2 | ||||
| -rwxr-xr-x | bin/reproducible_node_wrapper.sh | 18 | ||||
| -rw-r--r-- | hosts/profitbricks-build2-amd64/etc/apt/apt.conf.d/80proxy | 2 | ||||
| -rw-r--r-- | hosts/profitbricks-build2-amd64/etc/apt/sources.list | 11 | ||||
| -rwxr-xr-x | hosts/profitbricks-build2-amd64/etc/pbuilder/rebuild-hooks/D01_modify_environment | 12 | ||||
| -rw-r--r-- | hosts/profitbricks-build2-amd64/etc/pbuilderrc | 30 | ||||
| -rw-r--r-- | hosts/profitbricks-build2-amd64/etc/schroot/default/fstab | 24 | ||||
| -rw-r--r-- | hosts/profitbricks-build2-amd64/etc/schroot/default/nssdatabases | 11 | ||||
| -rw-r--r-- | hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins | 44 | ||||
| -rw-r--r-- | hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins-adm | 4 | ||||
| -rw-r--r-- | job-cfg/reproducible.yaml | 90 |
11 files changed, 247 insertions, 1 deletions
diff --git a/bin/common-functions.sh b/bin/common-functions.sh index 1784c962..b8efa6f6 100755 --- a/bin/common-functions.sh +++ b/bin/common-functions.sh @@ -61,7 +61,7 @@ else fi if [ -z "$http_proxy" ]; then case $HOSTNAME in - jenkins|profitbricks-build1-amd64) export http_proxy="http://localhost:3128" ;; + jenkins|profitbricks-build?-amd64) export http_proxy="http://localhost:3128" ;; bpi0|cbxi4pro0|hb0|wbq0) export http_proxy="http://10.0.0.15:8000/" ;; *) echo "unsupported host, exiting." ; exit 1 ;; esac diff --git a/bin/reproducible_node_wrapper.sh b/bin/reproducible_node_wrapper.sh index 574e0fa8..d9768fab 100755 --- a/bin/reproducible_node_wrapper.sh +++ b/bin/reproducible_node_wrapper.sh @@ -109,6 +109,24 @@ elif [ "$*" = "reproducible_setup_schroot_unstable_amd64_profitbricks1" ] ; then exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable unstable ; croak "Exec failed"; elif [ "$*" = "reproducible_setup_schroot_experimental_amd64_profitbricks1" ] ; then exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-experimental experimental ; croak "Exec failed"; +elif [ "$*" = "reproducible_setup_pbuilder_testing_amd64_profitbricks2" ] ; then + exec /srv/jenkins/bin/reproducible_setup_pbuilder.sh testing ; croak "Exec failed"; +elif [ "$*" = "reproducible_setup_pbuilder_unstable_amd64_profitbricks2" ] ; then + exec /srv/jenkins/bin/reproducible_setup_pbuilder.sh unstable ; croak "Exec failed"; +elif [ "$*" = "reproducible_setup_pbuilder_experimental_amd64_profitbricks2" ] ; then + exec /srv/jenkins/bin/reproducible_setup_pbuilder.sh experimental ; croak "Exec failed"; +elif [ "$*" = "reproducible_maintenance_amd64_profitbricks2" ] ; then + exec /srv/jenkins/bin/reproducible_maintenance.sh ; croak "Exec failed"; +elif [ "$*" = "reproducible_setup_schroot_testing_debbindiff_amd64_profitbricks2" ] ; then + exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-testing-debbindiff testing debbindiff locales-all ; croak "Exec failed"; +elif [ "$*" = "reproducible_setup_schroot_unstable_debbindiff_amd64_profitbricks2" ] ; then + exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable-debbindiff unstable debbindiff locales-all ; croak "Exec failed"; +elif [ "$*" = "reproducible_setup_schroot_testing_amd64_profitbricks2" ] ; then + exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-testing testing ; croak "Exec failed"; +elif [ "$*" = "reproducible_setup_schroot_unstable_amd64_profitbricks2" ] ; then + exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable unstable ; croak "Exec failed"; +elif [ "$*" = "reproducible_setup_schroot_experimental_amd64_profitbricks2" ] ; then + exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-experimental experimental ; croak "Exec failed"; fi croak "Command '$*' not found in allowed commands." diff --git a/hosts/profitbricks-build2-amd64/etc/apt/apt.conf.d/80proxy b/hosts/profitbricks-build2-amd64/etc/apt/apt.conf.d/80proxy new file mode 100644 index 00000000..fe4b3b77 --- /dev/null +++ b/hosts/profitbricks-build2-amd64/etc/apt/apt.conf.d/80proxy @@ -0,0 +1,2 @@ +Acquire::http::Proxy "http://127.0.0.1:3128/"; + diff --git a/hosts/profitbricks-build2-amd64/etc/apt/sources.list b/hosts/profitbricks-build2-amd64/etc/apt/sources.list new file mode 100644 index 00000000..c9de5374 --- /dev/null +++ b/hosts/profitbricks-build2-amd64/etc/apt/sources.list @@ -0,0 +1,11 @@ +deb http://ftp.de.debian.org/debian/ jessie main non-free +#deb-src http://ftp.de.debian.org/debian/ jessie main non-free + +deb http://ftp.de.debian.org/debian/ jessie-updates main contrib non-free +#deb-src http://ftp.de.debian.org/debian/ jessie-updates main contrib non-free + +deb http://security.debian.org/ jessie/updates main non-free +#deb-src http://security.debian.org/ jessie/updates main non-free + +deb http://ftp.de.debian.org/debian/ jessie-backports main non-free +#deb-src http://ftp.de.debian.org/debian/ jessie-backports main non-free diff --git a/hosts/profitbricks-build2-amd64/etc/pbuilder/rebuild-hooks/D01_modify_environment b/hosts/profitbricks-build2-amd64/etc/pbuilder/rebuild-hooks/D01_modify_environment new file mode 100755 index 00000000..d9550045 --- /dev/null +++ b/hosts/profitbricks-build2-amd64/etc/pbuilder/rebuild-hooks/D01_modify_environment @@ -0,0 +1,12 @@ +#!/bin/sh + +set -e + +# exit if we are in the same UTS namespace than init +[ "$(readlink /proc/1/ns/uts)" = "$(readlink /proc/self/ns/uts)" ] && exit 0 + +echo "I: Changing hostname to test build reproducibility" >&2 +sed -e '/^127.0.0.1/s/$/ i-capture-the-hostname i-capture-the-hostname.i-capture-the-domain/' -i /etc/hosts +hostname i-capture-the-hostname +domainname i-capture-the-domain +export CAPTURE_ENVIRONMENT="I capture the environment" diff --git a/hosts/profitbricks-build2-amd64/etc/pbuilderrc b/hosts/profitbricks-build2-amd64/etc/pbuilderrc new file mode 100644 index 00000000..6c3e4a21 --- /dev/null +++ b/hosts/profitbricks-build2-amd64/etc/pbuilderrc @@ -0,0 +1,30 @@ +# this is your configuration file for pbuilder. +# the file in /usr/share/pbuilder/pbuilderrc is the default template. +# /etc/pbuilderrc is the one meant for overwriting defaults in +# the default template +# +# read pbuilderrc.5 document for notes on specific options. +case $HOSTNAME in + jenkins|profitbricks-build?-amd64) MIRRORSITE=http://ftp.de.debian.org/debian ;; + bpi0|cbxi4pro0|hb0|wbq0) MIRRORSITE=http://ftp.us.debian.org/debian ;; + *) echo "unsupported host, exiting." ; exit 1 ;; +esac +EXTRAPACKAGES="fakeroot" +APTCACHE="" +COMPRESSPROG="pigz" +# unset proxy, see #780587 +unset http_proxy +export -n http_proxy +# /sys and /proc were added as workarounds for #773767 +# there are software wanting /dev/shm writable, so give them so. There are tons +# of bug shm related in pbuilder, that's just a workaround for all of them. +# /run/shm is already mounted as a tmpfs by default. +BINDMOUNTS="$BINDMOUNTS /sys /dev/shm" +# set PATH to predictable values, see #780729 and #780725 +PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games" + +# used on reproducible builds +if [ "$(readlink /proc/1/ns/uts)" != "$(readlink /proc/self/ns/uts)" ]; then + PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" +fi + diff --git a/hosts/profitbricks-build2-amd64/etc/schroot/default/fstab b/hosts/profitbricks-build2-amd64/etc/schroot/default/fstab new file mode 100644 index 00000000..74468dd2 --- /dev/null +++ b/hosts/profitbricks-build2-amd64/etc/schroot/default/fstab @@ -0,0 +1,24 @@ +# fstab: static file system information for chroots. +# Note that the mount point will be prefixed by the chroot path +# (CHROOT_PATH) +# +# <file system> <mount point> <type> <options> <dump> <pass> +/proc /proc none rw,bind 0 0 +/sys /sys none rw,bind 0 0 +/dev /dev none rw,bind 0 0 +/dev/pts /dev/pts none rw,bind 0 0 +/home /home none rw,bind 0 0 +/tmp /tmp none rw,bind 0 0 +/srv/reproducible-results /srv/reproducible-results none rw,bind 0 0 +/srv/d-i /srv/d-i none rw,bind 0 0 +/srv/jenkins /srv/jenkins none rw,bind 0 0 +/srv/live-build /srv/live-build none rw,bind 0 0 + +# It may be desirable to have access to /run, especially if you wish +# to run additional services in the chroot. However, note that this +# may potentially cause undesirable behaviour on upgrades, such as +# killing services on the host. +#/run /run none rw,bind 0 0 +#/run/lock /run/lock none rw,bind 0 0 +/dev/shm /dev/shm none rw,bind 0 0 +/run/shm /run/shm none rw,bind 0 0 diff --git a/hosts/profitbricks-build2-amd64/etc/schroot/default/nssdatabases b/hosts/profitbricks-build2-amd64/etc/schroot/default/nssdatabases new file mode 100644 index 00000000..72615e5d --- /dev/null +++ b/hosts/profitbricks-build2-amd64/etc/schroot/default/nssdatabases @@ -0,0 +1,11 @@ +# System databases to copy into the chroot from the host system. +# +# <database name> +passwd +shadow +group +gshadow +#services +protocols +networks +hosts diff --git a/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins b/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins new file mode 100644 index 00000000..d75335f9 --- /dev/null +++ b/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins @@ -0,0 +1,44 @@ +jenkins ALL= \ + NOPASSWD: /usr/sbin/debootstrap *, \ + /usr/bin/tee /chroots/*, \ + /usr/bin/tee -a /chroots/*, \ + /usr/bin/tee /etc/schroot/chroot.d/jenkins*, \ + /bin/chmod +x /chroots/*, \ + /usr/sbin/chroot /chroots/*, \ + /usr/sbin/chroot /media/*, \ + /bin/ls -la /media/*, \ + /bin/rm -rf --one-file-system /chroots/*, \ + /bin/rm -rf --one-file-system /schroots/*, \ + /bin/rm -rf --one-file-system /srv/live-build/*, \ + /bin/cp -v *.iso /srv/live-build/results/*, \ + /bin/mv /chroots/* /schroots/*, \ + /bin/mv /schroots/* /schroots/*, \ + /bin/umount -l /chroots/*, \ + /bin/umount -l /media/*, \ + /bin/rmdir /media/*, \ + /bin/mount -o loop*, \ + /bin/mount --bind *, \ + /usr/bin/du *, \ + /bin/kill *, \ + /usr/bin/file *, \ + /bin/dd if=/dev/zero of=/dev/jenkins*, \ + /usr/bin/qemu-system-x86_64 *, \ + /usr/bin/qemu-img *, \ + /sbin/lvcreate *, /sbin/lvremove *, \ + /bin/mkdir -p /media/*, \ + /usr/bin/guestmount *, \ + /bin/cp -rv /media/*, \ + /bin/chown -R jenkins\:jenkins /var/lib/jenkins/jobs/*,\ + SETENV: NOPASSWD: /usr/sbin/pbuilder *, \ + SETENV: NOPASSWD: /usr/bin/timeout -k 12.1h 12h /usr/bin/ionice -c 3 /usr/bin/nice /usr/sbin/pbuilder *, \ + SETENV: NOPASSWD: /usr/bin/timeout -k 12.1h 12h /usr/bin/ionice -c 3 /usr/bin/nice /usr/bin/linux64 --uname-2.6 /usr/bin/unshare --uts -- /usr/sbin/pbuilder *, \ + /bin/mv /var/cache/pbuilder/*base*.tgz /var/cache/pbuilder/*base*.tgz, \ + /bin/rm /var/cache/pbuilder/*base*.tgz, \ + /bin/rm -v /var/cache/pbuilder/*base*.tgz, \ + /bin/rm /var/cache/pbuilder/result/*, \ + /usr/bin/dcmd rm *.changes, \ + /usr/bin/dcmd rm *.dsc, \ + /usr/bin/apt-get update + +# keep these environment variables +Defaults env_keep += "http_proxy", env_reset diff --git a/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins-adm b/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins-adm new file mode 100644 index 00000000..e585d03b --- /dev/null +++ b/hosts/profitbricks-build2-amd64/etc/sudoers.d/jenkins-adm @@ -0,0 +1,4 @@ +# allow member of the jenkins-adm group to sudo-to the jenkins-adm user (owner +# of jenkins script) and the jenkins user itself +%jenkins-adm ALL=(jenkins:jenkins) NOPASSWD: ALL +%jenkins-adm ALL=(jenkins-adm:jenkins-adm) NOPASSWD: ALL diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index 949227cc..b3a2b0b0 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -190,6 +190,10 @@ - job-template: defaults: reproducible + name: '{name}_maintenance_amd64_profitbricks2' + +- job-template: + defaults: reproducible name: '{name}_setup_pbuilder_unstable' - job-template: @@ -198,6 +202,10 @@ - job-template: defaults: reproducible + name: '{name}_setup_pbuilder_unstable_amd64_profitbricks2' + +- job-template: + defaults: reproducible name: '{name}_setup_pbuilder_testing' - job-template: @@ -222,6 +230,10 @@ - job-template: defaults: reproducible + name: '{name}_setup_pbuilder_testing_amd64_profitbricks2' + +- job-template: + defaults: reproducible name: '{name}_setup_pbuilder_experimental' - job-template: @@ -230,6 +242,10 @@ - job-template: defaults: reproducible + name: '{name}_setup_pbuilder_experimental_amd64_profitbricks2' + +- job-template: + defaults: reproducible name: '{name}_setup_schroot_unstable_debbindiff' - job-template: @@ -238,6 +254,10 @@ - job-template: defaults: reproducible + name: '{name}_setup_schroot_unstable_debbindiff_amd64_profitbricks2' + +- job-template: + defaults: reproducible name: '{name}_setup_schroot_testing_debbindiff' - job-template: @@ -262,6 +282,10 @@ - job-template: defaults: reproducible + name: '{name}_setup_schroot_testing_debbindiff_amd64_profitbricks2' + +- job-template: + defaults: reproducible name: '{name}_setup_schroot_unstable' - job-template: @@ -270,6 +294,10 @@ - job-template: defaults: reproducible + name: '{name}_setup_schroot_unstable_amd64_profitbricks2' + +- job-template: + defaults: reproducible name: '{name}_setup_schroot_testing' - job-template: @@ -294,6 +322,10 @@ - job-template: defaults: reproducible + name: '{name}_setup_schroot_testing_amd64_profitbricks2' + +- job-template: + defaults: reproducible name: '{name}_setup_schroot_experimental' - job-template: @@ -302,6 +334,10 @@ - job-template: defaults: reproducible + name: '{name}_setup_schroot_experimental_amd64_profitbricks2' + +- job-template: + defaults: reproducible name: '{name}_scheduler' - job-template: @@ -439,6 +475,12 @@ my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' my_node: 'profitbricks1' + - '{name}_maintenance_amd64_profitbricks2': + my_description: 'Do some maintenance: check for old directories laying around, do backups, etc.' + my_timed: '5 0,4,8,12,16,20 * * *' + my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' + my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' + my_node: 'profitbricks2' - '{name}_setup_pbuilder_testing': my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' my_timed: '23 0,4,8,12,16,20 * * *' @@ -475,6 +517,12 @@ my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' my_node: 'profitbricks1' + - '{name}_setup_pbuilder_testing_amd64_profitbricks2': + my_description: 'Setup and update pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' + my_timed: '23 0,4,8,12,16,20 * * *' + my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' + my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' + my_node: 'profitbricks2' - '{name}_setup_pbuilder_unstable': my_description: 'Setup and update pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' my_timed: '23 0,4,8,12,16,20 * * *' @@ -487,6 +535,12 @@ my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' my_node: 'profitbricks1' + - '{name}_setup_pbuilder_unstable_amd64_profitbricks2': + my_description: 'Setup and update pbuilder for reproducible builds of packages from unstable as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' + my_timed: '23 0,4,8,12,16,20 * * *' + my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' + my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' + my_node: 'profitbricks2' - '{name}_setup_pbuilder_experimental': my_description: 'Setup and update pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' my_timed: '23 0,4,8,12,16,20 * * *' @@ -499,6 +553,12 @@ my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' my_node: 'profitbricks1' + - '{name}_setup_pbuilder_experimental_amd64_profitbricks2': + my_description: 'Setup and update pbuilder for reproducible builds of packages from experimental as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' + my_timed: '23 0,4,8,12,16,20 * * *' + my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' + my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' + my_node: 'profitbricks2' - '{name}_setup_schroot_unstable_debbindiff': my_description: 'Setup schroot for running debbindiff in a unstable environment (this is needed to be able to correctly investigate haskell binaries...)' my_timed: '23 0 * * *' @@ -511,6 +571,12 @@ my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' my_node: 'profitbricks1' + - '{name}_setup_schroot_unstable_debbindiff_amd64_profitbricks2': + my_description: 'Setup schroot for running debbindiff in a unstable environment (this is needed to be able to correctly investigate haskell binaries...)' + my_timed: '23 0 * * *' + my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' + my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' + my_node: 'profitbricks2' - '{name}_setup_schroot_testing_debbindiff': my_description: 'Setup schroot for running debbindiff in a testing environment (this is needed to be able to correctly investigate haskell binaries...)' my_timed: '23 0 * * *' @@ -547,6 +613,12 @@ my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' my_node: 'profitbricks1' + - '{name}_setup_schroot_testing_debbindiff_amd64_profitbricks2': + my_description: 'Setup schroot for running debbindiff in a testing environment (this is needed to be able to correctly investigate haskell binaries...)' + my_timed: '23 0 * * *' + my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' + my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' + my_node: 'profitbricks2' - '{name}_setup_schroot_unstable': my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.' my_timed: '23 1 * * *' @@ -559,6 +631,12 @@ my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' my_node: 'profitbricks1' + - '{name}_setup_schroot_unstable_amd64_profitbricks2': + my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.' + my_timed: '23 1 * * *' + my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' + my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' + my_node: 'profitbricks2' - '{name}_setup_schroot_testing': my_description: 'Setup testing schroot for fetching source packages for the builder jobs.' my_timed: '23 1 * * *' @@ -595,6 +673,12 @@ my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' my_node: 'profitbricks1' + - '{name}_setup_schroot_testing_amd64_profitbricks2': + my_description: 'Setup testing schroot for fetching source packages for the builder jobs.' + my_timed: '23 1 * * *' + my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' + my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' + my_node: 'profitbricks2' - '{name}_setup_schroot_experimental': my_description: 'Setup experimental schroot for fetching source packages for the builder jobs.' my_timed: '23 1 * * *' @@ -607,6 +691,12 @@ my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' my_node: 'profitbricks1' + - '{name}_setup_schroot_experimental_amd64_profitbricks2': + my_description: 'Setup experimental schroot for fetching source packages for the builder jobs.' + my_timed: '23 1 * * *' + my_shell: '/srv/jenkins/bin/reproducible_master_wrapper.sh' + my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' + my_node: 'profitbricks2' - '{name}_scheduler': my_description: 'Schedule packages to be tested for reproducibility.' my_timed: '42 * * * *' |