reproducible: finally working version. now fails due to a (known) problem in reproducible build toolchain. added TODO for jenkins jobs. added new job to test all core packages (one day)

4 files changed, 35 insertions, 14 deletions

diff --git a/TODO b/TODO
index cd7d9227..3a5ab4ba 100644
--- a/TODO
+++ b/TODO
@@ -97,6 +97,16 @@ general
 * use jigdo to download images - but no edu jigdo images.
 * still havent got http://www.os-autoinst.org/ out of my mind
 
+=== reproducible
+
+* pbuilder results cleanup
+* pbuilder cleanup cache
+* time based triggers reproducible build jobs
+* use git repo in job config?
+* implement logic / counter for getting results from testing 100 packages in one job
+* job to build minimal debootstrap packages
+* job to build all build-essential packages
+
 === Test Debian live
 
 * daily build from sid
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index e8733435..07f4056f 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -25,7 +25,9 @@ for PACKAGE in "$@" ; do
 	dcmd cp /var/cache/pbuilder/result/${PACKAGE}_*.changes b2
 	sudo dcmd rm /var/cache/pbuilder/result/${PACKAGE}_*.changes
 
-	./misc.git/diffp b1/*.changes b2/*.changes
+	TMPFILE=$(mktemp)
+	./misc.git/diffp b1/*.changes b2/*.changes | tee $TMPFILE
+	grep -qv '^\*\*\*\*\*' $TMPFILE && figlet ${PACKAGE}_&& echo && echo "${PACKAGE} build successfull."
 
-	rm b1 b2 -rf
+	rm b1 b2 $TMPFILE -rf
 done
diff --git a/bin/reproducible_setup.sh b/bin/reproducible_setup.sh
index 48e12698..8ca78b9a 100755
--- a/bin/reproducible_setup.sh
+++ b/bin/reproducible_setup.sh
@@ -14,6 +14,6 @@ echo "Warning: Usage of --force-yes to override the apt authentication warning.
 apt-get install --force-yes -y dpkg dpkg-dev debhelper dh-python proot
 EOF
 
-sudo pbuilder --create /var/cache/pbuilder/base-reproducible.tgz --distribution sid
-sudo pbuilder --execute $TMPFILE --save-after-exec --basetgz /var/cache/pbuilder/base-reproducible.tgz
+sudo pbuilder --create --basetgz /var/cache/pbuilder/base-reproducible.tgz --distribution sid
+sudo pbuilder --execute --save-after-exec --basetgz /var/cache/pbuilder/base-reproducible.tgz --
 rm $TMPFILE
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index 6dbb9965..9b54a029 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -14,7 +14,7 @@
           url: http://www.profitbricks.com
           text: Sponsored by Profitbricks
           icon: /userContent/images/profitbricks-24x24.png
-    description: '{my_description}<br><br>Job configuration source is <a href="http://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/job-cfg/sid-reproducible.yaml">sid-reproducible.yaml</a>.'
+    description: '{my_description}<br><br>Job configuration source is <a href="http://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/job-cfg/reproducible.yaml">reproducible.yaml</a>.'
     logrotate:
       daysToKeep: 180
       numToKeep: 100
@@ -28,21 +28,28 @@
     name: '{name}_setup'
     publishers:
       - email:
-          recipients: 'jenkins+sid-reproducible holger@layer-acht.org lunar@debian.org'
+          recipients: 'jenkins+reproducible holger@layer-acht.org lunar@debian.org'
 
 - job-template:
     defaults: reproducible
-    name: '{name}_build_core'
+    name: '{name}_build_core_good'
     publishers:
       - email:
-          recipients: 'jenkins+sid-reproducible holger@layer-acht.org' # FIXME add lunar here again
+          recipients: 'jenkins+reproducible holger@layer-acht.org lunar@debian.org'
 
 - job-template:
     defaults: reproducible
     name: '{name}_build_core_fail'
     publishers:
       - email:
-          recipients: 'jenkins+sid-reproducible holger@layer-acht.org' # FIXME add lunar here again
+          recipients: 'jenkins+reproducible holger@layer-acht.org lunar@debian.org'
+
+- job-template:
+    defaults: reproducible
+    name: '{name}_build_core'
+    publishers:
+      - email:
+          recipients: 'jenkins+reproducible holger@layer-acht.org lunar@debian.org'
 
 - project:
     name: reproducible
@@ -50,11 +57,13 @@
         - '{name}_setup':
             my_description: 'Setup pbuilder for reproducible builds as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
             my_shell: '/srv/jenkins/bin/reproducible_setup.sh'
-        - '{name}_build_core':
-            my_description: 'Build one package with pbuilder so far....'
+        - '{name}_build_example_good':
+            my_description: 'Build one package (one we know will succeed) with pbuilder so far....'
             my_shell: '/srv/jenkins/bin/reproducible_build.sh lsb'
-        - '{name}_build_core_fail':
-            my_description: 'Build one package with pbuilder so far....'
+        - '{name}_build_example_fail':
+            my_description: 'Build one package (one we know will fail) with pbuilder so far....'
             my_shell: '/srv/jenkins/bin/reproducible_build.sh lsof'
-            #my_shell: '/srv/jenkins/bin/reproducible_build.sh acl adduser base-files base-passwd bash binutils bsdmainutils bsdutils build-essential bzip2 coreutils cpp cpp-4.9 dash debconf debhelper debianutils dh-buildinfo diffutils dmsetup dpkg dpkg-dev e2fslibs e2fsprogs file findutils g++ g++-4.9 gcc gcc-4.9 gcc-4.9-base gettext gettext-base grep groff-base gzip hostname init initscripts insserv intltool-debian libacl1 libasan1 libasprintf0c2 libatomic1 libattr1 libaudit1 libaudit-common libblkid1 libbz2-1.0 libc6 libc6-dev libcap2 libcap2-bin libc-bin libc-dev-bin libcilkrts5 libcloog-isl4 libcomerr2 libcroco3 libcryptsetup4 libdb5.3 libdbus-1-3 libdebconfclient0 libdevmapper1.02.1 libdpkg-perl libffi6 libgcc1 libgcc-4.9-dev libgcrypt11 libgcrypt20 libgdbm3 libglib2.0-0 libgmp10 libgomp1 libgpg-error0 libintl-perl libisl10 libitm1 libkmod2 liblzma5 libmagic1 libmount1 libmpc3 libmpfr4 libncurses5 libncurses5-dev libncursesw5 libpam0g libpam-modules libpam-modules-bin libpam-runtime libpcre3 libpipeline1 libprocps3 libquadmath0 libselinux1 libsemanage1 libsemanage-common libsepol1 libss2 libstdc++-4.9-dev libstdc++6 libsystemd-journal0 libsystemd-login0 libtext-unidecode-perl libtimedate-perl libtinfo5 libtinfo-dev libubsan0 libudev1 libunistring0 libustr-1.0-1 libuuid1 libwrap0 libxml2 libxml-libxml-perl libxml-namespacesupport-perl libxml-sax-base-perl libxml-sax-perl linux-libc-dev login lsb-base make man-db mawk mount ncurses-base ncurses-bin passwd patch perl perl-base perl-modules po-debconf procps sed sensible-utils startpar systemd systemd-sysv sysvinit-utils sysv-rc tar texinfo tzdata ucf udev util-linux xz-utils zlib1g'
+        - '{name}_build_core':
+            my_description: 'Build all the core packages as defined by UDD as described in https://wiki.debian.org/ReproducibleBuilds'
+            my_shell: '/srv/jenkins/bin/reproducible_build.sh acl adduser base-files base-passwd bash binutils bsdmainutils bsdutils build-essential bzip2 coreutils cpp cpp-4.9 dash debconf debhelper debianutils dh-buildinfo diffutils dmsetup dpkg dpkg-dev e2fslibs e2fsprogs file findutils g++ g++-4.9 gcc gcc-4.9 gcc-4.9-base gettext gettext-base grep groff-base gzip hostname init initscripts insserv intltool-debian libacl1 libasan1 libasprintf0c2 libatomic1 libattr1 libaudit1 libaudit-common libblkid1 libbz2-1.0 libc6 libc6-dev libcap2 libcap2-bin libc-bin libc-dev-bin libcilkrts5 libcloog-isl4 libcomerr2 libcroco3 libcryptsetup4 libdb5.3 libdbus-1-3 libdebconfclient0 libdevmapper1.02.1 libdpkg-perl libffi6 libgcc1 libgcc-4.9-dev libgcrypt11 libgcrypt20 libgdbm3 libglib2.0-0 libgmp10 libgomp1 libgpg-error0 libintl-perl libisl10 libitm1 libkmod2 liblzma5 libmagic1 libmount1 libmpc3 libmpfr4 libncurses5 libncurses5-dev libncursesw5 libpam0g libpam-modules libpam-modules-bin libpam-runtime libpcre3 libpipeline1 libprocps3 libquadmath0 libselinux1 libsemanage1 libsemanage-common libsepol1 libss2 libstdc++-4.9-dev libstdc++6 libsystemd-journal0 libsystemd-login0 libtext-unidecode-perl libtimedate-perl libtinfo5 libtinfo-dev libubsan0 libudev1 libunistring0 libustr-1.0-1 libuuid1 libwrap0 libxml2 libxml-libxml-perl libxml-namespacesupport-perl libxml-sax-base-perl libxml-sax-perl linux-libc-dev login lsb-base make man-db mawk mount ncurses-base ncurses-bin passwd patch perl perl-base perl-modules po-debconf procps sed sensible-utils startpar systemd systemd-sysv sysvinit-utils sysv-rc tar texinfo tzdata ucf udev util-linux xz-utils zlib1g'