reproducible debian: introduce buster

Signed-off-by: Mattia Rizzolo <mattia@debian.org>
Signed-off-by: Holger Levsen <holger@layer-acht.org>

12 files changed, 69 insertions, 23 deletions

diff --git a/bin/jenkins_node_wrapper.sh b/bin/jenkins_node_wrapper.sh
index eab69738..61674505 100755
--- a/bin/jenkins_node_wrapper.sh
+++ b/bin/jenkins_node_wrapper.sh
@@ -2,6 +2,7 @@
 
 # Copyright (c) 2009, 2010, 2012, 2015 Peter Palfrader
 #               2015-2017 Holger Levsen
+#               2017      Mattia Rizzolo <mattia@debian.org>
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -101,6 +102,8 @@ elif [[ "$*" =~ ^rm\ -r\ /srv/reproducible-results/rbuild.* ]] ; then
 	exec rm -r "$3" ; croak "Exec failed";
 elif [[ "$*" =~ ^rm\ -r\ /var/lib/jenkins/jobs/.*/workspace/results ]] ; then
 	exec rm -r "$3" ; croak "Exec failed";
+elif [[ "$*" =~ ^reproducible_setup_pbuilder_buster_.*_.* ]] ; then
+	exec /srv/jenkins/bin/reproducible_setup_pbuilder.sh buster ; croak "Exec failed";
 elif [[ "$*" =~ ^reproducible_setup_pbuilder_unstable_.*_.* ]] ; then
 	exec /srv/jenkins/bin/reproducible_setup_pbuilder.sh unstable ; croak "Exec failed";
 elif [[ "$*" =~ ^reproducible_setup_pbuilder_stretch_.*_.* ]] ; then
@@ -113,6 +116,8 @@ elif [[ "$*" =~ ^reproducible_node_health_check_.*_.* ]] ; then
 	exec /srv/jenkins/bin/reproducible_node_health_check.sh ; croak "Exec failed";
 elif [[ "$*" =~ ^reproducible_setup_schroot_unstable_diffoscope_.*_.* ]] ; then
 	exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable-diffoscope unstable diffoscope locales-all ; croak "Exec failed";
+elif [[ "$*" =~ ^reproducible_setup_schroot_buster_.*_.* ]] ; then
+	exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-buster buster ; croak "Exec failed";
 elif [[ "$*" =~ ^reproducible_setup_schroot_unstable_.*_.* ]] ; then
 	exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable unstable botch ; croak "Exec failed";
 elif [[ "$*" =~ ^reproducible_setup_schroot_stretch_.*_.* ]] ; then
diff --git a/bin/reproducible_blacklist.sh b/bin/reproducible_blacklist.sh
index e60639ef..7a2c5c05 100755
--- a/bin/reproducible_blacklist.sh
+++ b/bin/reproducible_blacklist.sh
@@ -74,7 +74,7 @@ case $SUITE in
 	sid) 	echo "WARNING: sid has been renamed to unstable."
 		SUITE=unstable
 		;;
-	stretch|unstable|experimental) ;;
+	stretch|buster|unstable|experimental) ;;
 	*)	echo "$SUITE is not a valid suite".
 		explain_syntax
 		exit 1
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index 5185a12a..2746c066 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -133,7 +133,7 @@ update_db_and_html() {
 	local OLD_STATUS=$(query_db "SELECT status FROM results WHERE package_id='${SRCPKGID}'" || \
 			   query_db "SELECT status FROM results WHERE package_id='${SRCPKGID}'")
 	# irc+mail notifications for changing status in unstable and experimental
-	if [ "$SUITE" != "stretch" ] ; then
+	if [ "$SUITE" = "unstable" ] || [ "$SUITE" = "experimental" ] ; then
 		if [ "${OLD_STATUS}" = "reproducible" ] && [ "$STATUS" != "depwait" ] && \
 		  ( [ "$STATUS" = "unreproducible" ] || [ "$STATUS" = "FTBFS" ] ) ; then
 			MESSAGE="${DEBIAN_URL}/${SUITE}/${ARCH}/${SRCPACKAGE} : reproducible ➤ ${STATUS}"
@@ -626,8 +626,8 @@ export LANG="C"
 unset LC_ALL
 export LANGUAGE="en_US:en"
 EOF
-	# build path is not yet varied on stretch
-	if [ "${SUITE}" != "stretch" ]; then
+	# build path is only varied on unstable and experimental
+	if [ "${SUITE}" = "unstable" ] || [ "$SUITE" = "experimental" ]; then
 		echo "BUILDDIR=/build/1st" >> "$TMPCFG"
 	else
 		echo "BUILDDIR=/build" >> "$TMPCFG"
@@ -689,8 +689,8 @@ export LC_ALL="$locale.UTF-8"
 export LANGUAGE="$locale:$language"
 umask 0002
 EOF
-	# build path is not yet varied on stretch
-	if [ "${SUITE}" != "stretch" ]; then
+	# build path is only varied on unstable and experimental
+	if [ "${SUITE}" = "unstable" ] || [ "$SUITE" = "experimental" ]; then
 		local src_dir_name="$(perl -mDpkg::Source::Package -e '$_ = Dpkg::Source::Package->new(filename => $ARGV[0])->get_basename; s/_/-/g; print' -- "${SRCPACKAGE}_${EVERSION}.dsc")"
 		echo "BUILDDIR=/build/$src_dir_name" >> "$TMPCFG"
 		echo "BUILDSUBDIR=2nd" >> "$TMPCFG"
diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py
index 368b70c2..afcadc42 100755
--- a/bin/reproducible_common.py
+++ b/bin/reproducible_common.py
@@ -39,7 +39,7 @@ if os.uname()[1] == 'jenkins-test-vm':
     sys.exit()
 
 # tested suites
-SUITES = ['stretch', 'unstable', 'experimental']
+SUITES = ['buster', 'unstable', 'stretch', 'experimental']
 # tested architectures
 ARCHS = ['amd64', 'i386', 'arm64', 'armhf']
 # defaults
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index 0ffbc05a..948fac4f 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -32,7 +32,7 @@ BIN_PATH=/srv/jenkins/bin
 TEMPLATE_PATH=/srv/jenkins/mustache-templates/reproducible
 
 # Debian suites being tested
-SUITES="stretch unstable experimental"
+SUITES="buster unstable stretch experimental"
 # Debian architectures being tested
 ARCHS="amd64 i386 arm64 armhf"
 
@@ -416,7 +416,7 @@ write_variation_table() {
 		write_page "<tr><td>uid</td><td>uid=1111</td><td>uid=2222</td></tr>"
 		write_page "<tr><td>gid</td><td>gid=1111</td><td>gid=2222</td></tr>"
 		write_page "<tr><td>/bin/sh</td><td>/bin/dash</td><td>/bin/bash</td></tr>"
-		write_page "<tr><td>build path</td><td>/build/1st/\$pkg-\$ver <em>(not varied for stretch)</em></td><td>/build/\$pkg-\$ver/2nd <em>(not varied for stretch)</em></td></tr>"
+		write_page "<tr><td>build path</td><td>/build/1st/\$pkg-\$ver <em>(not varied for stretch/buster)</em></td><td>/build/\$pkg-\$ver/2nd <em>(not varied for stretch/buster)</em></td></tr>"
 		write_page "<tr><td>user's login shell</td><td>/bin/sh</td><td>/bin/bash</td></tr>"
 		write_page "<tr><td>user's <a href="https://en.wikipedia.org/wiki/Gecos_field">GECOS</a></td><td>first user,first room,first work-phone,first home-phone,first other</td><td>second user,second room,second work-phone,second home-phone,second other</td></tr>"
 		write_page "<tr><td>env DEB_BUILD_OPTIONS</td><td>DEB_BUILD_OPTIONS=\"parallel=XXX\"<br />&nbsp;&nbsp;XXX on amd64: 16 or 15<br />&nbsp;&nbsp;XXX on i386: 10 or 9<br />&nbsp;&nbsp;XXX on armhf: 8, 4 or 2</td><td>DEB_BUILD_OPTIONS=\"parallel=YYY\"<br />&nbsp;&nbsp;YYY on amd64: 16 or 15 (!= the first build)<br />&nbsp;&nbsp;YYY on i386: 10 or 9 (!= the first build)<br />&nbsp;&nbsp;YYY is the same as XXX on arm64<br />&nbsp;&nbsp;YYY on armhf: 8, 4, or 2 (not varied systematically)</td></tr>"
diff --git a/bin/reproducible_db_maintenance.py b/bin/reproducible_db_maintenance.py
index 825a38ae..27f3f56c 100755
--- a/bin/reproducible_db_maintenance.py
+++ b/bin/reproducible_db_maintenance.py
@@ -606,6 +606,26 @@ schema_updates = {
                 "stats_builds_age", "stats_meta_pkg_state", "stats_build")] + [
         "INSERT INTO rb_schema (version, date) VALUES (31, '" + now + "')"
     ],
+    32: [ # copy stretch packages (includng results) in buster
+        """INSERT INTO sources (name, version, suite, architecture, notify_maintainer)
+            SELECT name, version, 'buster', architecture, notify_maintainer
+            FROM sources
+            WHERE suite = 'stretch'"""
+        """WITH buster AS (
+                SELECT id, name, suite, architecture, version
+                FROM sources WHERE suite = 'buster'),
+            sr AS (
+                SELECT s.name, s.architecture, r.id, r.version, r.status,
+                    r.build_date, r.build_duration, r.node1, r.node2, r.job
+                FROM sources AS s JOIN results AS r ON s.id=r.package_id
+                WHERE s.suite = 'stretch')
+            INSERT INTO results (package_id, version, status, build_date,
+                    build_duration, node1, node2, job)
+                SELECT b.id, sr.version, sr.status, sr.build_date,
+                    sr.build_duration, sr.node1, sr.node2, sr.job
+                FROM buster AS b JOIN sr ON b.name=sr.name
+                    AND b.architecture=sr.architecture""",
+        "INSERT INTO rb_schema (version, date) VALUES (32, '" + now + "')"
 }
 
 
diff --git a/bin/reproducible_html_breakages.py b/bin/reproducible_html_breakages.py
index 929da615..708056f9 100755
--- a/bin/reproducible_html_breakages.py
+++ b/bin/reproducible_html_breakages.py
@@ -109,12 +109,13 @@ def lack_buildinfo():
 def pbuilder_dep_fail():
     log.info('running pbuilder_dep_fail check...')
     bad_pkgs = []
-    # we only care about these failures in the stretch suite as they happen
-    # all the time in other suites, as packages are buggy
+    # we only care about these failures in the !unstable !experimental suites
+    # as they happen all the time in there, as packages are buggy
     # and specific versions also come and go
     query = '''SELECT s.name, r.version, s.suite, s.architecture
                FROM sources AS s JOIN results AS r ON r.package_id=s.id
-               WHERE r.status = 'FTBFS' AND s.suite = 'stretch'
+               WHERE r.status = 'FTBFS'
+               AND s.suite NOT IN ('unstable', 'experimental')
                ORDER BY s.name ASC, s.suite DESC, s.architecture ASC'''
     results = query_db(query)
     for pkg, version, suite, arch in results:
diff --git a/bin/reproducible_json.py b/bin/reproducible_json.py
index 33e218b6..b8bd2866 100755
--- a/bin/reproducible_json.py
+++ b/bin/reproducible_json.py
@@ -48,8 +48,8 @@ for row in result:
     log.debug(pkg)
     output.append(pkg)
 
-    # tracker.d.o should only care about results in stretch
-    if pkg['suite'] == 'stretch':
+    # tracker.d.o should only care about results in testing
+    if pkg['suite'] == 'buster':
 
         package = pkg['package']
         if package in crossarch:
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index ffc5c804..3e508523 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -419,7 +419,7 @@ fi
 
 # find + chmod files with bad permissions
 echo "$(date -u) - Checking for files with bad permissions."
-BADPERMS=$(find $DEBIAN_BASE/{buildinfo,dbd,rbuild,artifacts,unstable,experimental,stretch,rb-pkg} ! -perm 644 -type f 2>/dev/null|| true)
+BADPERMS=$(find $DEBIAN_BASE/{buildinfo,dbd,rbuild,artifacts,stretch,buster,unstable,experimental,rb-pkg} ! -perm 644 -type f 2>/dev/null|| true)
 if [ ! -z "$BADPERMS" ] ; then
     DIRTY=true
     echo
diff --git a/bin/reproducible_scheduler.py b/bin/reproducible_scheduler.py
index 9138e174..686db7e8 100755
--- a/bin/reproducible_scheduler.py
+++ b/bin/reproducible_scheduler.py
@@ -41,7 +41,7 @@ packages are already scheduled in that category, in a 3 steps process.
 
 Only when scheduling old versions MINIMUM_AGE is respected.
 
-
+(
 Let's go by an example:
     'unstable': {1: (250, 40), 2: (350, 20), '*': 5},
 is translated to:
@@ -72,21 +72,25 @@ LIMITS = {
     'untested': {
         'amd64': {
             'stretch': {'*': 100},
+            'buster': {'*': 100},
             'unstable': {'*': 100},
             'experimental': {'*': 100},
         },
         'i386': {
             'stretch': {'*': 100},
+            'buster': {'*': 100},
             'unstable': {'*': 100},
             'experimental': {'*': 100},
         },
        'arm64': {
             'stretch': {'*': 100},
+            'buster': {'*': 100},
             'unstable': {'*': 100},
             'experimental': {'*': 100},
         },
        'armhf': {
             'stretch': {'*': 100},
+            'buster': {'*': 100},
             'unstable': {'*': 100},
             'experimental': {'*': 100},
         },
@@ -94,21 +98,25 @@ LIMITS = {
     'new': {
         'amd64': {
             'stretch': {1: (100, 250), 2: (200, 200), '*': 100},
+            'buster': {1: (100, 250), 2: (200, 200), '*': 100},
             'unstable': {1: (100, 250), 2: (200, 200), '*': 150},
             'experimental': {1: (100, 250), 2: (200, 200), '*': 50},
         },
         'i386': {
             'stretch': {1: (100, 250), 2: (200, 200), '*': 100},
+            'buster': {1: (100, 250), 2: (200, 200), '*': 100},
             'unstable': {1: (100, 250), 2: (200, 200), '*': 150},
             'experimental': {1: (100, 250), 2: (200, 200), '*': 50},
         },
         'arm64': {
             'stretch': {1: (100, 250), 2: (200, 200), '*': 50},
+            'buster': {1: (100, 250), 2: (200, 200), '*': 50},
             'unstable': {1: (100, 250), 2: (200, 200), '*': 75},
             'experimental': {1: (100, 200), 2: (200, 200), '*': 25},
         },
         'armhf': {
             'stretch': {1: (100, 200), 2: (200, 200), '*': 50},
+            'buster': {1: (100, 200), 2: (200, 200), '*': 50},
             'unstable': {1: (100, 200), 2: (200, 200), '*': 75},
             'experimental': {1: (100, 200), 2: (200, 200), '*': 25},
         },
@@ -116,21 +124,25 @@ LIMITS = {
     'ftbfs': {
         'amd64': {
             'stretch': {1: (700, 40), 2: (500, 20), '*': 5},
+            'buster': {1: (700, 40), 2: (500, 20), '*': 5},
             'unstable': {1: (700, 40), 2: (500, 20), '*': 5},
             'experimental': {1: (700, 40), 2: (500, 20), '*': 2},
         },
         'i386': {
             'stretch': {1: (700, 40), 2: (500, 20), '*': 5},
+            'buster': {1: (700, 40), 2: (500, 20), '*': 5},
             'unstable': {1: (700, 40), 2: (500, 20), '*': 5},
             'experimental': {1: (700, 40), 2: (500, 20), '*': 2},
         },
         'arm64': {
             'stretch': {1: (700, 40), 2: (500, 20), '*': 5},
+            'buster': {1: (700, 40), 2: (500, 20), '*': 5},
             'unstable': {1: (700, 40), 2: (500, 20), '*': 5},
             'experimental': {1: (700, 40), 2: (500, 20), '*': 2},
         },
         'armhf': {
             'stretch': {1: (575, 20), 2: (450, 10), '*': 5},
+            'buster': {1: (575, 20), 2: (450, 10), '*': 5},
             'unstable': {1: (575, 20), 2: (450, 10), '*': 5},
             'experimental': {1: (575, 20), 2: (450, 10), '*': 2},
         }
@@ -138,21 +150,25 @@ LIMITS = {
     'depwait': {
         'amd64': {
             'stretch': {1: (700, 400), 2: (500, 200), '*': 50},
+            'buster': {1: (700, 400), 2: (500, 200), '*': 50},
             'unstable': {1: (700, 400), 2: (500, 200), '*': 50},
             'experimental': {1: (700, 400), 2: (500, 200), '*': 20},
         },
         'i386': {
             'stretch': {1: (700, 400), 2: (500, 200), '*': 50},
+            'buster': {1: (700, 400), 2: (500, 200), '*': 50},
             'unstable': {1: (700, 400), 2: (500, 200), '*': 50},
             'experimental': {1: (700, 400), 2: (500, 200), '*': 20},
         },
         'arm64': {
             'stretch': {1: (700, 400), 2: (500, 200), '*': 50},
+            'buster': {1: (700, 400), 2: (500, 200), '*': 50},
             'unstable': {1: (700, 400), 2: (500, 200), '*': 50},
             'experimental': {1: (700, 400), 2: (500, 200), '*': 20},
         },
         'armhf': {
             'stretch': {1: (575, 200), 2: (450, 100), '*': 50},
+            'buster': {1: (575, 200), 2: (450, 100), '*': 50},
             'unstable': {1: (575, 200), 2: (450, 100), '*': 50},
             'experimental': {1: (575, 200), 2: (450, 100), '*': 20},
         }
@@ -160,21 +176,25 @@ LIMITS = {
     'old': {
         'amd64': {
             'stretch': {1: (700, 1000), 2: (1100, 850), '*': 0},
+            'buster': {1: (700, 1000), 2: (1100, 850), '*': 0},
             'unstable': {1: (700, 1200), 2: (1100, 1050), '*': 0},
             'experimental': {1: (700, 70), 2: (1100, 50), '*': 0},
         },
         'i386': {
             'stretch': {1: (700, 1000), 2: (1100, 850), '*': 0},
+            'buster': {1: (700, 1000), 2: (1100, 850), '*': 0},
             'unstable': {1: (700, 1200), 2: (1100, 1050), '*': 0},
             'experimental': {1: (700, 70), 2: (1100, 50), '*': 0},
         },
         'arm64': {
             'stretch': {1: (700, 1000), 2: (1100, 850), '*': 0},
+            'buster': {1: (700, 1000), 2: (1100, 850), '*': 0},
             'unstable': {1: (700, 1200), 2: (1100, 1050), '*': 0},
             'experimental': {1: (700, 70), 2: (1100, 50), '*': 0},
         },
         'armhf': {
             'stretch': {1: (600, 800), 2: (850, 500), '*': 0},
+            'buster': {1: (600, 800), 2: (850, 500), '*': 0},
             'unstable': {1: (600, 1000), 2: (850, 700), '*': 0},
             'experimental': {1: (600, 70), 2: (850, 50), '*': 0},
         }
@@ -449,7 +469,7 @@ def query_new_versions(suite, arch, limit):
                LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit)
     pkgs = query_db(query)
     # the next line avoids constant rescheduling of packages:
-    # packages in our repository != sid or stretch,
+    # packages in our repository != official repo,
     # so they will always be selected by the query above
     # so we only accept them if there version is greater than the already tested one
     packages = [(x[0], x[1]) for x in pkgs if version_compare(x[2], x[3]) > 0]
diff --git a/bin/reproducible_setup_pbuilder.sh b/bin/reproducible_setup_pbuilder.sh
index c64e2fa9..43943d5c 100755
--- a/bin/reproducible_setup_pbuilder.sh
+++ b/bin/reproducible_setup_pbuilder.sh
@@ -127,8 +127,8 @@ setup_pbuilder() {
 	sudo pbuilder --execute $pbuilder_http_proxy --save-after-exec --basetgz /var/cache/pbuilder/${NAME}-new.tgz -- ${TMPFILE} | tee ${LOG}
 	rm ${TMPFILE}
 
-	# add repo only for experimental and sid - keep stretch "real" (and sid progressive!)
-	if [ "$SUITE" != "stretch" ] ; then
+	# add repo only for experimental and unstable - keep stretch/buster "real" (and sid progressive!)
+	if [ "$SUITE" = "unstable" ] || [ "$SUITE" = "experimental" ]; then
 		# apply further customisations, eg. install $PACKAGES from our repo
 		create_setup_our_repo_tmpfile ${TMPFILE} "${PACKAGES}"
 		if [ "$DEBUG" = "true" ] ; then
diff --git a/bin/schroot-create.sh b/bin/schroot-create.sh
index 94a2fbb6..e956adce 100755
--- a/bin/schroot-create.sh
+++ b/bin/schroot-create.sh
@@ -182,14 +182,14 @@ bootstrap() {
 		fi
 		robust_chroot_apt install -y --no-install-recommends sudo
 		robust_chroot_apt install -y --no-install-recommends $@
-		# try to use diffoscope from experimental
-		if ([ "$SUITE" = "unstable" ] || [ "$SUITE" = "stretch" ] ) && [ "$1" = "diffoscope" ] ; then
+		# try to use diffoscope from experimental if available
+		if [ "$SUITE" != "experimental" ] && [ "$1" = "diffoscope" ] ; then
 			echo "deb $MIRROR experimental main"        | sudo tee -a $SCHROOT_TARGET/etc/apt/sources.list > /dev/null
 			robust_chroot_apt update
 			# install diffoscope from experimental without re-adding all recommends...
 			sudo chroot $SCHROOT_TARGET apt-get install -y -t experimental --no-install-recommends diffoscope || echo "Warning: diffoscope from experimental is uninstallable at the moment."
-		elif [ "$SUITE" = "stretch" ] && [ "$1" = "diffoscope" ] ; then
-			# always try to use diffoscope from unstable on stretch
+        elif ([ "$SUITE" != "unstable" ] && [ "$SUITE" != "experimental" ]) && [ "$1" = "diffoscope" ] ; then
+			# always try to use diffoscope from unstable on stretch/buster
 			echo "deb $MIRROR unstable main"        | sudo tee -a $SCHROOT_TARGET/etc/apt/sources.list > /dev/null
 			robust_chroot_apt update
 			# install diffoscope from unstable without re-adding all recommends...