diff options
| -rw-r--r-- | TODO4stretch_rename.txt | 7 | ||||
| -rwxr-xr-x | bin/jenkins_node_wrapper.sh | 5 | ||||
| -rwxr-xr-x | bin/reproducible_blacklist.sh | 2 | ||||
| -rwxr-xr-x | bin/reproducible_build.sh | 10 | ||||
| -rwxr-xr-x | bin/reproducible_common.py | 2 | ||||
| -rwxr-xr-x | bin/reproducible_common.sh | 4 | ||||
| -rwxr-xr-x | bin/reproducible_db_maintenance.py | 20 | ||||
| -rwxr-xr-x | bin/reproducible_html_breakages.py | 7 | ||||
| -rwxr-xr-x | bin/reproducible_json.py | 4 | ||||
| -rwxr-xr-x | bin/reproducible_maintenance.sh | 2 | ||||
| -rwxr-xr-x | bin/reproducible_scheduler.py | 24 | ||||
| -rwxr-xr-x | bin/reproducible_setup_pbuilder.sh | 4 | ||||
| -rwxr-xr-x | bin/schroot-create.sh | 8 | ||||
| -rw-r--r-- | hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf | 4 |
14 files changed, 76 insertions, 27 deletions
diff --git a/TODO4stretch_rename.txt b/TODO4stretch_rename.txt index af8c737d..231106b2 100644 --- a/TODO4stretch_rename.txt +++ b/TODO4stretch_rename.txt @@ -4,6 +4,9 @@ special deployement needed for this: changes not yet prepared in git ------------------------------- -completly prepare to add buster on 2017-06-17 -(including changing the apache redirect testing→stretch to testing→buster) +missing bits for the buster coming: + * _common.sh + * _dashboard.sh + * _repository_comparison.sh + * create the needed jenkins jobs finally, remove this file again. diff --git a/bin/jenkins_node_wrapper.sh b/bin/jenkins_node_wrapper.sh index eab69738..61674505 100755 --- a/bin/jenkins_node_wrapper.sh +++ b/bin/jenkins_node_wrapper.sh @@ -2,6 +2,7 @@ # Copyright (c) 2009, 2010, 2012, 2015 Peter Palfrader # 2015-2017 Holger Levsen +# 2017 Mattia Rizzolo <mattia@debian.org> # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -101,6 +102,8 @@ elif [[ "$*" =~ ^rm\ -r\ /srv/reproducible-results/rbuild.* ]] ; then exec rm -r "$3" ; croak "Exec failed"; elif [[ "$*" =~ ^rm\ -r\ /var/lib/jenkins/jobs/.*/workspace/results ]] ; then exec rm -r "$3" ; croak "Exec failed"; +elif [[ "$*" =~ ^reproducible_setup_pbuilder_buster_.*_.* ]] ; then + exec /srv/jenkins/bin/reproducible_setup_pbuilder.sh buster ; croak "Exec failed"; elif [[ "$*" =~ ^reproducible_setup_pbuilder_unstable_.*_.* ]] ; then exec /srv/jenkins/bin/reproducible_setup_pbuilder.sh unstable ; croak "Exec failed"; elif [[ "$*" =~ ^reproducible_setup_pbuilder_stretch_.*_.* ]] ; then @@ -113,6 +116,8 @@ elif [[ "$*" =~ ^reproducible_node_health_check_.*_.* ]] ; then exec /srv/jenkins/bin/reproducible_node_health_check.sh ; croak "Exec failed"; elif [[ "$*" =~ ^reproducible_setup_schroot_unstable_diffoscope_.*_.* ]] ; then exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable-diffoscope unstable diffoscope locales-all ; croak "Exec failed"; +elif [[ "$*" =~ ^reproducible_setup_schroot_buster_.*_.* ]] ; then + exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-buster buster ; croak "Exec failed"; elif [[ "$*" =~ ^reproducible_setup_schroot_unstable_.*_.* ]] ; then exec /srv/jenkins/bin/schroot-create.sh reproducible reproducible-unstable unstable botch ; croak "Exec failed"; elif [[ "$*" =~ ^reproducible_setup_schroot_stretch_.*_.* ]] ; then diff --git a/bin/reproducible_blacklist.sh b/bin/reproducible_blacklist.sh index e60639ef..7a2c5c05 100755 --- a/bin/reproducible_blacklist.sh +++ b/bin/reproducible_blacklist.sh @@ -74,7 +74,7 @@ case $SUITE in sid) echo "WARNING: sid has been renamed to unstable." SUITE=unstable ;; - stretch|unstable|experimental) ;; + stretch|buster|unstable|experimental) ;; *) echo "$SUITE is not a valid suite". explain_syntax exit 1 diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh index 5185a12a..2746c066 100755 --- a/bin/reproducible_build.sh +++ b/bin/reproducible_build.sh @@ -133,7 +133,7 @@ update_db_and_html() { local OLD_STATUS=$(query_db "SELECT status FROM results WHERE package_id='${SRCPKGID}'" || \ query_db "SELECT status FROM results WHERE package_id='${SRCPKGID}'") # irc+mail notifications for changing status in unstable and experimental - if [ "$SUITE" != "stretch" ] ; then + if [ "$SUITE" = "unstable" ] || [ "$SUITE" = "experimental" ] ; then if [ "${OLD_STATUS}" = "reproducible" ] && [ "$STATUS" != "depwait" ] && \ ( [ "$STATUS" = "unreproducible" ] || [ "$STATUS" = "FTBFS" ] ) ; then MESSAGE="${DEBIAN_URL}/${SUITE}/${ARCH}/${SRCPACKAGE} : reproducible ➤ ${STATUS}" @@ -626,8 +626,8 @@ export LANG="C" unset LC_ALL export LANGUAGE="en_US:en" EOF - # build path is not yet varied on stretch - if [ "${SUITE}" != "stretch" ]; then + # build path is only varied on unstable and experimental + if [ "${SUITE}" = "unstable" ] || [ "$SUITE" = "experimental" ]; then echo "BUILDDIR=/build/1st" >> "$TMPCFG" else echo "BUILDDIR=/build" >> "$TMPCFG" @@ -689,8 +689,8 @@ export LC_ALL="$locale.UTF-8" export LANGUAGE="$locale:$language" umask 0002 EOF - # build path is not yet varied on stretch - if [ "${SUITE}" != "stretch" ]; then + # build path is only varied on unstable and experimental + if [ "${SUITE}" = "unstable" ] || [ "$SUITE" = "experimental" ]; then local src_dir_name="$(perl -mDpkg::Source::Package -e '$_ = Dpkg::Source::Package->new(filename => $ARGV[0])->get_basename; s/_/-/g; print' -- "${SRCPACKAGE}_${EVERSION}.dsc")" echo "BUILDDIR=/build/$src_dir_name" >> "$TMPCFG" echo "BUILDSUBDIR=2nd" >> "$TMPCFG" diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py index 368b70c2..afcadc42 100755 --- a/bin/reproducible_common.py +++ b/bin/reproducible_common.py @@ -39,7 +39,7 @@ if os.uname()[1] == 'jenkins-test-vm': sys.exit() # tested suites -SUITES = ['stretch', 'unstable', 'experimental'] +SUITES = ['buster', 'unstable', 'stretch', 'experimental'] # tested architectures ARCHS = ['amd64', 'i386', 'arm64', 'armhf'] # defaults diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 0ffbc05a..948fac4f 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -32,7 +32,7 @@ BIN_PATH=/srv/jenkins/bin TEMPLATE_PATH=/srv/jenkins/mustache-templates/reproducible # Debian suites being tested -SUITES="stretch unstable experimental" +SUITES="buster unstable stretch experimental" # Debian architectures being tested ARCHS="amd64 i386 arm64 armhf" @@ -416,7 +416,7 @@ write_variation_table() { write_page "<tr><td>uid</td><td>uid=1111</td><td>uid=2222</td></tr>" write_page "<tr><td>gid</td><td>gid=1111</td><td>gid=2222</td></tr>" write_page "<tr><td>/bin/sh</td><td>/bin/dash</td><td>/bin/bash</td></tr>" - write_page "<tr><td>build path</td><td>/build/1st/\$pkg-\$ver <em>(not varied for stretch)</em></td><td>/build/\$pkg-\$ver/2nd <em>(not varied for stretch)</em></td></tr>" + write_page "<tr><td>build path</td><td>/build/1st/\$pkg-\$ver <em>(not varied for stretch/buster)</em></td><td>/build/\$pkg-\$ver/2nd <em>(not varied for stretch/buster)</em></td></tr>" write_page "<tr><td>user's login shell</td><td>/bin/sh</td><td>/bin/bash</td></tr>" write_page "<tr><td>user's <a href="https://en.wikipedia.org/wiki/Gecos_field">GECOS</a></td><td>first user,first room,first work-phone,first home-phone,first other</td><td>second user,second room,second work-phone,second home-phone,second other</td></tr>" write_page "<tr><td>env DEB_BUILD_OPTIONS</td><td>DEB_BUILD_OPTIONS=\"parallel=XXX\"<br /> XXX on amd64: 16 or 15<br /> XXX on i386: 10 or 9<br /> XXX on armhf: 8, 4 or 2</td><td>DEB_BUILD_OPTIONS=\"parallel=YYY\"<br /> YYY on amd64: 16 or 15 (!= the first build)<br /> YYY on i386: 10 or 9 (!= the first build)<br /> YYY is the same as XXX on arm64<br /> YYY on armhf: 8, 4, or 2 (not varied systematically)</td></tr>" diff --git a/bin/reproducible_db_maintenance.py b/bin/reproducible_db_maintenance.py index 825a38ae..27f3f56c 100755 --- a/bin/reproducible_db_maintenance.py +++ b/bin/reproducible_db_maintenance.py @@ -606,6 +606,26 @@ schema_updates = { "stats_builds_age", "stats_meta_pkg_state", "stats_build")] + [ "INSERT INTO rb_schema (version, date) VALUES (31, '" + now + "')" ], + 32: [ # copy stretch packages (includng results) in buster + """INSERT INTO sources (name, version, suite, architecture, notify_maintainer) + SELECT name, version, 'buster', architecture, notify_maintainer + FROM sources + WHERE suite = 'stretch'""" + """WITH buster AS ( + SELECT id, name, suite, architecture, version + FROM sources WHERE suite = 'buster'), + sr AS ( + SELECT s.name, s.architecture, r.id, r.version, r.status, + r.build_date, r.build_duration, r.node1, r.node2, r.job + FROM sources AS s JOIN results AS r ON s.id=r.package_id + WHERE s.suite = 'stretch') + INSERT INTO results (package_id, version, status, build_date, + build_duration, node1, node2, job) + SELECT b.id, sr.version, sr.status, sr.build_date, + sr.build_duration, sr.node1, sr.node2, sr.job + FROM buster AS b JOIN sr ON b.name=sr.name + AND b.architecture=sr.architecture""", + "INSERT INTO rb_schema (version, date) VALUES (32, '" + now + "')" } diff --git a/bin/reproducible_html_breakages.py b/bin/reproducible_html_breakages.py index 929da615..708056f9 100755 --- a/bin/reproducible_html_breakages.py +++ b/bin/reproducible_html_breakages.py @@ -109,12 +109,13 @@ def lack_buildinfo(): def pbuilder_dep_fail(): log.info('running pbuilder_dep_fail check...') bad_pkgs = [] - # we only care about these failures in the stretch suite as they happen - # all the time in other suites, as packages are buggy + # we only care about these failures in the !unstable !experimental suites + # as they happen all the time in there, as packages are buggy # and specific versions also come and go query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id - WHERE r.status = 'FTBFS' AND s.suite = 'stretch' + WHERE r.status = 'FTBFS' + AND s.suite NOT IN ('unstable', 'experimental') ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: diff --git a/bin/reproducible_json.py b/bin/reproducible_json.py index 33e218b6..b8bd2866 100755 --- a/bin/reproducible_json.py +++ b/bin/reproducible_json.py @@ -48,8 +48,8 @@ for row in result: log.debug(pkg) output.append(pkg) - # tracker.d.o should only care about results in stretch - if pkg['suite'] == 'stretch': + # tracker.d.o should only care about results in testing + if pkg['suite'] == 'buster': package = pkg['package'] if package in crossarch: diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh index ffc5c804..3e508523 100755 --- a/bin/reproducible_maintenance.sh +++ b/bin/reproducible_maintenance.sh @@ -419,7 +419,7 @@ fi # find + chmod files with bad permissions echo "$(date -u) - Checking for files with bad permissions." -BADPERMS=$(find $DEBIAN_BASE/{buildinfo,dbd,rbuild,artifacts,unstable,experimental,stretch,rb-pkg} ! -perm 644 -type f 2>/dev/null|| true) +BADPERMS=$(find $DEBIAN_BASE/{buildinfo,dbd,rbuild,artifacts,stretch,buster,unstable,experimental,rb-pkg} ! -perm 644 -type f 2>/dev/null|| true) if [ ! -z "$BADPERMS" ] ; then DIRTY=true echo diff --git a/bin/reproducible_scheduler.py b/bin/reproducible_scheduler.py index 9138e174..686db7e8 100755 --- a/bin/reproducible_scheduler.py +++ b/bin/reproducible_scheduler.py @@ -41,7 +41,7 @@ packages are already scheduled in that category, in a 3 steps process. Only when scheduling old versions MINIMUM_AGE is respected. - +( Let's go by an example: 'unstable': {1: (250, 40), 2: (350, 20), '*': 5}, is translated to: @@ -72,21 +72,25 @@ LIMITS = { 'untested': { 'amd64': { 'stretch': {'*': 100}, + 'buster': {'*': 100}, 'unstable': {'*': 100}, 'experimental': {'*': 100}, }, 'i386': { 'stretch': {'*': 100}, + 'buster': {'*': 100}, 'unstable': {'*': 100}, 'experimental': {'*': 100}, }, 'arm64': { 'stretch': {'*': 100}, + 'buster': {'*': 100}, 'unstable': {'*': 100}, 'experimental': {'*': 100}, }, 'armhf': { 'stretch': {'*': 100}, + 'buster': {'*': 100}, 'unstable': {'*': 100}, 'experimental': {'*': 100}, }, @@ -94,21 +98,25 @@ LIMITS = { 'new': { 'amd64': { 'stretch': {1: (100, 250), 2: (200, 200), '*': 100}, + 'buster': {1: (100, 250), 2: (200, 200), '*': 100}, 'unstable': {1: (100, 250), 2: (200, 200), '*': 150}, 'experimental': {1: (100, 250), 2: (200, 200), '*': 50}, }, 'i386': { 'stretch': {1: (100, 250), 2: (200, 200), '*': 100}, + 'buster': {1: (100, 250), 2: (200, 200), '*': 100}, 'unstable': {1: (100, 250), 2: (200, 200), '*': 150}, 'experimental': {1: (100, 250), 2: (200, 200), '*': 50}, }, 'arm64': { 'stretch': {1: (100, 250), 2: (200, 200), '*': 50}, + 'buster': {1: (100, 250), 2: (200, 200), '*': 50}, 'unstable': {1: (100, 250), 2: (200, 200), '*': 75}, 'experimental': {1: (100, 200), 2: (200, 200), '*': 25}, }, 'armhf': { 'stretch': {1: (100, 200), 2: (200, 200), '*': 50}, + 'buster': {1: (100, 200), 2: (200, 200), '*': 50}, 'unstable': {1: (100, 200), 2: (200, 200), '*': 75}, 'experimental': {1: (100, 200), 2: (200, 200), '*': 25}, }, @@ -116,21 +124,25 @@ LIMITS = { 'ftbfs': { 'amd64': { 'stretch': {1: (700, 40), 2: (500, 20), '*': 5}, + 'buster': {1: (700, 40), 2: (500, 20), '*': 5}, 'unstable': {1: (700, 40), 2: (500, 20), '*': 5}, 'experimental': {1: (700, 40), 2: (500, 20), '*': 2}, }, 'i386': { 'stretch': {1: (700, 40), 2: (500, 20), '*': 5}, + 'buster': {1: (700, 40), 2: (500, 20), '*': 5}, 'unstable': {1: (700, 40), 2: (500, 20), '*': 5}, 'experimental': {1: (700, 40), 2: (500, 20), '*': 2}, }, 'arm64': { 'stretch': {1: (700, 40), 2: (500, 20), '*': 5}, + 'buster': {1: (700, 40), 2: (500, 20), '*': 5}, 'unstable': {1: (700, 40), 2: (500, 20), '*': 5}, 'experimental': {1: (700, 40), 2: (500, 20), '*': 2}, }, 'armhf': { 'stretch': {1: (575, 20), 2: (450, 10), '*': 5}, + 'buster': {1: (575, 20), 2: (450, 10), '*': 5}, 'unstable': {1: (575, 20), 2: (450, 10), '*': 5}, 'experimental': {1: (575, 20), 2: (450, 10), '*': 2}, } @@ -138,21 +150,25 @@ LIMITS = { 'depwait': { 'amd64': { 'stretch': {1: (700, 400), 2: (500, 200), '*': 50}, + 'buster': {1: (700, 400), 2: (500, 200), '*': 50}, 'unstable': {1: (700, 400), 2: (500, 200), '*': 50}, 'experimental': {1: (700, 400), 2: (500, 200), '*': 20}, }, 'i386': { 'stretch': {1: (700, 400), 2: (500, 200), '*': 50}, + 'buster': {1: (700, 400), 2: (500, 200), '*': 50}, 'unstable': {1: (700, 400), 2: (500, 200), '*': 50}, 'experimental': {1: (700, 400), 2: (500, 200), '*': 20}, }, 'arm64': { 'stretch': {1: (700, 400), 2: (500, 200), '*': 50}, + 'buster': {1: (700, 400), 2: (500, 200), '*': 50}, 'unstable': {1: (700, 400), 2: (500, 200), '*': 50}, 'experimental': {1: (700, 400), 2: (500, 200), '*': 20}, }, 'armhf': { 'stretch': {1: (575, 200), 2: (450, 100), '*': 50}, + 'buster': {1: (575, 200), 2: (450, 100), '*': 50}, 'unstable': {1: (575, 200), 2: (450, 100), '*': 50}, 'experimental': {1: (575, 200), 2: (450, 100), '*': 20}, } @@ -160,21 +176,25 @@ LIMITS = { 'old': { 'amd64': { 'stretch': {1: (700, 1000), 2: (1100, 850), '*': 0}, + 'buster': {1: (700, 1000), 2: (1100, 850), '*': 0}, 'unstable': {1: (700, 1200), 2: (1100, 1050), '*': 0}, 'experimental': {1: (700, 70), 2: (1100, 50), '*': 0}, }, 'i386': { 'stretch': {1: (700, 1000), 2: (1100, 850), '*': 0}, + 'buster': {1: (700, 1000), 2: (1100, 850), '*': 0}, 'unstable': {1: (700, 1200), 2: (1100, 1050), '*': 0}, 'experimental': {1: (700, 70), 2: (1100, 50), '*': 0}, }, 'arm64': { 'stretch': {1: (700, 1000), 2: (1100, 850), '*': 0}, + 'buster': {1: (700, 1000), 2: (1100, 850), '*': 0}, 'unstable': {1: (700, 1200), 2: (1100, 1050), '*': 0}, 'experimental': {1: (700, 70), 2: (1100, 50), '*': 0}, }, 'armhf': { 'stretch': {1: (600, 800), 2: (850, 500), '*': 0}, + 'buster': {1: (600, 800), 2: (850, 500), '*': 0}, 'unstable': {1: (600, 1000), 2: (850, 700), '*': 0}, 'experimental': {1: (600, 70), 2: (850, 50), '*': 0}, } @@ -449,7 +469,7 @@ def query_new_versions(suite, arch, limit): LIMIT {limit}""".format(suite=suite, arch=arch, limit=limit) pkgs = query_db(query) # the next line avoids constant rescheduling of packages: - # packages in our repository != sid or stretch, + # packages in our repository != official repo, # so they will always be selected by the query above # so we only accept them if there version is greater than the already tested one packages = [(x[0], x[1]) for x in pkgs if version_compare(x[2], x[3]) > 0] diff --git a/bin/reproducible_setup_pbuilder.sh b/bin/reproducible_setup_pbuilder.sh index c64e2fa9..43943d5c 100755 --- a/bin/reproducible_setup_pbuilder.sh +++ b/bin/reproducible_setup_pbuilder.sh @@ -127,8 +127,8 @@ setup_pbuilder() { sudo pbuilder --execute $pbuilder_http_proxy --save-after-exec --basetgz /var/cache/pbuilder/${NAME}-new.tgz -- ${TMPFILE} | tee ${LOG} rm ${TMPFILE} - # add repo only for experimental and sid - keep stretch "real" (and sid progressive!) - if [ "$SUITE" != "stretch" ] ; then + # add repo only for experimental and unstable - keep stretch/buster "real" (and sid progressive!) + if [ "$SUITE" = "unstable" ] || [ "$SUITE" = "experimental" ]; then # apply further customisations, eg. install $PACKAGES from our repo create_setup_our_repo_tmpfile ${TMPFILE} "${PACKAGES}" if [ "$DEBUG" = "true" ] ; then diff --git a/bin/schroot-create.sh b/bin/schroot-create.sh index 94a2fbb6..e956adce 100755 --- a/bin/schroot-create.sh +++ b/bin/schroot-create.sh @@ -182,14 +182,14 @@ bootstrap() { fi robust_chroot_apt install -y --no-install-recommends sudo robust_chroot_apt install -y --no-install-recommends $@ - # try to use diffoscope from experimental - if ([ "$SUITE" = "unstable" ] || [ "$SUITE" = "stretch" ] ) && [ "$1" = "diffoscope" ] ; then + # try to use diffoscope from experimental if available + if [ "$SUITE" != "experimental" ] && [ "$1" = "diffoscope" ] ; then echo "deb $MIRROR experimental main" | sudo tee -a $SCHROOT_TARGET/etc/apt/sources.list > /dev/null robust_chroot_apt update # install diffoscope from experimental without re-adding all recommends... sudo chroot $SCHROOT_TARGET apt-get install -y -t experimental --no-install-recommends diffoscope || echo "Warning: diffoscope from experimental is uninstallable at the moment." - elif [ "$SUITE" = "stretch" ] && [ "$1" = "diffoscope" ] ; then - # always try to use diffoscope from unstable on stretch + elif ([ "$SUITE" != "unstable" ] && [ "$SUITE" != "experimental" ]) && [ "$1" = "diffoscope" ] ; then + # always try to use diffoscope from unstable on stretch/buster echo "deb $MIRROR unstable main" | sudo tee -a $SCHROOT_TARGET/etc/apt/sources.list > /dev/null robust_chroot_apt update # install diffoscope from unstable without re-adding all recommends... diff --git a/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf b/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf index eb2585a5..5f6c4f83 100644 --- a/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf +++ b/hosts/jenkins/etc/apache2/sites-available/jenkins.debian.net.conf @@ -268,11 +268,11 @@ Use common-debian-service-https-redirect www.reproducible-builds.org RewriteCond %{REQUEST_URI} ^/(debian/|)index_all_abc.html$ RewriteRule ^/(debian/|)?(.+) /debian/unstable/amd64/$2 [R=301,L] - # redirect /testing to /stretch + # redirect /testing to /buster # note: no flags on the rule, will cause apache to continue after rewriting # the url and apply the next rule Rewritecond %{REQUEST_URI} ^/(debian/|)testing(/|)$ - RewriteRule ^/(debian/|)(.*) /debian/stretch/ + RewriteRule ^/(debian/|)(.*) /debian/buster/ # redirect /$suite to /$suite/index_suite_amd64_stats.html # note: the missing slash in the RewriteRule is wanted to avoid a double slash |