diff options
| -rw-r--r-- | README | 2 | ||||
| -rwxr-xr-x | bin/reproducible_blacklist.sh | 4 | ||||
| -rwxr-xr-x | bin/reproducible_common.py | 2 | ||||
| -rwxr-xr-x | bin/reproducible_common.sh | 10 | ||||
| -rwxr-xr-x | bin/reproducible_html_dashboard.sh | 16 | ||||
| -rwxr-xr-x | bin/reproducible_html_indexes.py | 4 | ||||
| -rwxr-xr-x | bin/reproducible_html_notes.py | 2 | ||||
| -rwxr-xr-x | bin/reproducible_html_packages.py | 6 | ||||
| -rwxr-xr-x | bin/reproducible_maintenance.sh | 6 | ||||
| -rwxr-xr-x | bin/reproducible_scheduler.py | 26 | ||||
| -rw-r--r-- | job-cfg/reproducible.yaml | 15 |
11 files changed, 16 insertions, 77 deletions
@@ -137,7 +137,7 @@ Installation tests inside chroot environments. * The (current) purpose of https://reproducible.debian.net is to show the prospects of reproducible builds for Debian. IOW: this is research, showing what could (and should) be done... check https://wiki.debian.org/ReproducibleBuilds for the real status of the project! -* Currently, three suites are tested on amd64: 'testing', 'unstable' and 'experimental'. The tests are done using 'pbuilder' using link:https://wiki.debian.org/ReproducibleBuilds/ExperimentalToolchain[our toolchain] through concurrent builder jobs, 32 for 'amd64' and 21 for 'armhf', which are each constantly testing packages and saving the results of these tests. +* Currently, three suites are tested on 'amd64' and 'armhf' architectures: 'testing', 'unstable' and 'experimental'. The tests are done using 'pbuilder' using link:https://wiki.debian.org/ReproducibleBuilds/ExperimentalToolchain[our toolchain] through concurrent builder jobs, 32 for 'amd64' and 21 for 'armhf', which are each constantly testing packages and saving the results of these tests. ** These builds on remote nodes run on very different hardware: for 'amd64' we are now using four virtual machines, profitbricks-build(1+2+5+6)-amd64, which have 16 or 17 cores and 48gb ram each and are sponsored by link:https://jenkins.debian.net/userContent/thanks.html[Profitbricks]. ** To test 'armhf' we are using ten small boards donated by vagrant@d.o: six quad cores (wbq0, cbxi4pro0, ff2a, odxu4, odxu4b and odxu4c) with 2gb ram, another quad-core (rpi2b) with 1gb ram and three dual cores (bpi0, hb0 and wbd0) with 1gb ram, each. We would love to have more or more powerful ARM hardware in the future, if you can help, please talk to us! diff --git a/bin/reproducible_blacklist.sh b/bin/reproducible_blacklist.sh index a22b48f7..0efa56dd 100755 --- a/bin/reproducible_blacklist.sh +++ b/bin/reproducible_blacklist.sh @@ -69,9 +69,7 @@ case $SUITE in sid) echo "WARNING: sid has been renamed to unstable." SUITE=unstable ;; - unstable|experimental) ;; - testing) if [ "$ARCH" = "armhf" ] ; then echo "Testing is not yet tested on $ARCH, exiting." ; exit 0 ; fi - ;; + testing|unstable|experimental) ;; *) echo "$SUITE is not a valid suite". explain_syntax exit 1 diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py index 9c60f581..2676e81e 100755 --- a/bin/reproducible_common.py +++ b/bin/reproducible_common.py @@ -295,8 +295,6 @@ def _gen_links(suite, arch): continue html += link[1].format(suite=suite, arch=arch) + '\n' for i in SUITES: # suite links - if arch == 'armhf' and i == 'testing': - continue html += '<li><a href="/' + i + '/index_suite_' + arch + '_stats.html">suite: ' + i + '</a></li>' if arch == 'amd64': html += '<li><a href="/unstable/index_suite_armhf_stats.html\">arch: armhf</a></li>' diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 5fdac339..4cdc28f5 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -235,7 +235,7 @@ write_page_header() { continue fi if [ "$TARGET" = "pkg_sets" ] && [ "$ARCH" = "armhf" ] && [ "$SUITE" = "testing" ] ; then - # testing/armhf is not being tested yet _yet_ (so I think this is never met…) + # no pkg_sets for testing/amd64 yet… continue fi SPOKEN_TARGET=${SPOKENTARGET[$TARGET]} @@ -253,10 +253,6 @@ write_page_header() { done if [ "$TARGET" = "suite_stats" ] ; then for i in $SUITES ; do - if [ "$i" = "testing" ] && [ "$ARCH" = "armhf" ] ; then - # only unstable and experimental are tested on armhf atm - continue - fi write_page "<li><a href=\"/$i/index_suite_${ARCH}_stats.html\">suite: $i</a></li>" done elif [ "$TARGET" = "scheduled" ] ; then @@ -618,10 +614,10 @@ create_png_from_table() { WHERE_EXTRA="$WHERE_EXTRA AND architecture = \"$ARCH\"" if [ "$ARCH" = "armhf" ] ; then if [ $1 -eq 2 ] ; then - # unstable/armhf was only build since 2015-08-30 (and experimental/armhf since 2015-12-19) + # unstable/armhf was only build since 2015-08-30 (and experimental/armhf since 2015-12-19 and testing/armhf since 2016-01-01) WHERE_EXTRA="$WHERE_EXTRA AND datum >= '2015-08-30'" elif [ $1 -eq 6 ] ; then - # armhf only has pkg sets since its complete, aka 2015-12-22 + # armhf only has pkg sets since its complete (and only for unstable), aka 2015-12-22 WHERE_EXTRA="$WHERE_EXTRA AND datum >= '2015-12-22'" fi fi diff --git a/bin/reproducible_html_dashboard.sh b/bin/reproducible_html_dashboard.sh index 1140af75..8c2d8957 100755 --- a/bin/reproducible_html_dashboard.sh +++ b/bin/reproducible_html_dashboard.sh @@ -295,11 +295,7 @@ write_build_performance_stats() { PERF_STATS[$ARCH]=$(mktemp -t reproducible-dashboard-perf-XXXXXXXX) AGE_UNSTABLE=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='unstable' AND architecture='$ARCH' AND datum='$DATE'") AGE_EXPERIMENTAL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='experimental' AND architecture='$ARCH' AND datum='$DATE'") - if [ "$ARCH" != "armhf" ] ; then - AGE_TESTING=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='testing' AND architecture='$ARCH' AND datum='$DATE'") - else - AGE_TESTING="-" - fi + AGE_TESTING=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='testing' AND architecture='$ARCH' AND datum='$DATE'") write_page "<td>$AGE_TESTING / $AGE_UNSTABLE / $AGE_EXPERIMENTAL days</td>" done write_page "</tr><tr><td>average test duration (on $DATE)</td>" @@ -339,9 +335,6 @@ write_suite_table() { write_page "<p>" write_page "<table class=\"main\"><tr><th>suite</th><th>all sources packages</th><th>reproducible packages</th><th>unreproducible packages</th><th>packages failing to build</th><th>other packages</th></tr>" for SUITE in $SUITES ; do - if [ "$ARCH" = "armhf" ] && [ "$SUITE" = "testing" ] ; then - continue - fi gather_suite_arch_stats write_page "<tr><td>$SUITE/$ARCH</td><td>$AMOUNT" if [ $(echo $PERCENT_TOTAL/1|bc) -lt 99 ] ; then @@ -510,9 +503,6 @@ create_dashboard_page() { write_page "</p><p style=\"clear:both;\">" for ARCH in ${ARCHS} ; do for SUITE in $SUITES ; do - if [ "$ARCH" = "armhf" ] && [ "$SUITE" = testing ] ; then - continue - fi write_page " <a href=\"/$SUITE/$ARCH/${TABLE[2]}.png\"><img src=\"/$SUITE/$ARCH/${TABLE[2]}.png\" class=\"overview\" alt=\"age of oldest reproducible build result in $SUITE/$ARCH\"></a>" done write_page "</p><p style=\"clear:both;\">" @@ -533,10 +523,6 @@ update_bug_stats update_notes_stats for ARCH in ${ARCHS} ; do for SUITE in $SUITES ; do - if [ "$SUITE" = "testing" ] && [ "$ARCH" = "armhf" ] ; then - # we only test unstable and experimental on armhf atm - continue - fi update_suite_arch_stats gather_suite_arch_stats create_suite_arch_stats_page diff --git a/bin/reproducible_html_indexes.py b/bin/reproducible_html_indexes.py index 05786765..a6bd4424 100755 --- a/bin/reproducible_html_indexes.py +++ b/bin/reproducible_html_indexes.py @@ -512,8 +512,6 @@ def build_page(page, suite=None, arch=None): else: for suite in SUITES: for arch in ARCHS: - if arch == 'armhf' and suite == 'testing': - continue log.debug('global page §' + section['db_status'] + ' in ' + page + ' for ' + suite + '/' + arch) html += build_page_section(page, section, suite, arch)[0] @@ -539,8 +537,6 @@ bugs = get_bugs() # this variable should not be global, else merely importing _h if __name__ == '__main__': for arch in ARCHS: for suite in SUITES: - if arch == 'armhf' and suite == 'testing': - continue for page in pages.keys(): if 'global' not in pages[page] or not pages[page]['global']: build_page(page, suite, arch) diff --git a/bin/reproducible_html_notes.py b/bin/reproducible_html_notes.py index 320f75a1..f7d045f0 100755 --- a/bin/reproducible_html_notes.py +++ b/bin/reproducible_html_notes.py @@ -439,8 +439,6 @@ if __name__ == '__main__': gen_packages_html([Package(x) for x in notes]) for suite in SUITES: for arch in ARCHS: - if arch == 'armhf' and suite == 'testing': - continue build_page('notes', suite, arch) build_page('no_notes', suite, arch) build_page('FTBFS', suite, arch) diff --git a/bin/reproducible_html_packages.py b/bin/reproducible_html_packages.py index 7d6db715..95d7e499 100755 --- a/bin/reproducible_html_packages.py +++ b/bin/reproducible_html_packages.py @@ -172,8 +172,6 @@ def gen_suites_links(package, current_suite, current_arch): html += tab + '<li>{}\n'.format(a) html += tab + '<ul class="children">\n' for s in SUITES: - if a == 'armhf' and s == 'testing': - continue status = package.get_status(s, a) if not status: # The package is not available in that suite/arch continue @@ -249,8 +247,6 @@ def gen_packages_html(packages, no_clean=False): pkg = package.name for suite in SUITES: for arch in ARCHS: - if arch == 'armhf' and suite == 'testing': - continue status = package.get_status(suite, arch) version = package.get_tested_version(suite, arch) build_date = package.get_build_date(suite, arch) @@ -306,8 +302,6 @@ def gen_all_rb_pkg_pages(no_clean=False): def purge_old_pages(): for suite in SUITES: for arch in ARCHS: - if arch == 'armhf' and suite == 'testing': - continue log.info('Removing old pages from ' + suite + '/' + arch + '.') try: presents = sorted(os.listdir(RB_PKG_PATH + '/' + suite + '/' + diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh index ba777dd8..2a60a70c 100755 --- a/bin/reproducible_maintenance.sh +++ b/bin/reproducible_maintenance.sh @@ -51,9 +51,6 @@ if [ ! -z "$http_proxy" ] ; then pbuilder_http_proxy="--http-proxy $http_proxy" fi for s in $SUITES ; do - if [ "$ARCH" = "armhf" ] && [ "$s" = "testing" ] ; then - continue - fi # # schroot update # @@ -177,9 +174,6 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then REASON="maintenance reschedule: reschedule builds which failed due to network errors" for SUITE in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f8 | sort -u) ; do for ARCH in $(echo $FAILED_BUILDS | sed "s# #\n#g" | cut -d "/" -f9 | sort -u) ; do - if [ "$ARCH" = "armhf" ] && [ "$SUITE" = "testing" ] ; then - continue - fi CANDIDATES=$(for PKG in $(echo $FAILED_BUILDS | sed "s# #\n#g" | grep "/$SUITE/$ARCH/" | cut -d "/" -f10 | cut -d "_" -f1) ; do echo "$PKG" ; done) # double check those builds actually failed TO_SCHEDULE="" diff --git a/bin/reproducible_scheduler.py b/bin/reproducible_scheduler.py index 6e18f5cf..f6c447e9 100755 --- a/bin/reproducible_scheduler.py +++ b/bin/reproducible_scheduler.py @@ -71,7 +71,7 @@ LIMITS = { 'experimental': {'*': 440}, }, 'armhf': { - 'testing': {'*': 0}, + 'testing': {'*': 42}, 'unstable': {'*': 150}, 'experimental': {'*': 150}, }, @@ -83,7 +83,7 @@ LIMITS = { 'experimental': {1: (100, 250), 2: (200, 200), '*': 150}, }, 'armhf': { - 'testing': {1: (100, 0), 2: (200, 0), '*': 0}, + 'testing': {1: (100, 42), 2: (200, 42), '*': 0}, 'unstable': {1: (100, 75), 2: (200, 60), '*': 45}, 'experimental': {1: (100, 75), 2: (200, 60), '*': 45}, }, @@ -95,7 +95,7 @@ LIMITS = { 'experimental': {1: (250, 40), 2: (350, 20), '*': 0}, }, 'armhf': { - 'testing': {1: (250, 0), 2: (350, 0), '*': 0}, + 'testing': {1: (250, 12), 2: (350, 6), '*': 0}, 'unstable': {1: (250, 12), 2: (350, 6), '*': 0}, 'experimental': {1: (250, 12), 2: (350, 6), '*': 0}, } @@ -172,11 +172,8 @@ def update_sources(suite): sources = lzma.decompress(urlopen(remotefile).read()).decode('utf8') log.debug('\tdownloaded') for arch in ARCHS: - if arch == 'armhf' and suite == 'testing': - continue - else: - log.info('Updating sources db for %s/%s...', suite, arch) - update_sources_db(suite, arch, sources) + log.info('Updating sources db for %s/%s...', suite, arch) + update_sources_db(suite, arch, sources) def update_sources_db(suite, arch, sources): @@ -312,8 +309,6 @@ def add_up_numbers(packages, arch): packages_sum = '+'.join([str(len(packages[x])) for x in SUITES]) if packages_sum == '0+0+0': packages_sum = '0' - elif arch == 'armhf': - packages_sum = str(len(packages['unstable']))+'+'+str(len(packages['experimental'])) return packages_sum @@ -497,9 +492,6 @@ def scheduler(arch): if suite not in priotized_suite_order: priotized_suite_order.append(suite) for suite in priotized_suite_order: - if arch == 'armhf' and suite == 'testing': - now_queued_here[suite] = 0 - continue query = 'SELECT count(*) ' \ 'FROM schedule AS p JOIN sources AS s ON p.package_id=s.id ' \ 'WHERE s.suite="{suite}" AND s.architecture="{arch}"' @@ -516,10 +508,7 @@ def scheduler(arch): # update the scheduled page generate_schedule(arch) # from reproducible_html_indexes # build the kgb message text - if arch != 'armhf': - message = 'Scheduled in ' + '+'.join(SUITES) + ' (' + arch + '): ' - else: - message = 'Scheduled in unstable+experimental (' + arch + '): ' + message = 'Scheduled in ' + '+'.join(SUITES) + ' (' + arch + '): ' if msg_untested: message += msg_untested message += ' and ' if msg_new and not msg_old_ftbfs_and_depwait and not msg_old else '' @@ -535,8 +524,7 @@ def scheduler(arch): message += msg_old total = [now_queued_here[x] for x in SUITES] message += ', for ' + str(sum(total)) - if arch != 'armhf': - message += ' or ' + '+'.join([str(now_queued_here[x]) for x in SUITES]) + message += ' or ' + '+'.join([str(now_queued_here[x]) for x in SUITES]) message += ' packages in total.' # only notifiy irc if there were packages scheduled in any suite for x in SUITES: diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index e4332283..396b0444 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -280,18 +280,6 @@ - 'setup_pbuilder_testing': my_description: 'Setup pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' my_timed: '23 7 * * *' - - 'setup_schroot_testing': - my_description: 'Setup testing schroot for fetching source packages for the builder jobs.' - my_timed: '0 1 * * *' - my_hname: - - 'profitbricks1': { my_arch: 'amd64' } - - 'profitbricks2': { my_arch: 'amd64' } - - 'profitbricks5': { my_arch: 'amd64' } - - 'profitbricks6': { my_arch: 'amd64' } - my_shell: '/srv/jenkins/bin/jenkins_master_wrapper.sh' - - - '{name}_{my_task}_{my_arch}_{my_hname}': - my_task: - 'setup_pbuilder_unstable': my_description: 'Setup pbuilder for reproducible builds of packages from testing as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' my_timed: '42 7 * * *' @@ -301,6 +289,9 @@ - 'setup_schroot_unstable': my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.' my_timed: '20 1 * * *' + - 'setup_schroot_testing': + my_description: 'Setup testing schroot for fetching source packages for the builder jobs.' + my_timed: '0 1 * * *' - 'setup_schroot_experimental': my_description: 'Setup experimental schroot for fetching source packages for the builder jobs.' my_timed: '40 1 * * *' |