From 16b8750afdb9275f645e361fea85a5405568fdb2 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Sun, 20 Dec 2015 18:34:03 +0100 Subject: reproducible armhf: enable package sets --- TODO | 1 - bin/reproducible_common.sh | 4 ++-- bin/reproducible_create_meta_pkg_sets.sh | 3 +++ bin/reproducible_db_maintenance.py | 21 ++++++++++++++++++++ bin/reproducible_html_dashboard.sh | 33 ++++++++++++++++---------------- bin/reproducible_html_pkg_sets.sh | 17 ++++++++-------- 6 files changed, 52 insertions(+), 27 deletions(-) diff --git a/TODO b/TODO index 8cc6364e..1218c733 100644 --- a/TODO +++ b/TODO @@ -160,7 +160,6 @@ The plan is to run a jenkins.d.o host, which is maintained by DSA, but we are ma ** improve ftbfs page: list packages without bugs and notes first ** mattia: .py scripts: UDD or any db connection errors should either be retried or cause an abort (not failure!) of the job ** repo-comparison: check for binaries without source -** pkg sets are still amd64 only atm… (and there is 404 link to the armhf page) ** bin/_html_indexes.py: bugs = get_bugs() # this variable should not be global, else merely importing _html_indexes always queries UDD ** re-enable linux-2.6 usage once 806911 has been fixed diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 0872dd73..23a1eb9d 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -234,8 +234,8 @@ write_page_header() { # no pkg_sets are tested in experimental continue fi - if [ "$TARGET" = "pkg_sets" ] && [ "$ARCH" = "armhf" ] ; then - # no pkg_sets for armhf _yet_ + if [ "$TARGET" = "pkg_sets" ] && [ "$ARCH" = "armhf" ] && [ "$SUITE" = "testing" ] ; then + # testing/armhf is not being tested yet _yet_ (so I think this is never met…) continue fi SPOKEN_TARGET=${SPOKENTARGET[$TARGET]} diff --git a/bin/reproducible_create_meta_pkg_sets.sh b/bin/reproducible_create_meta_pkg_sets.sh index 58e42019..6aba06d7 100755 --- a/bin/reproducible_create_meta_pkg_sets.sh +++ b/bin/reproducible_create_meta_pkg_sets.sh @@ -10,6 +10,9 @@ common_init "$@" # common code defining db access . /srv/jenkins/bin/reproducible_common.sh +# we only generate the meta pkg sets on amd64 +# (else this script would need a lot of changes for little gain) +# but these are source package sets so there is a difference only very rarely anyway ARCH=amd64 # everything should be ok… diff --git a/bin/reproducible_db_maintenance.py b/bin/reproducible_db_maintenance.py index fd9a49f8..61fb92f7 100755 --- a/bin/reproducible_db_maintenance.py +++ b/bin/reproducible_db_maintenance.py @@ -550,6 +550,27 @@ schema_updates = { 'ALTER TABLE schedule ADD COLUMN message TEXT', 'ALTER TABLE stats_build ADD COLUMN schedule_message TEXT NOT NULL DEFAULT ""', 'INSERT INTO rb_schema VALUES ("26", "' + now + '")'], + 27: [ # add column architecture to stats_meta_pkg_state and set previous values to amd64 + 'ALTER TABLE stats_meta_pkg_state ADD COLUMN architecture TEXT NOT NULL DEFAULT "amd64"', + 'INSERT INTO rb_schema VALUES ("27", "' + now + '")'], + 28: [ # use (datum, suite, architecture, meta_pkg) as primary key for stats_meta_pkg_state + '''CREATE TABLE stats_meta_pkg_state_tmp + (datum TEXT NOT NULL, + suite TEXT NOT NULL, + architecture TEXT NOT NULL, + meta_pkg TEXT NOT NULL, + reproducible INTEGER, + unreproducible INTEGER, + FTBFS INTEGER, + other INTEGER, + PRIMARY KEY (datum, suite, architecture, meta_pkg))''', + '''INSERT INTO stats_meta_pkg_state_tmp (datum, suite, architecture, meta_pkg, + reproducible, unreproducible, FTBFS, other) + SELECT datum, suite, architecture, meta_pkg, reproducible, unreproducible, + FTBFS, other FROM stats_meta_pkg_state;''', + '''DROP TABLE stats_meta_pkg_state;''', + '''ALTER TABLE stats_meta_pkg_state_tmp RENAME TO stats_meta_pkg_state;''', + 'INSERT INTO rb_schema VALUES ("28", "' + now + '")'], } diff --git a/bin/reproducible_html_dashboard.sh b/bin/reproducible_html_dashboard.sh index d2ae676a..39d3968c 100755 --- a/bin/reproducible_html_dashboard.sh +++ b/bin/reproducible_html_dashboard.sh @@ -412,6 +412,17 @@ create_suite_arch_stats_page() { publish_page $SUITE } +write_meta_pkg_graphs_links () { + local SUITE=unstable # ARCH is taken from global namespace + write_page "

" + for i in $(seq 1 ${#META_PKGSET[@]}) ; do + THUMB=${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png + LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" + write_page "\"$LABEL\"" + done + write_page "

" +} + # # create dashboard page # @@ -425,23 +436,10 @@ create_dashboard_page() { for SUITE in $SUITES ; do write_page " \"$SUITE/$ARCH" done - write_page "

" - write_page "

" - # write meta pkg graphs per suite - for SUITE in $SUITES ; do - if [ "$SUITE" != "unstable" ] ; then - # only show pkg sets from unstable - continue - fi - for i in $(seq 1 ${#META_PKGSET[@]}) ; do - THUMB=${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png - LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" - write_page "\"$LABEL\"" - done - done - write_page "

" + write_page "

" + write_meta_pkg_graphs_links # write inventory table - write_page "" + write_page "

Various reproducibility statistics
" write_page "" write_page "" write_page "" @@ -495,9 +493,12 @@ create_dashboard_page() { ARCH="armhf" write_page "

" write_suite_table + # write suite graphs for SUITE in unstable experimental ; do write_page " \"$SUITE/$ARCH" done + write_page "

" + write_meta_pkg_graphs_links # write performance stats and build per day graphs write_page "

" write_build_performance_stats diff --git a/bin/reproducible_html_pkg_sets.sh b/bin/reproducible_html_pkg_sets.sh index f861808f..dc7d294c 100755 --- a/bin/reproducible_html_pkg_sets.sh +++ b/bin/reproducible_html_pkg_sets.sh @@ -14,7 +14,6 @@ common_init "$@" # # init some variables # -ARCH="amd64" # we only care about amd64 status here (for now) # we only do stats up until yesterday... we also could do today too but not update the db yet... DATE=$(date -d "1 day ago" '+%Y-%m-%d') FORCE_DATE=$(date -d "3 days ago" '+%Y-%m-%d') @@ -184,12 +183,14 @@ create_pkg_sets_pages() { # # main # -for SUITE in $SUITES ; do - if [ "$SUITE" = "experimental" ] ; then - # no pkg sets in experimental - continue - fi - update_meta_pkg_stats - create_pkg_sets_pages +for ARCH in $ARCHS ; do + for SUITE in $SUITES ; do + if [ "$SUITE" = "experimental" ] || ( [ "$SUITE" = "testing" ] && [ "$ARCH" = "armhf" ] ) ; then + # no pkg sets in experimental and not yet for testing on armhf + continue + fi + update_meta_pkg_stats + create_pkg_sets_pages + done done -- cgit v1.2.3-70-g09d2

Various reproducibility statistics
identified distinct and categorized issues$ISSUES
total number of identified issues in packages$COUNT_ISSUES
packages with notes about these issues$NOTES