From 1aa7e0422d9f0a4561aa952ce6a89a1ba9bedd31 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Fri, 17 Oct 2014 16:02:50 +0200 Subject: reproducible: refactor, move common functions out, new script+job to just create the dd-list page daily --- bin/reproducible_stats.sh | 132 ++-------------------------------------------- 1 file changed, 4 insertions(+), 128 deletions(-) (limited to 'bin/reproducible_stats.sh') diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh index 560b678a..08e1d730 100755 --- a/bin/reproducible_stats.sh +++ b/bin/reproducible_stats.sh @@ -14,6 +14,8 @@ common_init "$@" cp $PACKAGES_DB /var/lib/jenkins/userContent/reproducible.db set +x +init_html + declare -A GOOD declare -A BAD declare -A UGLY @@ -21,19 +23,12 @@ declare -A SOURCELESS declare -A NOTFORUS declare -A STAR declare -A LINKTARGET -declare -A SPOKENTARGET LAST24="AND build_date > datetime('now', '-24 hours') " LAST48="AND build_date > datetime('now', '-48 hours') " -SUITE=sid -AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(name) FROM sources") -ALLSTATES="reproducible FTBR_with_buildinfo FTBR FTBFS 404 not_for_us blacklisted" -MAINVIEW="stats" -ALLVIEWS="last_24h last_48h all_abc" GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY build_date DESC" | xargs echo) GOOD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) GOOD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) GOOD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY name" | xargs echo) -COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"reproducible\"") BAD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo) BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) BAD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) @@ -52,28 +47,10 @@ NOTFORUS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_ COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"not for us\"" | xargs echo) BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"blacklisted\" ORDER BY name" | xargs echo) COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"blacklisted\"" | xargs echo) -COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages") -PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc) -PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc) PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) -GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc) -SPOKENTARGET["last_24h"]="packages tested in the last 24h" -SPOKENTARGET["last_48h"]="packages tested in the last 48h" -SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" -SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" -SPOKENTARGET["stats"]="various statistics about reproducible builds" -SPOKENTARGET["notes"]="packages with notes" -SPOKENTARGET["issues"]="known issues related to reproducible builds" -SPOKENTARGET["reproducible"]="packages which built reproducibly" -SPOKENTARGET["FTBR"]="packages which failed to build reproducibly and don't create a .buildinfo file" -SPOKENTARGET["FTBR_with_buildinfo"]="packages which failed to build reproducibly and create a .buildinfo file" -SPOKENTARGET["FTBFS"]="packages which failed to build from source" -SPOKENTARGET["404"]="packages where the sources failed to downloaded" -SPOKENTARGET["not_for_us"]="packages which should not be build on 'amd64'" -SPOKENTARGET["blacklisted"]="packages which have been blacklisted" # # gather notes @@ -87,11 +64,9 @@ else git clone git://git.debian.org/git/reproducible/notes.git notes.git fi cd $WORKSPACE + PACKAGES_YML=/var/lib/jenkins/notes.git/packages.yml ISSUES_YML=/var/lib/jenkins/notes.git/issues.yml -NOTES_PATH=/var/lib/jenkins/userContent/notes -ISSUES_PATH=/var/lib/jenkins/userContent/issues -mkdir -p $NOTES_PATH $ISSUES_PATH declare -A NOTES_VERSION declare -A NOTES_ISSUES @@ -173,7 +148,7 @@ create_pkg_note() { write_page "" write_page "" write_page "Notes for $1" - write_page "
" + write_page "
" write_page "" write_page "" @@ -353,45 +328,6 @@ validate_yaml() { # end note parsing functions... # -mkdir -p /var/lib/jenkins/userContent/rb-pkg/ - -write_page() { - echo "$1" >> $PAGE -} - -set_icon() { - # icons taken from tango-icon-theme (0.8.90-5) - # licenced under http://creativecommons.org/licenses/publicdomain/ - STATE_TARGET_NAME="$1" - case "$1" in - reproducible) ICON=weather-clear.png - ;; - unreproducible|FTBR*) if [ "$2" != "" ] ; then - ICON=weather-showers-scattered.png - STATE_TARGET_NAME=FTBR_with_buildinfo - else - ICON=weather-showers.png - STATE_TARGET_NAME=FTBR - fi - ;; - FTBFS) ICON=weather-storm.png - ;; - 404) ICON=weather-severe-alert.png - ;; - not_for_us|"not for us") ICON=weather-few-clouds-night.png - STATE_TARGET_NAME="not_for_us" - ;; - blacklisted) ICON=error.png - ;; - *) ICON="" - esac -} - -write_icon() { - # ICON and STATE_TARGET_NAME are set by set_icon() - write_page "\"${STATE_TARGET_NAME}" -} - init_pkg_page() { echo "" > ${PKG_FILE} echo "" >> ${PKG_FILE} @@ -496,66 +432,6 @@ link_packages() { done } -write_page_header() { - rm -f $PAGE - write_page "" - write_page "" - write_page "" - write_page "$2" - write_page "

$2

" - if [ "$1" = "$MAINVIEW" ] || ; then - write_page "

These pages are updated every six hours. Results are obtained from several jobs running on jenkins.debian.net. Thanks to Profitbricks for donating the virtual machine it's running on!

" - fi - write_page "

$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!" - if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] || [ "$1" = "stats" ] ; then - write_page " Join #debian-reproducible on OFTC to get support for making sure your packages build reproducibly too!" - fi - write_page "

" - write_page "" - write_page "
" -} - -write_page_footer() { - write_page "

There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014 Holger Levsen, GPL-2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library.

" - write_page "" -} - -write_page_meta_sign() { - write_page "

An underlined package is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue." - if $BETA_SIGN ; then - write_page "A β sign after a package which is unreproducible indicates that a .buildinfo file was generated." - write_page "And that means the basics for building packages reproducibly are covered." - fi - write_page "

" -} - -publish_page() { - cp $PAGE /var/lib/jenkins/userContent/ - if [ "$VIEW" = "$MAINVIEW" ] ; then - cp $PAGE /var/lib/jenkins/userContent/reproducible.html - fi - rm $PAGE -} - # # actually parse the notes # -- cgit v1.2.3-70-g09d2
Version annotated:${NOTES_VERSION[$1]}