diff options
| -rwxr-xr-x | bin/reproducible_common.sh | 131 | ||||
| -rwxr-xr-x | bin/reproducible_html_dd_list.sh | 41 | ||||
| -rwxr-xr-x | bin/reproducible_stats.sh | 132 | ||||
| -rw-r--r-- | job-cfg/reproducible.yaml | 10 |
4 files changed, 185 insertions, 129 deletions
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index c828759e..954aa384 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -100,3 +100,134 @@ fi # shop trailing slash JENKINS_URL=${JENKINS_URL:0:-1} +init_html() { + SUITE=sid + ALLSTATES="reproducible FTBR_with_buildinfo FTBR FTBFS 404 not_for_us blacklisted" + MAINVIEW="stats" + ALLVIEWS="last_24h last_48h all_abc" + declare -A SPOKENTARGET + SPOKENTARGET["last_24h"]="packages tested in the last 24h" + SPOKENTARGET["last_48h"]="packages tested in the last 48h" + SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" + SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" + SPOKENTARGET["stats"]="various statistics about reproducible builds" + SPOKENTARGET["notes"]="packages with notes" + SPOKENTARGET["issues"]="known issues related to reproducible builds" + SPOKENTARGET["reproducible"]="packages which built reproducibly" + SPOKENTARGET["FTBR"]="packages which failed to build reproducibly and don't create a .buildinfo file" + SPOKENTARGET["FTBR_with_buildinfo"]="packages which failed to build reproducibly and create a .buildinfo file" + SPOKENTARGET["FTBFS"]="packages which failed to build from source" + SPOKENTARGET["404"]="packages where the sources failed to downloaded" + SPOKENTARGET["not_for_us"]="packages which should not be build on 'amd64'" + SPOKENTARGET["blacklisted"]="packages which have been blacklisted" + NOTES_PATH=/var/lib/jenkins/userContent/notes + ISSUES_PATH=/var/lib/jenkins/userContent/issues + mkdir -p $NOTES_PATH $ISSUES_PATH + # FIXME RB_PATH would also be a good idea + mkdir -p /var/lib/jenkins/userContent/rb-pkg/ + # query some data we need everywhere + AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(name) FROM sources") + COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages") + COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"reproducible\"") + PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc) + PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc) + GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc) +} + +write_page() { + echo "$1" >> $PAGE +} + +set_icon() { + # icons taken from tango-icon-theme (0.8.90-5) + # licenced under http://creativecommons.org/licenses/publicdomain/ + STATE_TARGET_NAME="$1" + case "$1" in + reproducible) ICON=weather-clear.png + ;; + unreproducible|FTBR*) if [ "$2" != "" ] ; then + ICON=weather-showers-scattered.png + STATE_TARGET_NAME=FTBR_with_buildinfo + else + ICON=weather-showers.png + STATE_TARGET_NAME=FTBR + fi + ;; + FTBFS) ICON=weather-storm.png + ;; + 404) ICON=weather-severe-alert.png + ;; + not_for_us|"not for us") ICON=weather-few-clouds-night.png + STATE_TARGET_NAME="not_for_us" + ;; + blacklisted) ICON=error.png + ;; + *) ICON="" + esac +} + +write_icon() { + # ICON and STATE_TARGET_NAME are set by set_icon() + write_page "<a href=\"$JENKINS_URL/userContent/index_${STATE_TARGET_NAME}.html\" target=\"_parent\"><img src=\"$JENKINS_URL/userContent/static/$ICON\" alt=\"${STATE_TARGET_NAME} icon\" /></a>" +} + +write_page_header() { + rm -f $PAGE + write_page "<!DOCTYPE html><html><head>" + write_page "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />" + write_page "<link href=\"$JENKINS_URL/userContent/static/style.css\" type=\"text/css\" rel=\"stylesheet\" />" + write_page "<title>$2</title></head>" + write_page "<body><header><h2>$2</h2>" + if [ "$1" = "$MAINVIEW" ] ; then + write_page "<p>These pages are updated every six hours. Results are obtained from <a href=\"$JENKINS_URL/view/reproducible\">several jobs running on jenkins.debian.net</a>. Thanks to <a href=\"https://www.profitbricks.com\">Profitbricks</a> for donating the virtual machine it's running on!</p>" + fi + write_page "<p>$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD <a href=\"https://wiki.debian.org/ReproducibleBuilds\">packages should be reproducibly buildable!</a>" + if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] || [ "$1" = "stats" ] ; then + write_page " Join <code>#debian-reproducible</code> on OFTC to get support for making sure your packages build reproducibly too!" + fi + write_page "</p>" + write_page "<ul><li>Have a look at:</li>" + for MY_STATE in $ALLSTATES ; do + WITH="" + if [ "$MY_STATE" = "FTBR_with_buildinfo" ] ; then + WITH="YES" + fi + set_icon $MY_STATE $WITH + write_page "<li>" + write_icon + write_page "</li>" + done + for TARGET in issues notes $ALLVIEWS dd-list stats ; do + if [ "$TARGET" = "issues" ] || [ "$TARGET" = "stats" ]; then + SPOKEN_TARGET=$TARGET + else + SPOKEN_TARGET=${SPOKENTARGET[$TARGET]} + fi + write_page "<li><a href=\"$JENKINS_URL/userContent/index_${TARGET}.html\">${SPOKEN_TARGET}</a></li>" + done + write_page "</ul>" + write_page "</header>" +} + +write_page_footer() { + write_page "<hr/><p style=\"font-size:0.9em;\">There is more information <a href=\"$JENKINS_URL/userContent/about.html\">about jenkins.debian.net</a> and about <a href=\"https://wiki.debian.org/ReproducibleBuilds\"> reproducible builds of Debian</a> available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014 <a href=\"mailto:holger@layer-acht.org\">Holger Levsen</a>, GPL-2 licensed. The weather icons are public domain and have been taken from the <a href="http://tango.freedesktop.org/Tango_Icon_Library" target="_blank">Tango Icon Library</a>.</p>" + write_page "</body></html>" +} + +write_page_meta_sign() { + write_page "<p style=\"font-size:0.9em;\">An underlined package is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue." + if $BETA_SIGN ; then + write_page "A β sign after a package which is unreproducible indicates that a .buildinfo file was generated." + write_page "And that means the <a href=\"https://wiki.debian.org/ReproducibleBuilds#The_basics_for_making_packages_build_reproducible\">basics for building packages reproducibly are covered</a>." + fi + write_page "</p>" +} + +publish_page() { + cp $PAGE /var/lib/jenkins/userContent/ + if [ "$VIEW" = "$MAINVIEW" ] ; then + cp $PAGE /var/lib/jenkins/userContent/reproducible.html + fi + rm $PAGE +} + diff --git a/bin/reproducible_html_dd_list.sh b/bin/reproducible_html_dd_list.sh new file mode 100755 index 00000000..07c6ba5a --- /dev/null +++ b/bin/reproducible_html_dd_list.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Copyright 2014 Holger Levsen <holger@layer-acht.org> +# released under the GPLv=2 + +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code defining db access +. /srv/jenkins/bin/reproducible_common.sh + +set +x +init_html + +VIEW=dd-list +PAGE=index_${VIEW}.html +echo "$(date) - starting to write $PAGE page." +write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}" +TMPFILE=$(mktemp) +BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo) +echo "${BAD}" | dd-list -i > $TMPFILE || true +write_page "<p>The following maintainers and uploaders are listed for packages which have built unreproducibly:</p><p><pre>" +while IFS= read -r LINE ; do + if [ "${LINE:0:3}" = " " ] ; then + PACKAGE=$(echo "${LINE:3}" | cut -d " " -f1) + UPLOADERS=$(echo "${LINE:3}" | cut -d " " -f2-) + if [ "$UPLOADERS" = "$PACKAGE" ] ; then + UPLOADERS="" + fi + write_page " <a href=\"$JENKINS_URL/userContent/rb-pkg/$PACKAGE.html\">$PACKAGE</a> $UPLOADERS" + else + LINE="$(echo $LINE | sed 's#&#\&#g ; s#<#\<#g ; s#>#\>#g')" + write_page "$LINE" + fi +done < $TMPFILE +write_page "</pre></p>" +rm $TMPFILE +write_page_footer +publish_page + +echo "Enjoy $JENKINS_URL/userContent/$PAGE" diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh index 560b678a..08e1d730 100755 --- a/bin/reproducible_stats.sh +++ b/bin/reproducible_stats.sh @@ -14,6 +14,8 @@ common_init "$@" cp $PACKAGES_DB /var/lib/jenkins/userContent/reproducible.db set +x +init_html + declare -A GOOD declare -A BAD declare -A UGLY @@ -21,19 +23,12 @@ declare -A SOURCELESS declare -A NOTFORUS declare -A STAR declare -A LINKTARGET -declare -A SPOKENTARGET LAST24="AND build_date > datetime('now', '-24 hours') " LAST48="AND build_date > datetime('now', '-48 hours') " -SUITE=sid -AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(name) FROM sources") -ALLSTATES="reproducible FTBR_with_buildinfo FTBR FTBFS 404 not_for_us blacklisted" -MAINVIEW="stats" -ALLVIEWS="last_24h last_48h all_abc" GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY build_date DESC" | xargs echo) GOOD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) GOOD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) GOOD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY name" | xargs echo) -COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"reproducible\"") BAD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo) BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) BAD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) @@ -52,28 +47,10 @@ NOTFORUS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_ COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"not for us\"" | xargs echo) BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"blacklisted\" ORDER BY name" | xargs echo) COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"blacklisted\"" | xargs echo) -COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages") -PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc) -PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc) PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) -GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc) -SPOKENTARGET["last_24h"]="packages tested in the last 24h" -SPOKENTARGET["last_48h"]="packages tested in the last 48h" -SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" -SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" -SPOKENTARGET["stats"]="various statistics about reproducible builds" -SPOKENTARGET["notes"]="packages with notes" -SPOKENTARGET["issues"]="known issues related to reproducible builds" -SPOKENTARGET["reproducible"]="packages which built reproducibly" -SPOKENTARGET["FTBR"]="packages which failed to build reproducibly and don't create a .buildinfo file" -SPOKENTARGET["FTBR_with_buildinfo"]="packages which failed to build reproducibly and create a .buildinfo file" -SPOKENTARGET["FTBFS"]="packages which failed to build from source" -SPOKENTARGET["404"]="packages where the sources failed to downloaded" -SPOKENTARGET["not_for_us"]="packages which should not be build on 'amd64'" -SPOKENTARGET["blacklisted"]="packages which have been blacklisted" # # gather notes @@ -87,11 +64,9 @@ else git clone git://git.debian.org/git/reproducible/notes.git notes.git fi cd $WORKSPACE + PACKAGES_YML=/var/lib/jenkins/notes.git/packages.yml ISSUES_YML=/var/lib/jenkins/notes.git/issues.yml -NOTES_PATH=/var/lib/jenkins/userContent/notes -ISSUES_PATH=/var/lib/jenkins/userContent/issues -mkdir -p $NOTES_PATH $ISSUES_PATH declare -A NOTES_VERSION declare -A NOTES_ISSUES @@ -173,7 +148,7 @@ create_pkg_note() { write_page "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />" write_page "<link href=\"$JENKINS_URL/userContent/static/style.css\" type=\"text/css\" rel=\"stylesheet\" />" write_page "<title>Notes for $1</title></head>" - write_page "<body><header>" + write_page "<body></header>" write_page "<table class=\"body\">" write_page "<tr><td>Version annotated:</td><td colspan=\"2\">${NOTES_VERSION[$1]}</td></tr>" @@ -353,45 +328,6 @@ validate_yaml() { # end note parsing functions... # -mkdir -p /var/lib/jenkins/userContent/rb-pkg/ - -write_page() { - echo "$1" >> $PAGE -} - -set_icon() { - # icons taken from tango-icon-theme (0.8.90-5) - # licenced under http://creativecommons.org/licenses/publicdomain/ - STATE_TARGET_NAME="$1" - case "$1" in - reproducible) ICON=weather-clear.png - ;; - unreproducible|FTBR*) if [ "$2" != "" ] ; then - ICON=weather-showers-scattered.png - STATE_TARGET_NAME=FTBR_with_buildinfo - else - ICON=weather-showers.png - STATE_TARGET_NAME=FTBR - fi - ;; - FTBFS) ICON=weather-storm.png - ;; - 404) ICON=weather-severe-alert.png - ;; - not_for_us|"not for us") ICON=weather-few-clouds-night.png - STATE_TARGET_NAME="not_for_us" - ;; - blacklisted) ICON=error.png - ;; - *) ICON="" - esac -} - -write_icon() { - # ICON and STATE_TARGET_NAME are set by set_icon() - write_page "<a href=\"$JENKINS_URL/userContent/index_${STATE_TARGET_NAME}.html\" target=\"_parent\"><img src=\"$JENKINS_URL/userContent/static/$ICON\" alt=\"${STATE_TARGET_NAME} icon\" /></a>" -} - init_pkg_page() { echo "<!DOCTYPE html><html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />" > ${PKG_FILE} echo "<link href=\"../static/style.css\" type=\"text/css\" rel=\"stylesheet\" />" >> ${PKG_FILE} @@ -496,66 +432,6 @@ link_packages() { done } -write_page_header() { - rm -f $PAGE - write_page "<!DOCTYPE html><html><head>" - write_page "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />" - write_page "<link href=\"$JENKINS_URL/userContent/static/style.css\" type=\"text/css\" rel=\"stylesheet\" />" - write_page "<title>$2</title></head>" - write_page "<body><header><h2>$2</h2>" - if [ "$1" = "$MAINVIEW" ] || ; then - write_page "<p>These pages are updated every six hours. Results are obtained from <a href=\"$JENKINS_URL/view/reproducible\">several jobs running on jenkins.debian.net</a>. Thanks to <a href=\"https://www.profitbricks.com\">Profitbricks</a> for donating the virtual machine it's running on!</p>" - fi - write_page "<p>$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD <a href=\"https://wiki.debian.org/ReproducibleBuilds\">packages should be reproducibly buildable!</a>" - if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] || [ "$1" = "stats" ] ; then - write_page " Join <code>#debian-reproducible</code> on OFTC to get support for making sure your packages build reproducibly too!" - fi - write_page "</p>" - write_page "<ul><li>Have a look at:</li>" - for MY_STATE in $ALLSTATES ; do - WITH="" - if [ "$MY_STATE" = "FTBR_with_buildinfo" ] ; then - WITH="YES" - fi - set_icon $MY_STATE $WITH - write_page "<li>" - write_icon - write_page "</li>" - done - for TARGET in issues notes $ALLVIEWS dd-list stats ; do - if [ "$TARGET" = "issues" ] || [ "$TARGET" = "stats" ]; then - SPOKEN_TARGET=$TARGET - else - SPOKEN_TARGET=${SPOKENTARGET[$TARGET]} - fi - write_page "<li><a href=\"$JENKINS_URL/userContent/index_${TARGET}.html\">${SPOKEN_TARGET}</a></li>" - done - write_page "</ul>" - write_page "</header>" -} - -write_page_footer() { - write_page "<hr/><p style=\"font-size:0.9em;\">There is more information <a href=\"$JENKINS_URL/userContent/about.html\">about jenkins.debian.net</a> and about <a href=\"https://wiki.debian.org/ReproducibleBuilds\"> reproducible builds of Debian</a> available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014 <a href=\"mailto:holger@layer-acht.org\">Holger Levsen</a>, GPL-2 licensed. The weather icons are public domain and have been taken from the <a href="http://tango.freedesktop.org/Tango_Icon_Library" target="_blank">Tango Icon Library</a>.</p>" - write_page "</body></html>" -} - -write_page_meta_sign() { - write_page "<p style=\"font-size:0.9em;\">An underlined package is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue." - if $BETA_SIGN ; then - write_page "A β sign after a package which is unreproducible indicates that a .buildinfo file was generated." - write_page "And that means the <a href=\"https://wiki.debian.org/ReproducibleBuilds#The_basics_for_making_packages_build_reproducible\">basics for building packages reproducibly are covered</a>." - fi - write_page "</p>" -} - -publish_page() { - cp $PAGE /var/lib/jenkins/userContent/ - if [ "$VIEW" = "$MAINVIEW" ] ; then - cp $PAGE /var/lib/jenkins/userContent/reproducible.html - fi - rm $PAGE -} - # # actually parse the notes # diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index eb2980b8..95572bed 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -85,6 +85,10 @@ name: '{name}_stats' - job-template: + defaults: reproducible + name: '{name}_dd_list' + +- job-template: defaults: reproducible_builder name: '{name}_builder_alpha' @@ -108,9 +112,13 @@ my_timed: '42 0 * * *' my_shell: '/srv/jenkins/bin/reproducible_setup.sh' - '{name}_stats': - my_description: 'Generates HTML results for reproducible builds.' + my_description: 'Generates HTML results (various stats) for reproducible builds.' my_timed: '0 0,6,12,18 * * *' my_shell: '/srv/jenkins/bin/reproducible_stats.sh' + - '{name}_dd_list': + my_description: 'Generates HTML results (dd-list) for reproducible builds.' + my_timed: '53 23 * * *' + my_shell: '/srv/jenkins/bin/reproducible_html_dd_list.sh' - '{name}_builder_alpha': my_description: 'Try to reproducibly build a scheduled package. This is one of several builder jobs.' my_timed: 'H/2 * * * *' |