diff options
| -rwxr-xr-x | bin/reproducible_common.sh | 19 | ||||
| -rwxr-xr-x | bin/reproducible_html_graphs.sh (renamed from bin/reproducible_stats.sh) | 162 | ||||
| -rwxr-xr-x | bin/reproducible_html_indexes.sh | 165 | ||||
| -rw-r--r-- | job-cfg/reproducible.yaml | 26 |
4 files changed, 197 insertions, 175 deletions
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 346b8ce4..4808e57d 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -261,15 +261,12 @@ force_package_targets() { } link_packages() { + STAR="" for PKG in $@ ; do - STAR="" - write_page " ${LINKTARGET[$PKG]}" if $BUILDINFO_SIGNS ; then set_package_star - if [ ! -z "$STAR" ] ; then - write_page "$STAR " - fi fi + write_page " ${LINKTARGET[$PKG]} $STAR" done } @@ -351,4 +348,14 @@ process_packages() { done } - +gather_stats() { + COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"unreproducible\"") + COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"FTBFS\"") + COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"404\"") + COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"not for us\"") + COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"blacklisted\"") + PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) + PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) + PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) + PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) +} diff --git a/bin/reproducible_stats.sh b/bin/reproducible_html_graphs.sh index 88bd75b5..71d54ee9 100755 --- a/bin/reproducible_stats.sh +++ b/bin/reproducible_html_graphs.sh @@ -15,170 +15,12 @@ cp $PACKAGES_DB /var/lib/jenkins/userContent/reproducible.db set +x init_html - -declare -A GOOD -declare -A BAD -declare -A UGLY -declare -A SOURCELESS -declare -A NOTFORUS -LAST24="AND build_date > datetime('now', '-24 hours') " -LAST48="AND build_date > datetime('now', '-48 hours') " -GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY build_date DESC" | xargs echo) -GOOD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) -GOOD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) -GOOD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY name" | xargs echo) -BAD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo) -BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) -BAD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) -BAD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY name" | xargs echo) -COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"unreproducible\"") -UGLY["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY build_date DESC" | xargs echo) -UGLY["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST24 ORDER BY build_date DESC" | xargs echo) -UGLY["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST48 ORDER BY build_date DESC" | xargs echo) -UGLY["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY name" | xargs echo) -COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"FTBFS\"") -SOURCELESS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY build_date DESC" | xargs echo) -SOURCELESS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY name" | xargs echo) -COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"404\"" | xargs echo) -NOTFORUS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY build_date DESC" | xargs echo) -NOTFORUS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY name" | xargs echo) -COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"not for us\"" | xargs echo) -BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"blacklisted\" ORDER BY name" | xargs echo) -COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"blacklisted\"" | xargs echo) -PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) -PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) -PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) -PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) - -# -# actually build the package pages -# -echo "$(date) - processing $COUNT_TOTAL packages... this will take a while." -force_package_targets ${BAD["all"]} -force_package_targets ${UGLY["all"]} ${GOOD["all"]} ${SOURCELESS["all"]} ${NOTFORUS["all"]} $BLACKLISTED - -for VIEW in $ALLVIEWS ; do - BUILDINFO_SIGNS=true - PAGE=index_${VIEW}.html - echo "$(date) - starting to write $PAGE page." - write_page_header $VIEW "Overview of reproducible builds of ${SPOKENTARGET[$VIEW]}" - if [ "${VIEW:0:3}" = "all" ] ; then - FINISH=":" - else - SHORTER_SPOKENTARGET=$(echo ${SPOKENTARGET[$VIEW]} | cut -d "(" -f1) - FINISH=", from $SHORTER_SPOKENTARGET these were:" - fi - write_page "<p>" - set_icon unreproducible with - write_icon - set_icon unreproducible - write_icon - write_page "$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly in total$FINISH <code>" - link_packages ${BAD[$VIEW]} - write_page "</code></p>" - write_page - write_page "<p>" - set_icon FTBFS - write_icon - write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source in total$FINISH <code>" - link_packages ${UGLY[$VIEW]} - write_page "</code></p>" - if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_SOURCELESS -gt 0 ] ; then - write_page "<p>For " - set_icon 404 - write_icon - write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages in total sources could not be downloaded: <code>" - link_packages ${SOURCELESS[$VIEW]} - write_page "</code></p>" - fi - if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_NOTFORUS -gt 0 ] ; then - write_page "<p>In total there were " - set_icon not_for_us - write_icon - write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64': <code>" - link_packages ${NOTFORUS[$VIEW]} - write_page "</code></p>" - fi - if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_BLACKLISTED -gt 0 ] ; then - write_page "<p>" - set_icon blacklisted - write_icon - write_page "$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: <code>" - link_packages $BLACKLISTED - write_page "</code></p>" - fi - write_page "<p>" - set_icon reproducible - write_icon - write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH <code>" - BUILDINFO_SIGNS=false - link_packages ${GOOD[$VIEW]} - write_page "</code></p>" - write_page_meta_sign - write_page_footer - publish_page -done - -count_packages() { - COUNT=${#@} - PERCENT=$(echo "scale=1 ; ($COUNT*100/$COUNT_TOTAL)" | bc) -} - -for STATE in $ALLSTATES ; do - BUILDINFO_SIGNS=true - PAGE=index_${STATE}.html - echo "$(date) - starting to write $PAGE page." - write_page_header $STATE "Overview of ${SPOKENTARGET[$STATE]}" - WITH="" - case "$STATE" in - reproducible) BUILDINFO_SIGNS=false - PACKAGES=${GOOD["all"]} - ;; - FTBR) CANDIDATES=${BAD["all"]} - PACKAGES="" - for PKG in $CANDIDATES ; do - set_package_star - if [ "$STAR" = "" ] ; then - PACKAGES="$PACKAGES $PKG" - fi - done - ;; - FTBR_with_buildinfo) CANDIDATES=${BAD["all"]} - PACKAGES="" - for PKG in $CANDIDATES ; do - set_package_star - if [ "$STAR" != "" ] ; then - PACKAGES="$PACKAGES $PKG" - fi - done - WITH="YES" - ;; - FTBFS) PACKAGES=${UGLY["all"]} - ;; - 404) PACKAGES=${SOURCELESS["all"]} - ;; - not_for_us) PACKAGES=${NOTFORUS["all"]} - ;; - blacklisted) PACKAGES=${BLACKLISTED} - ;; - esac - count_packages ${PACKAGES} - write_page "<p>" - set_icon $STATE $WITH - write_icon - write_page "$COUNT ($PERCENT%) ${SPOKENTARGET[$STATE]}:<code>" - link_packages ${PACKAGES} - write_page "</code></p>" - write_page - write_page_meta_sign - write_page_footer - publish_page -done +gather_stats # # create stats # -# FIXME: we only do stats up until yesterday... we also could do today too but not update the db yet... +# FIXME?: we only do stats up until yesterday... we also could do today too but not update the db yet... DATE=$(date -d "1 day ago" '+%Y-%m-%d') TABLE[0]=stats_pkg_state TABLE[1]=stats_builds_per_day diff --git a/bin/reproducible_html_indexes.sh b/bin/reproducible_html_indexes.sh new file mode 100755 index 00000000..f6ed419f --- /dev/null +++ b/bin/reproducible_html_indexes.sh @@ -0,0 +1,165 @@ +#!/bin/bash + +# Copyright 2014 Holger Levsen <holger@layer-acht.org> +# released under the GPLv=2 + +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code defining db access +. /srv/jenkins/bin/reproducible_common.sh + +set +x +init_html + +declare -A GOOD +declare -A BAD +declare -A UGLY +declare -A SOURCELESS +declare -A NOTFORUS +LAST24="AND build_date > datetime('now', '-24 hours') " +LAST48="AND build_date > datetime('now', '-48 hours') " +GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY build_date DESC" | xargs echo) +GOOD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) +GOOD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) +GOOD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY name" | xargs echo) +BAD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo) +BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) +BAD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) +BAD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY name" | xargs echo) +UGLY["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY build_date DESC" | xargs echo) +UGLY["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST24 ORDER BY build_date DESC" | xargs echo) +UGLY["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST48 ORDER BY build_date DESC" | xargs echo) +UGLY["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY name" | xargs echo) +SOURCELESS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY build_date DESC" | xargs echo) +SOURCELESS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY name" | xargs echo) +NOTFORUS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY build_date DESC" | xargs echo) +NOTFORUS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY name" | xargs echo) +BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"blacklisted\" ORDER BY name" | xargs echo) +gather_stats + +# +# actually build the package pages +# +echo "$(date) - processing $COUNT_TOTAL packages... this will take a while." +force_package_targets ${BAD["all"]} +force_package_targets ${UGLY["all"]} ${GOOD["all"]} ${SOURCELESS["all"]} ${NOTFORUS["all"]} $BLACKLISTED + +for VIEW in $ALLVIEWS ; do + BUILDINFO_SIGNS=true + PAGE=index_${VIEW}.html + echo "$(date) - starting to write $PAGE page." + write_page_header $VIEW "Overview of reproducible builds of ${SPOKENTARGET[$VIEW]}" + if [ "${VIEW:0:3}" = "all" ] ; then + FINISH=":" + else + SHORTER_SPOKENTARGET=$(echo ${SPOKENTARGET[$VIEW]} | cut -d "(" -f1) + FINISH=", from $SHORTER_SPOKENTARGET these were:" + fi + write_page "<p>" + set_icon unreproducible with + write_icon + set_icon unreproducible + write_icon + write_page "$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly in total$FINISH <code>" + link_packages ${BAD[$VIEW]} + write_page "</code></p>" + write_page + write_page "<p>" + set_icon FTBFS + write_icon + write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source in total$FINISH <code>" + link_packages ${UGLY[$VIEW]} + write_page "</code></p>" + if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_SOURCELESS -gt 0 ] ; then + write_page "<p>For " + set_icon 404 + write_icon + write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages in total sources could not be downloaded: <code>" + link_packages ${SOURCELESS[$VIEW]} + write_page "</code></p>" + fi + if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_NOTFORUS -gt 0 ] ; then + write_page "<p>In total there were " + set_icon not_for_us + write_icon + write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64': <code>" + link_packages ${NOTFORUS[$VIEW]} + write_page "</code></p>" + fi + if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_BLACKLISTED -gt 0 ] ; then + write_page "<p>" + set_icon blacklisted + write_icon + write_page "$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: <code>" + link_packages $BLACKLISTED + write_page "</code></p>" + fi + write_page "<p>" + set_icon reproducible + write_icon + write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH <code>" + BUILDINFO_SIGNS=false + link_packages ${GOOD[$VIEW]} + write_page "</code></p>" + write_page_meta_sign + write_page_footer + publish_page +done + +count_packages() { + COUNT=${#@} + PERCENT=$(echo "scale=1 ; ($COUNT*100/$COUNT_TOTAL)" | bc) +} + +for STATE in $ALLSTATES ; do + BUILDINFO_SIGNS=true + PAGE=index_${STATE}.html + echo "$(date) - starting to write $PAGE page." + write_page_header $STATE "Overview of ${SPOKENTARGET[$STATE]}" + WITH="" + case "$STATE" in + reproducible) BUILDINFO_SIGNS=false + PACKAGES=${GOOD["all"]} + ;; + FTBR) CANDIDATES=${BAD["all"]} + PACKAGES="" + for PKG in $CANDIDATES ; do + set_package_star + if [ "$STAR" = "" ] ; then + PACKAGES="$PACKAGES $PKG" + fi + done + ;; + FTBR_with_buildinfo) CANDIDATES=${BAD["all"]} + PACKAGES="" + for PKG in $CANDIDATES ; do + set_package_star + if [ "$STAR" != "" ] ; then + PACKAGES="$PACKAGES $PKG" + fi + done + WITH="YES" + ;; + FTBFS) PACKAGES=${UGLY["all"]} + ;; + 404) PACKAGES=${SOURCELESS["all"]} + ;; + not_for_us) PACKAGES=${NOTFORUS["all"]} + ;; + blacklisted) PACKAGES=${BLACKLISTED} + ;; + esac + count_packages ${PACKAGES} + write_page "<p>" + set_icon $STATE $WITH + write_icon + write_page "$COUNT ($PERCENT%) ${SPOKENTARGET[$STATE]}:<code>" + link_packages ${PACKAGES} + write_page "</code></p>" + write_page + write_page_meta_sign + write_page_footer + publish_page +done + diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index 6390fac8..121285eb 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -82,15 +82,19 @@ - job-template: defaults: reproducible - name: '{name}_stats' + name: '{name}_html_graphs' - job-template: defaults: reproducible - name: '{name}_dd_list' + name: '{name}_html_indexes' - job-template: defaults: reproducible - name: '{name}_notes' + name: '{name}_html_dd_list' + +- job-template: + defaults: reproducible + name: '{name}_html_notes' - job-template: defaults: reproducible_builder @@ -115,15 +119,19 @@ my_description: 'Setup pbuilder for reproducible builds as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' my_timed: '42 0 * * *' my_shell: '/srv/jenkins/bin/reproducible_setup.sh' - - '{name}_stats': - my_description: 'Generates HTML results (various stats) for reproducible builds.' - my_timed: '0 0,6,12,18 * * *' - my_shell: '/srv/jenkins/bin/reproducible_stats.sh' - - '{name}_dd_list': + - '{name}_html_indexes': + my_description: 'Generates HTML results (indexes) for reproducible builds.' + my_timed: '0 3,6,9,12,15,18,21 * * *' + my_shell: '/srv/jenkins/bin/reproducible_html_indexes.sh' + - '{name}_html_graphs': + my_description: 'Generates HTML results (stats with graphs) for reproducible builds.' + my_timed: '0 0 * * *' + my_shell: '/srv/jenkins/bin/reproducible_html_graphs.sh' + - '{name}_html_dd_list': my_description: 'Generates HTML results (dd-list) for reproducible builds.' my_timed: '53 23 * * *' my_shell: '/srv/jenkins/bin/reproducible_html_dd_list.sh' - - '{name}_notes': + - '{name}_html_notes': my_description: 'Generates HTML results (notes) for reproducible builds, triggered by changes to the notes.git repository.' my_timed: '' my_shell: '/srv/jenkins/bin/reproducible_html_notes.sh' |