From 98f55ed3b3c5d25870cca37e4f4e8325a58514f3 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Fri, 17 Oct 2014 20:46:14 +0200 Subject: reproducible: refactor, split stats script+job in html_indexes and html_graphs. --- bin/reproducible_common.sh | 19 ++- bin/reproducible_html_graphs.sh | 186 +++++++++++++++++++++ bin/reproducible_html_indexes.sh | 165 +++++++++++++++++++ bin/reproducible_stats.sh | 344 --------------------------------------- job-cfg/reproducible.yaml | 26 ++- 5 files changed, 381 insertions(+), 359 deletions(-) create mode 100755 bin/reproducible_html_graphs.sh create mode 100755 bin/reproducible_html_indexes.sh delete mode 100755 bin/reproducible_stats.sh diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 346b8ce4..4808e57d 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -261,15 +261,12 @@ force_package_targets() { } link_packages() { + STAR="" for PKG in $@ ; do - STAR="" - write_page " ${LINKTARGET[$PKG]}" if $BUILDINFO_SIGNS ; then set_package_star - if [ ! -z "$STAR" ] ; then - write_page "$STAR " - fi fi + write_page " ${LINKTARGET[$PKG]} $STAR" done } @@ -351,4 +348,14 @@ process_packages() { done } - +gather_stats() { + COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"unreproducible\"") + COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"FTBFS\"") + COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"404\"") + COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"not for us\"") + COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"blacklisted\"") + PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) + PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) + PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) + PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) +} diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_graphs.sh new file mode 100755 index 00000000..71d54ee9 --- /dev/null +++ b/bin/reproducible_html_graphs.sh @@ -0,0 +1,186 @@ +#!/bin/bash + +# Copyright 2014 Holger Levsen +# released under the GPLv=2 + +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code defining db access +. /srv/jenkins/bin/reproducible_common.sh + +# FIXME: move to daily cleanup job +# cp db away for backup purposes +cp $PACKAGES_DB /var/lib/jenkins/userContent/reproducible.db + +set +x +init_html +gather_stats + +# +# create stats +# +# FIXME?: we only do stats up until yesterday... we also could do today too but not update the db yet... +DATE=$(date -d "1 day ago" '+%Y-%m-%d') +TABLE[0]=stats_pkg_state +TABLE[1]=stats_builds_per_day +TABLE[2]=stats_builds_age +TABLE[3]=stats_bugs +RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,suite from ${TABLE[0]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\"") +if [ -z $RESULT ] ; then + ALL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(name) from sources") + GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)<='$DATE';") + GOOAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)='$DATE';") + BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)<='$DATE';") + BAAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)='$DATE';") + UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)<='$DATE';") + UGLDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)='$DATE';") + REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)<='$DATE';") + RESDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)='$DATE';") + OLDESTG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT build_date FROM source_packages WHERE status = 'reproducible' AND NOT date(build_date)>='$DATE' ORDER BY build_date LIMIT 1;") + OLDESTB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT build_date FROM source_packages WHERE status = 'unreproducible' AND NOT date(build_date)>='$DATE' ORDER BY build_date LIMIT 1;") + OLDESTU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT build_date FROM source_packages WHERE status = 'FTBFS' AND NOT date(build_date)>='$DATE' ORDER BY build_date LIMIT 1;") + DIFFG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTG');") + if [ -z $DIFFG ] ; then DIFFG=0 ; fi + DIFFB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTB');") + if [ -z $DIFFB ] ; then DIFFB=0 ; fi + DIFFU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTU');") + if [ -z $DIFFU ] ; then DIFFU=0 ; fi + let "TOTAL=GOOD+BAD+UGLY+REST" + let "UNTESTED=ALL-TOTAL" + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[0]} VALUES (\"$DATE\", \"$SUITE\", $UNTESTED, $GOOD, $BAD, $UGLY, $REST)" + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)" + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")" + # FIXME: we don't do 2 / stats_builds_age.png yet :/ (also see below) + for i in 0 1 ; do + # force regeneration of the image + touch -d "$DATE 00:00" ${TABLE[$i]}.png + done +fi + +# query bts +USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness" +RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") +if [ -z $RESULT ] ; then + declare -a DONE + declare -a OPEN + SQL="INSERT INTO ${TABLE[3]} VALUES (\"$DATE\" " + for TAG in $USERTAGS ; do + OPEN[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:open status:forwarded 2>/dev/null|wc -l) + DONE[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:done archive:both 2>/dev/null|wc -l) + # test if both values are integers + if ! ( [[ ${DONE[$TAG]} =~ ^-?[0-9]+$ ]] && [[ ${OPEN[$TAG]} =~ ^-?[0-9]+$ ]] ) ; then + echo "Non-integers value detected, exiting." + echo "Usertag: $TAG" + echo "Open: ${OPEN[$TAG]}" + echo "Done: ${DONE[$TAG]}" + exit 1 + fi + SQL="$SQL, ${OPEN[$TAG]}, ${DONE[$TAG]}" + done + SQL="$SQL)" + echo $SQL + sqlite3 -init ${INIT} ${PACKAGES_DB} "$SQL" + # force regeneration of the image + touch -d "$DATE 00:00" ${TABLE[3]}.png +fi + +# used for redo_png (but only needed to define once) +FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested" +FIELDS[1]="datum, reproducible, unreproducible, FTBFS, other" +FIELDS[2]="datum, oldest_reproducible, oldest_unreproducible, oldest_FTBFS" +FIELDS[3]="datum " +for TAG in $USERTAGS ; do + FIELDS[3]="${FIELDS[3]}, open_$TAG, done_$TAG" +done +COLOR[0]=5 +COLOR[1]=4 +COLOR[2]=3 +COLOR[3]=18 +MAINLABEL[0]="Package reproducibility status" +MAINLABEL[1]="Amout of packages build each day" +MAINLABEL[2]="Age in days of oldest kind of logfile" +MAINLABEL[3]="Bugs with usertags for user reproducible-builds@lists.alioth.debian.org" +YLABEL[0]="Amount (total)" +YLABEL[1]="Amount (per day)" +YLABEL[2]="Age in days" +YLABEL[3]="Amount of bugs" +redo_png() { + echo "${FIELDS[$i]}" > ${TABLE[$i]}.csv + # TABLE[3] doesn't have a suite column... + if [ $i -ne 3 ] ; then + WHERE_SUITE="WHERE suite = '$SUITE'" + else + WHERE_SUITE="" + fi + sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$i]} from ${TABLE[$i]} ${WHERE_SUITE} ORDER BY datum" >> ${TABLE[$i]}.csv + /srv/jenkins/bin/make_graph.py ${TABLE[$i]}.csv ${TABLE[$i]}.png ${COLOR[$i]} "${MAINLABEL[$i]}" "${YLABEL[$i]}" + rm ${TABLE[$i]}.csv + mv ${TABLE[$i]}.png /var/lib/jenkins/userContent/ +} + +write_usertag_table() { + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") + if [ -z "$RESULTS" ] ; then + COUNT=0 + for FIELD in $(echo ${FIELDS[3]} | tr -d ,) ; do + let "COUNT+=1" + VALUE=$(echo $RESULT | cut -d "|" -f$COUNT) + if [ $COUNT -eq 1 ] ; then + write_page "" + elif [ $((COUNT%2)) -eq 0 ] ; then + write_page "" + else + write_page "" + fi + done + write_page "
Bugs with usertags for reproducible-builds@lists.alioth.debian.org on $VALUE
 ${FIELD:5}Open: $VALUEDone: $VALUE
" + fi +} + +VIEW=stats +PAGE=index_${VIEW}.html +echo "$(date) - starting to write $PAGE page." +write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}" +write_page "

" +set_icon reproducible +write_icon +write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly." +set_icon unreproducible with +write_icon +set_icon unreproducible +write_icon +write_page "$COUNT_BAD packages ($PERCENT_BAD%) failed to built reproducibly." +set_icon FTBFS +write_icon +write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source.

" +write_page "

" +if [ $COUNT_SOURCELESS -gt 0 ] ; then + write_page "For " + set_icon 404 + write_icon + write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages sources could not be downloaded," +fi +set_icon not_for_us +write_icon +write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64' will never be build here" +write_page "and those " +set_icon blacklisted +write_icon +write_page "$COUNT_BLACKLISTED blacklisted packages neither.

" +write_page "

" +# FIXME: we don't do 2 / stats_builds_age.png yet :/ (also see above) +for i in 0 1 3 ; do + if [ "$i" = "3" ] ; then + write_usertag_table + fi + write_page " \"${MAINLABEL[$i]}\"" + # redo pngs once a day + if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime -1 -name ${TABLE[$i]}.png) ] ; then + redo_png + fi +done +write_page "

" +write_page_footer +publish_page + diff --git a/bin/reproducible_html_indexes.sh b/bin/reproducible_html_indexes.sh new file mode 100755 index 00000000..f6ed419f --- /dev/null +++ b/bin/reproducible_html_indexes.sh @@ -0,0 +1,165 @@ +#!/bin/bash + +# Copyright 2014 Holger Levsen +# released under the GPLv=2 + +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code defining db access +. /srv/jenkins/bin/reproducible_common.sh + +set +x +init_html + +declare -A GOOD +declare -A BAD +declare -A UGLY +declare -A SOURCELESS +declare -A NOTFORUS +LAST24="AND build_date > datetime('now', '-24 hours') " +LAST48="AND build_date > datetime('now', '-48 hours') " +GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY build_date DESC" | xargs echo) +GOOD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) +GOOD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) +GOOD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY name" | xargs echo) +BAD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo) +BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) +BAD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) +BAD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY name" | xargs echo) +UGLY["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY build_date DESC" | xargs echo) +UGLY["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST24 ORDER BY build_date DESC" | xargs echo) +UGLY["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST48 ORDER BY build_date DESC" | xargs echo) +UGLY["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY name" | xargs echo) +SOURCELESS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY build_date DESC" | xargs echo) +SOURCELESS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY name" | xargs echo) +NOTFORUS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY build_date DESC" | xargs echo) +NOTFORUS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY name" | xargs echo) +BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"blacklisted\" ORDER BY name" | xargs echo) +gather_stats + +# +# actually build the package pages +# +echo "$(date) - processing $COUNT_TOTAL packages... this will take a while." +force_package_targets ${BAD["all"]} +force_package_targets ${UGLY["all"]} ${GOOD["all"]} ${SOURCELESS["all"]} ${NOTFORUS["all"]} $BLACKLISTED + +for VIEW in $ALLVIEWS ; do + BUILDINFO_SIGNS=true + PAGE=index_${VIEW}.html + echo "$(date) - starting to write $PAGE page." + write_page_header $VIEW "Overview of reproducible builds of ${SPOKENTARGET[$VIEW]}" + if [ "${VIEW:0:3}" = "all" ] ; then + FINISH=":" + else + SHORTER_SPOKENTARGET=$(echo ${SPOKENTARGET[$VIEW]} | cut -d "(" -f1) + FINISH=", from $SHORTER_SPOKENTARGET these were:" + fi + write_page "

" + set_icon unreproducible with + write_icon + set_icon unreproducible + write_icon + write_page "$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly in total$FINISH " + link_packages ${BAD[$VIEW]} + write_page "

" + write_page + write_page "

" + set_icon FTBFS + write_icon + write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source in total$FINISH " + link_packages ${UGLY[$VIEW]} + write_page "

" + if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_SOURCELESS -gt 0 ] ; then + write_page "

For " + set_icon 404 + write_icon + write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages in total sources could not be downloaded: " + link_packages ${SOURCELESS[$VIEW]} + write_page "

" + fi + if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_NOTFORUS -gt 0 ] ; then + write_page "

In total there were " + set_icon not_for_us + write_icon + write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64': " + link_packages ${NOTFORUS[$VIEW]} + write_page "

" + fi + if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_BLACKLISTED -gt 0 ] ; then + write_page "

" + set_icon blacklisted + write_icon + write_page "$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: " + link_packages $BLACKLISTED + write_page "

" + fi + write_page "

" + set_icon reproducible + write_icon + write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH " + BUILDINFO_SIGNS=false + link_packages ${GOOD[$VIEW]} + write_page "

" + write_page_meta_sign + write_page_footer + publish_page +done + +count_packages() { + COUNT=${#@} + PERCENT=$(echo "scale=1 ; ($COUNT*100/$COUNT_TOTAL)" | bc) +} + +for STATE in $ALLSTATES ; do + BUILDINFO_SIGNS=true + PAGE=index_${STATE}.html + echo "$(date) - starting to write $PAGE page." + write_page_header $STATE "Overview of ${SPOKENTARGET[$STATE]}" + WITH="" + case "$STATE" in + reproducible) BUILDINFO_SIGNS=false + PACKAGES=${GOOD["all"]} + ;; + FTBR) CANDIDATES=${BAD["all"]} + PACKAGES="" + for PKG in $CANDIDATES ; do + set_package_star + if [ "$STAR" = "" ] ; then + PACKAGES="$PACKAGES $PKG" + fi + done + ;; + FTBR_with_buildinfo) CANDIDATES=${BAD["all"]} + PACKAGES="" + for PKG in $CANDIDATES ; do + set_package_star + if [ "$STAR" != "" ] ; then + PACKAGES="$PACKAGES $PKG" + fi + done + WITH="YES" + ;; + FTBFS) PACKAGES=${UGLY["all"]} + ;; + 404) PACKAGES=${SOURCELESS["all"]} + ;; + not_for_us) PACKAGES=${NOTFORUS["all"]} + ;; + blacklisted) PACKAGES=${BLACKLISTED} + ;; + esac + count_packages ${PACKAGES} + write_page "

" + set_icon $STATE $WITH + write_icon + write_page "$COUNT ($PERCENT%) ${SPOKENTARGET[$STATE]}:" + link_packages ${PACKAGES} + write_page "

" + write_page + write_page_meta_sign + write_page_footer + publish_page +done + diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh deleted file mode 100755 index 88bd75b5..00000000 --- a/bin/reproducible_stats.sh +++ /dev/null @@ -1,344 +0,0 @@ -#!/bin/bash - -# Copyright 2014 Holger Levsen -# released under the GPLv=2 - -. /srv/jenkins/bin/common-functions.sh -common_init "$@" - -# common code defining db access -. /srv/jenkins/bin/reproducible_common.sh - -# FIXME: move to daily cleanup job -# cp db away for backup purposes -cp $PACKAGES_DB /var/lib/jenkins/userContent/reproducible.db - -set +x -init_html - -declare -A GOOD -declare -A BAD -declare -A UGLY -declare -A SOURCELESS -declare -A NOTFORUS -LAST24="AND build_date > datetime('now', '-24 hours') " -LAST48="AND build_date > datetime('now', '-48 hours') " -GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY build_date DESC" | xargs echo) -GOOD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) -GOOD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) -GOOD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY name" | xargs echo) -BAD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo) -BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) -BAD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) -BAD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY name" | xargs echo) -COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"unreproducible\"") -UGLY["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY build_date DESC" | xargs echo) -UGLY["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST24 ORDER BY build_date DESC" | xargs echo) -UGLY["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST48 ORDER BY build_date DESC" | xargs echo) -UGLY["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY name" | xargs echo) -COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"FTBFS\"") -SOURCELESS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY build_date DESC" | xargs echo) -SOURCELESS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY name" | xargs echo) -COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"404\"" | xargs echo) -NOTFORUS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY build_date DESC" | xargs echo) -NOTFORUS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY name" | xargs echo) -COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"not for us\"" | xargs echo) -BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"blacklisted\" ORDER BY name" | xargs echo) -COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"blacklisted\"" | xargs echo) -PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) -PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) -PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) -PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) - -# -# actually build the package pages -# -echo "$(date) - processing $COUNT_TOTAL packages... this will take a while." -force_package_targets ${BAD["all"]} -force_package_targets ${UGLY["all"]} ${GOOD["all"]} ${SOURCELESS["all"]} ${NOTFORUS["all"]} $BLACKLISTED - -for VIEW in $ALLVIEWS ; do - BUILDINFO_SIGNS=true - PAGE=index_${VIEW}.html - echo "$(date) - starting to write $PAGE page." - write_page_header $VIEW "Overview of reproducible builds of ${SPOKENTARGET[$VIEW]}" - if [ "${VIEW:0:3}" = "all" ] ; then - FINISH=":" - else - SHORTER_SPOKENTARGET=$(echo ${SPOKENTARGET[$VIEW]} | cut -d "(" -f1) - FINISH=", from $SHORTER_SPOKENTARGET these were:" - fi - write_page "

" - set_icon unreproducible with - write_icon - set_icon unreproducible - write_icon - write_page "$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly in total$FINISH " - link_packages ${BAD[$VIEW]} - write_page "

" - write_page - write_page "

" - set_icon FTBFS - write_icon - write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source in total$FINISH " - link_packages ${UGLY[$VIEW]} - write_page "

" - if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_SOURCELESS -gt 0 ] ; then - write_page "

For " - set_icon 404 - write_icon - write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages in total sources could not be downloaded: " - link_packages ${SOURCELESS[$VIEW]} - write_page "

" - fi - if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_NOTFORUS -gt 0 ] ; then - write_page "

In total there were " - set_icon not_for_us - write_icon - write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64': " - link_packages ${NOTFORUS[$VIEW]} - write_page "

" - fi - if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_BLACKLISTED -gt 0 ] ; then - write_page "

" - set_icon blacklisted - write_icon - write_page "$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: " - link_packages $BLACKLISTED - write_page "

" - fi - write_page "

" - set_icon reproducible - write_icon - write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH " - BUILDINFO_SIGNS=false - link_packages ${GOOD[$VIEW]} - write_page "

" - write_page_meta_sign - write_page_footer - publish_page -done - -count_packages() { - COUNT=${#@} - PERCENT=$(echo "scale=1 ; ($COUNT*100/$COUNT_TOTAL)" | bc) -} - -for STATE in $ALLSTATES ; do - BUILDINFO_SIGNS=true - PAGE=index_${STATE}.html - echo "$(date) - starting to write $PAGE page." - write_page_header $STATE "Overview of ${SPOKENTARGET[$STATE]}" - WITH="" - case "$STATE" in - reproducible) BUILDINFO_SIGNS=false - PACKAGES=${GOOD["all"]} - ;; - FTBR) CANDIDATES=${BAD["all"]} - PACKAGES="" - for PKG in $CANDIDATES ; do - set_package_star - if [ "$STAR" = "" ] ; then - PACKAGES="$PACKAGES $PKG" - fi - done - ;; - FTBR_with_buildinfo) CANDIDATES=${BAD["all"]} - PACKAGES="" - for PKG in $CANDIDATES ; do - set_package_star - if [ "$STAR" != "" ] ; then - PACKAGES="$PACKAGES $PKG" - fi - done - WITH="YES" - ;; - FTBFS) PACKAGES=${UGLY["all"]} - ;; - 404) PACKAGES=${SOURCELESS["all"]} - ;; - not_for_us) PACKAGES=${NOTFORUS["all"]} - ;; - blacklisted) PACKAGES=${BLACKLISTED} - ;; - esac - count_packages ${PACKAGES} - write_page "

" - set_icon $STATE $WITH - write_icon - write_page "$COUNT ($PERCENT%) ${SPOKENTARGET[$STATE]}:" - link_packages ${PACKAGES} - write_page "

" - write_page - write_page_meta_sign - write_page_footer - publish_page -done - -# -# create stats -# -# FIXME: we only do stats up until yesterday... we also could do today too but not update the db yet... -DATE=$(date -d "1 day ago" '+%Y-%m-%d') -TABLE[0]=stats_pkg_state -TABLE[1]=stats_builds_per_day -TABLE[2]=stats_builds_age -TABLE[3]=stats_bugs -RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,suite from ${TABLE[0]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\"") -if [ -z $RESULT ] ; then - ALL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(name) from sources") - GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)<='$DATE';") - GOOAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)='$DATE';") - BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)<='$DATE';") - BAAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)='$DATE';") - UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)<='$DATE';") - UGLDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)='$DATE';") - REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)<='$DATE';") - RESDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)='$DATE';") - OLDESTG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT build_date FROM source_packages WHERE status = 'reproducible' AND NOT date(build_date)>='$DATE' ORDER BY build_date LIMIT 1;") - OLDESTB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT build_date FROM source_packages WHERE status = 'unreproducible' AND NOT date(build_date)>='$DATE' ORDER BY build_date LIMIT 1;") - OLDESTU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT build_date FROM source_packages WHERE status = 'FTBFS' AND NOT date(build_date)>='$DATE' ORDER BY build_date LIMIT 1;") - DIFFG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTG');") - if [ -z $DIFFG ] ; then DIFFG=0 ; fi - DIFFB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTB');") - if [ -z $DIFFB ] ; then DIFFB=0 ; fi - DIFFU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTU');") - if [ -z $DIFFU ] ; then DIFFU=0 ; fi - let "TOTAL=GOOD+BAD+UGLY+REST" - let "UNTESTED=ALL-TOTAL" - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[0]} VALUES (\"$DATE\", \"$SUITE\", $UNTESTED, $GOOD, $BAD, $UGLY, $REST)" - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)" - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")" - # FIXME: we don't do 2 / stats_builds_age.png yet :/ (also see below) - for i in 0 1 ; do - # force regeneration of the image - touch -d "$DATE 00:00" ${TABLE[$i]}.png - done -fi - -# query bts -USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness" -RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") -if [ -z $RESULT ] ; then - declare -a DONE - declare -a OPEN - SQL="INSERT INTO ${TABLE[3]} VALUES (\"$DATE\" " - for TAG in $USERTAGS ; do - OPEN[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:open status:forwarded 2>/dev/null|wc -l) - DONE[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:done archive:both 2>/dev/null|wc -l) - # test if both values are integers - if ! ( [[ ${DONE[$TAG]} =~ ^-?[0-9]+$ ]] && [[ ${OPEN[$TAG]} =~ ^-?[0-9]+$ ]] ) ; then - echo "Non-integers value detected, exiting." - echo "Usertag: $TAG" - echo "Open: ${OPEN[$TAG]}" - echo "Done: ${DONE[$TAG]}" - exit 1 - fi - SQL="$SQL, ${OPEN[$TAG]}, ${DONE[$TAG]}" - done - SQL="$SQL)" - echo $SQL - sqlite3 -init ${INIT} ${PACKAGES_DB} "$SQL" - # force regeneration of the image - touch -d "$DATE 00:00" ${TABLE[3]}.png -fi - -# used for redo_png (but only needed to define once) -FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested" -FIELDS[1]="datum, reproducible, unreproducible, FTBFS, other" -FIELDS[2]="datum, oldest_reproducible, oldest_unreproducible, oldest_FTBFS" -FIELDS[3]="datum " -for TAG in $USERTAGS ; do - FIELDS[3]="${FIELDS[3]}, open_$TAG, done_$TAG" -done -COLOR[0]=5 -COLOR[1]=4 -COLOR[2]=3 -COLOR[3]=18 -MAINLABEL[0]="Package reproducibility status" -MAINLABEL[1]="Amout of packages build each day" -MAINLABEL[2]="Age in days of oldest kind of logfile" -MAINLABEL[3]="Bugs with usertags for user reproducible-builds@lists.alioth.debian.org" -YLABEL[0]="Amount (total)" -YLABEL[1]="Amount (per day)" -YLABEL[2]="Age in days" -YLABEL[3]="Amount of bugs" -redo_png() { - echo "${FIELDS[$i]}" > ${TABLE[$i]}.csv - # TABLE[3] doesn't have a suite column... - if [ $i -ne 3 ] ; then - WHERE_SUITE="WHERE suite = '$SUITE'" - else - WHERE_SUITE="" - fi - sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$i]} from ${TABLE[$i]} ${WHERE_SUITE} ORDER BY datum" >> ${TABLE[$i]}.csv - /srv/jenkins/bin/make_graph.py ${TABLE[$i]}.csv ${TABLE[$i]}.png ${COLOR[$i]} "${MAINLABEL[$i]}" "${YLABEL[$i]}" - rm ${TABLE[$i]}.csv - mv ${TABLE[$i]}.png /var/lib/jenkins/userContent/ -} - -write_usertag_table() { - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") - if [ -z "$RESULTS" ] ; then - COUNT=0 - for FIELD in $(echo ${FIELDS[3]} | tr -d ,) ; do - let "COUNT+=1" - VALUE=$(echo $RESULT | cut -d "|" -f$COUNT) - if [ $COUNT -eq 1 ] ; then - write_page "" - elif [ $((COUNT%2)) -eq 0 ] ; then - write_page "" - else - write_page "" - fi - done - write_page "
Bugs with usertags for reproducible-builds@lists.alioth.debian.org on $VALUE
 ${FIELD:5}Open: $VALUEDone: $VALUE
" - fi -} - -VIEW=stats -PAGE=index_${VIEW}.html -echo "$(date) - starting to write $PAGE page." -write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}" -write_page "

" -set_icon reproducible -write_icon -write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly." -set_icon unreproducible with -write_icon -set_icon unreproducible -write_icon -write_page "$COUNT_BAD packages ($PERCENT_BAD%) failed to built reproducibly." -set_icon FTBFS -write_icon -write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source.

" -write_page "

" -if [ $COUNT_SOURCELESS -gt 0 ] ; then - write_page "For " - set_icon 404 - write_icon - write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages sources could not be downloaded," -fi -set_icon not_for_us -write_icon -write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64' will never be build here" -write_page "and those " -set_icon blacklisted -write_icon -write_page "$COUNT_BLACKLISTED blacklisted packages neither.

" -write_page "

" -# FIXME: we don't do 2 / stats_builds_age.png yet :/ (also see above) -for i in 0 1 3 ; do - if [ "$i" = "3" ] ; then - write_usertag_table - fi - write_page " \"${MAINLABEL[$i]}\"" - # redo pngs once a day - if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime -1 -name ${TABLE[$i]}.png) ] ; then - redo_png - fi -done -write_page "

" -write_page_footer -publish_page - diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index 6390fac8..121285eb 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -82,15 +82,19 @@ - job-template: defaults: reproducible - name: '{name}_stats' + name: '{name}_html_graphs' - job-template: defaults: reproducible - name: '{name}_dd_list' + name: '{name}_html_indexes' - job-template: defaults: reproducible - name: '{name}_notes' + name: '{name}_html_dd_list' + +- job-template: + defaults: reproducible + name: '{name}_html_notes' - job-template: defaults: reproducible_builder @@ -115,15 +119,19 @@ my_description: 'Setup pbuilder for reproducible builds as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' my_timed: '42 0 * * *' my_shell: '/srv/jenkins/bin/reproducible_setup.sh' - - '{name}_stats': - my_description: 'Generates HTML results (various stats) for reproducible builds.' - my_timed: '0 0,6,12,18 * * *' - my_shell: '/srv/jenkins/bin/reproducible_stats.sh' - - '{name}_dd_list': + - '{name}_html_indexes': + my_description: 'Generates HTML results (indexes) for reproducible builds.' + my_timed: '0 3,6,9,12,15,18,21 * * *' + my_shell: '/srv/jenkins/bin/reproducible_html_indexes.sh' + - '{name}_html_graphs': + my_description: 'Generates HTML results (stats with graphs) for reproducible builds.' + my_timed: '0 0 * * *' + my_shell: '/srv/jenkins/bin/reproducible_html_graphs.sh' + - '{name}_html_dd_list': my_description: 'Generates HTML results (dd-list) for reproducible builds.' my_timed: '53 23 * * *' my_shell: '/srv/jenkins/bin/reproducible_html_dd_list.sh' - - '{name}_notes': + - '{name}_html_notes': my_description: 'Generates HTML results (notes) for reproducible builds, triggered by changes to the notes.git repository.' my_timed: '' my_shell: '/srv/jenkins/bin/reproducible_html_notes.sh' -- cgit v1.2.3-70-g09d2