diff options
author | Holger Levsen <holger@layer-acht.org> | 2015-09-08 00:39:21 +0200 |
---|---|---|
committer | Holger Levsen <holger@layer-acht.org> | 2015-09-08 00:39:21 +0200 |
commit | c5f83c96cd46b8dc324d720c84a9c9889e78c471 (patch) | |
tree | 5ac6ed844d016d28e058b2eefaed01db49bdea23 /bin/reproducible_html_dashboards.sh | |
parent | 3f382518d0d6d3598026d319ec47aeda1ead4641 (diff) | |
download | jenkins.debian.net-c5f83c96cd46b8dc324d720c84a9c9889e78c471.tar.xz |
reproducible: s#dashboards#dashboard#g
Diffstat (limited to 'bin/reproducible_html_dashboards.sh')
-rwxr-xr-x | bin/reproducible_html_dashboards.sh | 474 |
1 files changed, 0 insertions, 474 deletions
diff --git a/bin/reproducible_html_dashboards.sh b/bin/reproducible_html_dashboards.sh deleted file mode 100755 index 45af7eda..00000000 --- a/bin/reproducible_html_dashboards.sh +++ /dev/null @@ -1,474 +0,0 @@ -#!/bin/bash - -# Copyright 2014-2015 Holger Levsen <holger@layer-acht.org> -# © 2015 Mattia Rizzolo <mattia@mapreri.org> -# released under the GPLv=2 - -DEBUG=false -. /srv/jenkins/bin/common-functions.sh -common_init "$@" - -# common code defining db access -. /srv/jenkins/bin/reproducible_common.sh - -# -# init some variables -# -# we only do stats up until yesterday... we also could do today too but not update the db yet... -DATE=$(date -d "1 day ago" '+%Y-%m-%d') -FORCE_DATE=$(date -d "2 day ago" '+%Y-%m-%d') -NOTES_GIT_PATH="/var/lib/jenkins/jobs/reproducible_html_notes/workspace" - -# variables related to the stats we update -FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested" -FIELDS[1]="datum" -for i in reproducible unreproducible FTBFS other ; do - for j in $SUITES ; do - FIELDS[1]="${FIELDS[1]}, ${i}_${j}" - done -done -FIELDS[2]="datum, oldest" -FIELDS[3]="datum " -for TAG in $USERTAGS ; do - FIELDS[3]="${FIELDS[3]}, open_$TAG, done_$TAG" -done -FIELDS[4]="datum, packages_with_notes" -FIELDS[5]="datum, known_issues" -FIELDS[7]="datum, done_bugs, open_bugs" -SUM_DONE="(0" -SUM_OPEN="(0" -for TAG in $USERTAGS ; do - SUM_DONE="$SUM_DONE+done_$TAG" - SUM_OPEN="$SUM_OPEN+open_$TAG" -done -SUM_DONE="$SUM_DONE)" -SUM_OPEN="$SUM_OPEN)" -COLOR[0]=5 -COLOR[1]=12 -COLOR[2]=1 -COLOR[3]=32 -COLOR[4]=1 -COLOR[5]=1 -COLOR[7]=2 -MAINLABEL[1]="Amount of packages built each day" -MAINLABEL[3]="Usertags on bugs for user reproducible-builds@lists.alioth.debian.org" -MAINLABEL[4]="Packages which have notes" -MAINLABEL[5]="Identified issues" -MAINLABEL[7]="Open and closed bugs" -YLABEL[0]="Amount (total)" -YLABEL[1]="Amount (per day)" -YLABEL[2]="Age in days" -YLABEL[3]="Amount of bugs" -YLABEL[4]="Amount of packages" -YLABEL[5]="Amount of issues" -YLABEL[7]="Amount of bugs open / closed" - -# -# update package + build stats -# -update_suite_arch_stats() { - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,suite from ${TABLE[0]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND architecture = \"$ARCH\"") - if [ -z $RESULT ] ; then - echo "Updating packages and builds stats for $SUITE/$ARCH in $DATE." - ALL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(name) FROM sources WHERE suite='${SUITE}' AND architecture='$ARCH'") - GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE';") - GOOAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'reproducible' AND date(r.build_date)='$DATE';") - BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE';") - BAAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible' AND date(r.build_date)='$DATE';") - UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE';") - UGLDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND date(r.build_date)='$DATE';") - REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND s.architecture='$ARCH' AND date(r.build_date)<='$DATE';") - RESDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND s.architecture='$ARCH' AND date(r.build_date)='$DATE';") - OLDESTG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.status = 'reproducible' AND s.suite='$SUITE' AND s.architecture='$ARCH' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;") - OLDESTB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;") - OLDESTU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;") - DIFFG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTG');") - if [ -z $DIFFG ] ; then DIFFG=0 ; fi - DIFFB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTB');") - if [ -z $DIFFB ] ; then DIFFB=0 ; fi - DIFFU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTU');") - if [ -z $DIFFU ] ; then DIFFU=0 ; fi - let "TOTAL=GOOD+BAD+UGLY+REST" || true # let FOO=0+0 returns error in bash... - if [ "$ALL" != "$TOTAL" ] ; then - let "UNTESTED=ALL-TOTAL" - else - UNTESTED=0 - fi - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[0]} VALUES (\"$DATE\", \"$SUITE\", \"$ARCH\", $UNTESTED, $GOOD, $BAD, $UGLY, $REST)" - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", \"$ARCH\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)" - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$ARCH\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")" - # we do 3 later and 6 is special anyway... - for i in 0 1 2 4 5 ; do - PREFIX="" - if [ $i -eq 0 ] ; then - PREFIX=$SUITE - fi - # force regeneration of the image if it exists - if [ -f $BASE/$PREFIX/${TABLE[$i]}.png ] ; then - echo "Touching $PREFIX/${TABLE[$i]}.png..." - touch -d "$FORCE_DATE 00:00" $BASE/$PREFIX/${TABLE[$i]}.png - fi - done - fi -} - -# -# update notes stats -# -update_notes_stats() { - NOTES=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(package_id) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite=\"unstable\"") - ISSUES=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(name) FROM issues") - # the following is a hack to workaround the bad sql db design which is the issue_s_ column in the notes table... - # it assumes we don't have packages with more than 7 issues. (we have one with 6...) - COUNT_ISSUES=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT \ - (SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite=\"unstable\" AND n.issues = \"[]\") \ - + \ - (SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite=\"unstable\" AND n.issues != \"[]\" AND n.issues NOT LIKE \"%,%\") \ - + \ - (SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite=\"unstable\" AND n.issues LIKE \"%,%\") \ - + \ - (SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite=\"unstable\" AND n.issues LIKE \"%,%,%\") \ - + \ - (SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite=\"unstable\" AND n.issues LIKE \"%,%,%,%\") \ - + \ - (SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite=\"unstable\" AND n.issues LIKE \"%,%,%,%,%\") \ - + \ - (SELECT COUNT(issues) FROM notes AS n JOIN sources AS s ON n.package_id=s.id WHERE s.suite=\"unstable\" AND n.issues LIKE \"%,%,%,%,%,%\") \ - ") - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum from ${TABLE[4]} WHERE datum = \"$DATE\"") - if [ -z $RESULT ] ; then - echo "Updating notes stats for $DATE." - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[4]} VALUES (\"$DATE\", \"$NOTES\")" - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[5]} VALUES (\"$DATE\", \"$ISSUES\")" - fi -} - -# -# gather suite/arch stats -# -gather_suite_arch_stats() { - AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(*) FROM sources WHERE suite=\"${SUITE}\" AND architecture=\"$ARCH\"") - COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite=\"${SUITE}\" AND s.architecture=\"$ARCH\"") - COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite=\"${SUITE}\" AND s.architecture=\"$ARCH\" AND r.status=\"reproducible\"") - COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture=\"$ARCH\" AND r.status = \"unreproducible\"") - COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture=\"$ARCH\" AND r.status = \"FTBFS\"") - COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture=\"$ARCH\" AND r.status = \"404\"") - COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture=\"$ARCH\" AND r.status = \"not for us\"") - COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture=\"$ARCH\" AND r.status = \"blacklisted\"") - COUNT_DEPWAIT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture=\"$ARCH\" AND r.status = \"depwait\"") - COUNT_OTHER=$(( $COUNT_SOURCELESS+$COUNT_NOTFORUS+$COUNT_BLACKLISTED+$COUNT_DEPWAIT )) - PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc) - PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc || echo 0) - PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc || echo 0) - PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc || echo 0) - PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc || echo 0) - PERCENT_DEPWAIT=$(echo "scale=1 ; ($COUNT_DEPWAIT*100/$COUNT_TOTAL)" | bc || echo 0) - PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc || echo 0) - PERCENT_OTHER=$(echo "scale=1 ; ($COUNT_OTHER*100/$COUNT_TOTAL)" | bc || echo 0) -} - -# -# update bug stats -# -update_bug_stats() { - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") - if [ -z $RESULT ] ; then - echo "Updating bug stats for $DATE." - declare -a DONE - declare -a OPEN - GOT_BTS_RESULTS=false - SQL="INSERT INTO ${TABLE[3]} VALUES (\"$DATE\" " - for TAG in $USERTAGS ; do - OPEN[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:open status:forwarded 2>/dev/null|wc -l) - DONE[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:done archive:both 2>/dev/null|wc -l) - # test if both values are integers - if ! ( [[ ${DONE[$TAG]} =~ ^-?[0-9]+$ ]] && [[ ${OPEN[$TAG]} =~ ^-?[0-9]+$ ]] ) ; then - echo "Non-integers value detected, exiting." - echo "Usertag: $TAG" - echo "Open: ${OPEN[$TAG]}" - echo "Done: ${DONE[$TAG]}" - exit 1 - elif [ ! "${DONE[$TAG]}" = "0" ] || [ ! "${OPEN[$TAG]}" = "0" ] ; then - GOT_BTS_RESULTS=true - fi - SQL="$SQL, ${OPEN[$TAG]}, ${DONE[$TAG]}" - done - SQL="$SQL)" - echo $SQL - if $GOT_BTS_RESULTS ; then - echo "Updating ${PACKAGES_DB} with bug stats for $DATE." - sqlite3 -init ${INIT} ${PACKAGES_DB} "$SQL" - # force regeneration of the image - echo "Touching ${TABLE[3]}.png..." - touch -d "$FORCE_DATE 00:00" $BASE/${TABLE[3]}.png - echo "Touching ${TABLE[7]}.png..." - touch -d "$FORCE_DATE 00:00" $BASE/${TABLE[7]}.png - fi - fi -} - -# -# gather bugs stats and generate html table -# -write_usertag_table() { - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") - if [ ! -z "$RESULT" ] ; then - COUNT=0 - TOPEN=0 ; TDONE=0 ; TTOTAL=0 - for FIELD in $(echo ${FIELDS[3]} | tr -d ,) ; do - let "COUNT+=1" - VALUE=$(echo $RESULT | cut -d "|" -f$COUNT) - if [ $COUNT -eq 1 ] ; then - write_page "<table class=\"main\" id=\"usertagged-bugs\"><tr><th>Usertagged bugs</th><th>Open</th><th>Done</th><th>Total</th></tr>" - elif [ $((COUNT%2)) -eq 0 ] ; then - write_page "<tr><td><a href=\"https://bugs.debian.org/cgi-bin/pkgreport.cgi?tag=${FIELD:5};users=reproducible-builds@lists.alioth.debian.org&archive=both\">${FIELD:5}</a></td><td>$VALUE</td>" - TOTAL=$VALUE - let "TOPEN=TOPEN+VALUE" || TOPEN=0 - else - write_page "<td>$VALUE</td>" - let "TOTAL=TOTAL+VALUE" || true # let FOO=0+0 returns error in bash... - let "TDONE=TDONE+VALUE" - write_page "<td>$TOTAL</td></tr>" - let "TTOTAL=TTOTAL+TOTAL" - fi - done - write_page "<tr><td>Total number of <a href=\"https://wiki.debian.org/ReproducibleBuilds/Contribute#How_to_report_bugs\">usertags related to reproducible builds</a> on $DATE<br />(this is not the number of bugs as bugs can have several tags)</td><td>$TOPEN</td><td>$TDONE</td><td>$TTOTAL</td></tr>" - write_page "</table>" - fi -} - -# -# write build performace stats -# -write_build_performace_stats() { - if [ "$ARCH" != "armhf" ] ; then - TIMESPAN_VERBOSE="4 weeks" - TIMESPAN_RAW="28" - else - TIMESPAN_VERBOSE="week" - TIMESPAN_RAW="7" - fi - write_page "<table class=\"main\"><tr><th colspan=\"2\">Build statistics for $ARCH</th></tr>" - AGE_UNSTABLE=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='unstable' AND architecture='$ARCH' AND datum='$DATE'") - if [ "$ARCH" != "armhf" ] ; then - AGE_TESTING=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='testing' AND architecture='$ARCH' AND datum='$DATE'") - AGE_EXPERIMENTAL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='experimental' AND architecture='$ARCH' AND datum='$DATE'") - write_page "<tr><td>oldest $ARCH build result in testing / unstable / experimental</td><td>$AGE_TESTING / $AGE_UNSTABLE / $AGE_EXPERIMENTAL days</td></tr>" - else - write_page "<tr><td>oldest $ARCH build result in unstable </td><td>$AGE_UNSTABLE days</td></tr>" - fi - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(AVG(r.build_duration) AS INTEGER) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.build_duration!='' AND r.build_duration!='0' AND r.build_date LIKE '%$DATE%' AND s.architecture='$ARCH'") - MIN=$(echo $RESULT/60|bc) - SEC=$(echo "$RESULT-($MIN*60)"|bc) - write_page "<tr><td>average test duration (on $DATE)</td><td>$MIN minutes, $SEC seconds</td></tr>" - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(AVG(r.build_duration) AS INTEGER) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.build_duration!='' AND r.build_duration!='0' AND r.build_date > datetime('$DATE', '-$TIMESPAN_RAW days') AND s.architecture='$ARCH'") - MIN=$(echo $RESULT/60|bc) - SEC=$(echo "$RESULT-($MIN*60)"|bc) - write_page "<tr><td>average test duration (in the last $TIMESPAN_VERBOSE)</td><td>$MIN minutes, $SEC seconds</td></tr>" - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(r.build_date) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.build_date LIKE '%$DATE%' AND s.architecture='$ARCH'") - write_page "<tr><td>packages tested on $DATE</td><td>$RESULT</td></tr>" - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(r.build_date) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.build_date > datetime('$DATE', '-$TIMESPAN_RAW days') AND s.architecture='$ARCH'") - RESULT="$(echo $RESULT/$TIMESPAN_RAW|bc)" - write_page "<tr><td>packages tested on average per day in the last $TIMESPAN_VERBOSE</td><td>$RESULT</td></tr>" - write_page "</table>" -} - -# -# create suite stats page -# -create_suite_arch_stats_page() { - VIEW=suite_${ARCH}_stats - PAGE=index_${VIEW}.html - MAINLABEL[0]="Reproducibility status for packages in '$SUITE' for '$ARCH'" - MAINLABEL[2]="Age in days of oldest reproducible build result in '$SUITE' for '$ARCH'" - echo "$(date) - starting to write $PAGE page." - write_page_header $VIEW "Overview of reproducible builds for packages in $SUITE for $ARCH" - if [ $(echo $PERCENT_TOTAL/1|bc) -lt 98 ] ; then - write_page "<p>$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE/$ARCH.</p>" - fi - write_page "<p>" - set_icon reproducible - write_icon - write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly in $SUITE/$ARCH." - set_icon unreproducible - write_icon - write_page "$COUNT_BAD packages ($PERCENT_BAD%) failed to build reproducibly." - set_icon FTBFS - write_icon - write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source.</p>" - write_page "<p>" - if [ $COUNT_DEPWAIT -gt 0 ] ; then - write_page "For " - set_icon depwait - write_icon - write_page "$COUNT_DEPWAIT ($PERCENT_DEPWAIT%) source packages the build-depends cannot be satisfied." - fi - if [ $COUNT_SOURCELESS -gt 0 ] ; then - write_page "For " - set_icon 404 - write_icon - write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) source packages could not be downloaded," - fi - set_icon not_for_us - write_icon - if [ "$ARCH" = "armhf" ] ; then - ARMSPECIALARCH=" 'any-arm'," - fi - write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', '$ARCH', 'linux-any', 'linux-$ARCH'$ARMSPECIALARCH nor 'any-$ARCH' will not be build here" - write_page "and those " - set_icon blacklisted - write_icon - write_page "$COUNT_BLACKLISTED blacklisted packages neither.</p>" - write_page "<p>" - write_page " <a href=\"/$SUITE/$ARCH/${TABLE[0]}.png\"><img src=\"/$SUITE/$ARCH/${TABLE[0]}.png\" alt=\"${MAINLABEL[0]}\"></a>" - for i in 0 2 ; do - # recreate png once a day - if [ ! -f $BASE/$SUITE/$ARCH/${TABLE[$i]}.png ] || [ ! -z $(find $BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then - create_png_from_table $i $SUITE/$ARCH/${TABLE[$i]}.png - fi - done - write_page "</p>" - write_page_footer - publish_page $SUITE -} - -# -# create main stats page -# -create_main_stats_page() { - VIEW=stats - PAGE=index_${VIEW}.html - echo "$(date) - starting to write $PAGE page." - write_page_header $VIEW "Overview of various statistics about reproducible builds" - # write suite table - write_page "<p>" - write_page "<table class=\"main\"><tr><th>suite</th><th>all sources packages</th><th>reproducible packages</th><th>unreproducible packages</th><th>packages failing to build</th><th>other packages</th></tr>" - for SUITE in $SUITES ; do - gather_suite_arch_stats - write_page "<tr><td>$SUITE</td><td>$AMOUNT" - if [ $(echo $PERCENT_TOTAL/1|bc) -lt 98 ] ; then - write_page "<span style=\"font-size:0.8em;\">($PERCENT_TOTAL% tested)</span>" - fi - write_page "</td><td>$COUNT_GOOD / $PERCENT_GOOD%</td><td>$COUNT_BAD / $PERCENT_BAD%</td><td>$COUNT_UGLY / $PERCENT_UGLY%</td><td>$COUNT_OTHER / $PERCENT_OTHER%</td></tr>" - done - write_page "</table>" - # write suite graphs - write_page "</p><p style=\"clear:both;\">" - for SUITE in $SUITES ; do - write_page " <a href=\"/$SUITE\"><img src=\"/$SUITE/$ARCH/${TABLE[0]}.png\" class=\"overview\" alt=\"$SUITE/$ARCH stats\"></a>" - done - write_page "</p><p><center>" - # write meta pkg graphs per suite - for SUITE in $SUITES ; do - if [ "$SUITE" != "unstable" ] ; then - # only show pkg sets from unstable - continue - fi - for i in $(seq 1 ${#META_PKGSET[@]}) ; do - THUMB=${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png - LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" - write_page "<a href=\"/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"><img src=\"/$SUITE/$ARCH/$THUMB\" class=\"metaoverview\" alt=\"$LABEL\"></a>" - done - done - write_page "</center></p><p>" - # write inventory table - write_page "<table class=\"main\"><tr><th colspan=\"2\">Various reproducibility statistics</th></tr>" - write_page "<tr><td>identified <a href=\"/index_issues.html\">distinct and categorized issues</a></td><td>$ISSUES</td></tr>" - write_page "<tr><td>total number of identified issues in packages</td><td>$COUNT_ISSUES</td></tr>" - write_page "<tr><td>packages with notes about these issues</td><td>$NOTES</td></tr>" - SUITE="unstable" - gather_suite_arch_stats - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('unreproducible', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") - write_page "<tr><td>packages in $SUITE with issues but <a href=\"/$SUITE/$ARCH/index_no_notes.html\">without identified ones</a></td><td>$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td></tr>" - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='unreproducible' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") - write_page "<tr><td> - unreproducible ones</a></td><td>$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td></tr>" - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") - write_page "<tr><td> - failing to build</a></td><td>$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%</td></tr>" - write_page "<tr><td>packages in $SUITE which need to be fixed</td><td>$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%</td></tr>" - SUITE="testing" - gather_suite_arch_stats - write_page "<tr><td> - in $SUITE</td><td>$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%</td></tr>" - SUITE="unstable" - if [ -f ${NOTES_GIT_PATH}/packages.yml ] && [ -f ${NOTES_GIT_PATH}/issues.yml ] ; then - write_page "<tr><td>committers to <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\" target=\"_parent\">notes.git</a> (in the last three months)</td><td>$(cd ${NOTES_GIT_PATH} ; git log --since="3 months ago"|grep Author|sort -u |wc -l)</td></tr>" - write_page "<tr><td>committers to notes.git (in total)</td><td>$(cd ${NOTES_GIT_PATH} ; git log |grep Author|sort -u |wc -l)</td></tr>" - fi - RESULT=$(cat /srv/reproducible-results/modified_in_sid.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh - write_page "<tr><td>packages <a href=\"/index_repositories.html\">modified in our toolchain</a> (in unstable)</td><td>$(echo $RESULT)</td></tr>" - RESULT=$(cat /srv/reproducible-results/modified_in_exp.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh - write_page "<tr><td> - (in experimental)</td><td>$(echo $RESULT)</td></tr>" - RESULT=$(cat /srv/reproducible-results/binnmus_needed.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh - if [ "$RESULT" != "0" ] ; then - write_page "<tr><td> - which need to be build on some archs</td><td>$(echo $RESULT)</td></tr>" - fi - write_page "</table>" - # write bugs with usertags table - write_usertag_table - write_page "</p><p style=\"clear:both;\">" - # do other global graphs - for i in 3 7 4 5 ; do - write_page " <a href=\"/${TABLE[$i]}.png\"><img src=\"/${TABLE[$i]}.png\" class="halfview" alt=\"${MAINLABEL[$i]}\"></a>" - # redo pngs once a day - if [ ! -f $BASE/${TABLE[$i]}.png ] || [ ! -z $(find $BASE -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then - create_png_from_table $i ${TABLE[$i]}.png - fi - done - write_page "</p>" - # explain setup - write_explaination_table debian - # write build per day graph - write_page "<p style=\"clear:both;\">" - write_page " <a href=\"/${TABLE[1]}_$ARCH.png\"><img src=\"/${TABLE[1]}_$ARCH.png\" alt=\"${MAINLABEL[$i]}\"></a>" - # redo png once a day - for ARCH in ${ARCHS} ; do - if [ ! -f $BASE/${TABLE[1]}_$ARCH.png ] || [ ! -z $(find $BASE -maxdepth 1 -mtime +0 -name ${TABLE[1]}_$ARCH.png) ] ; then - create_png_from_table 1 ${TABLE[1]}_$ARCH.png - fi - done - ARCH="amd64" - # write suite builds age graphs - write_page "</p><p style=\"clear:both;\">" - for SUITE in $SUITES ; do - write_page " <a href=\"/$SUITE/\"><img src=\"/$SUITE/$ARCH/${TABLE[2]}.png\" class=\"overview\" alt=\"age of oldest reproducible build result in $SUITE/$ARCH\"></a>" - done - write_build_performace_stats - # other archs: armhf - ARCH="armhf" - SUITE="unstable" - write_page "</p><p style=\"clear:both;\">" - write_page " <hr />" - write_page " <a href=\"/$SUITE/index_suite_${ARCH}_stats.html\"><img src=\"/$SUITE/$ARCH/${TABLE[0]}.png\" class=\"overview\" alt=\"$SUITE/$ARCH stats\"></a>" - write_page " <a href=\"/${TABLE[1]}_$ARCH.png\"><img src=\"/${TABLE[1]}_$ARCH.png\" class=\"overview\" alt=\"${MAINLABEL[$i]}\"></a>" - write_page " <a href=\"/$SUITE/$ARCH/${TABLE[2]}.png\"><img src=\"/$SUITE/$ARCH/${TABLE[2]}.png\" class=\"overview\" alt=\"age of oldest reproducible build result in $SUITE/$ARCH\"></a>" - write_build_performace_stats - write_page "</p><p style=\"clear:both;\">" - write_page " <hr />" - # link to index_breakages - write_page "<br />There are <a href=\"$BASEURL/index_breakages.html\">some problems in this setup</a> too. And there is <a href=\"https://jenkins.debian.net/userContent/about.html#_reproducible_builds_jobs\">documentation</a> too, in case you missed the link at the top. More feedback is always welcome!</p>" - # the end - write_page_footer - cp $PAGE $BASE/reproducible.html - publish_page -} - -# -# main -# -SUITE="unstable" -update_bug_stats -update_notes_stats -for ARCH in ${ARCHS} ; do - for SUITE in $SUITES ; do - if [ "$SUITE" != "unstable" ] && [ "$ARCH" = "armhf" ] ; then - # we only test unstable on armhf atm - continue - fi - update_suite_arch_stats - gather_suite_arch_stats - create_suite_arch_stats_page - done -done -ARCH="amd64" -SUITE="unstable" -create_main_stats_page - |