From 23b8654156a569eaa3516b38d3ed372a72010785 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Sat, 18 Jul 2015 18:40:54 +0200 Subject: reproducible: split _html_graphs.sh into _dashboards.sh and _pkg_sets.sh --- bin/reproducible_common.sh | 80 +++++ bin/reproducible_html_dashboards.sh | 430 +++++++++++++++++++++++ bin/reproducible_html_graphs.sh | 673 ------------------------------------ bin/reproducible_html_pkg_sets.sh | 196 +++++++++++ 4 files changed, 706 insertions(+), 673 deletions(-) create mode 100755 bin/reproducible_html_dashboards.sh delete mode 100755 bin/reproducible_html_graphs.sh create mode 100755 bin/reproducible_html_pkg_sets.sh (limited to 'bin') diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 7d73afdb..361b231f 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -425,3 +425,83 @@ get_filesize() { SIZE="$BYTESIZE bytes" fi } + +# +# create the png (and query the db to populate a csv file...) +# +create_png_from_table() { + echo "Checking whether to update $2..." + # $1 = id of the stats table + # $2 = image file name + # $3 = meta package set, only sensible if $1=6 + echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv + # prepare query + WHERE_EXTRA="WHERE suite = '$SUITE'" + if [ $1 -eq 3 ] || [ $1 -eq 4 ] || [ $1 -eq 5 ] ; then + # TABLE[3+4+5] don't have a suite column: + WHERE_EXTRA="" + elif [ $1 -eq 6 ] ; then + # 6 is special too: + WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$3'" + fi + # run query + if [ $1 -eq 1 ] ; then + # not sure if it's worth to generate the following query... + sqlite3 -init ${INIT} --nullvalue 0 -csv ${PACKAGES_DB} "SELECT s.datum, + COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='testing'),0) as 'reproducible_testing', + COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='unstable'),0) as 'reproducible_unstable', + COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='experimental'),0) as 'reproducible_experimental', + (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS unreproducible_testing, + (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS unreproducible_unstable, + (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS unreproducible_experimental, + (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS FTBFS_testing, + (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS FTBFS_unstable, + (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS FTBFS_experimental, + (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS other_testing, + (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS other_unstable, + (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS other_experimental + FROM stats_builds_per_day AS s GROUP BY s.datum" >> ${TABLE[$1]}.csv + elif [ $1 -eq 2 ] ; then + # just make a graph of the oldest reproducible build (ignore FTBFS and unreproducible) + sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv + elif [ $1 -eq 7 ] ; then + sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, $SUM_DONE, $SUM_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv + else + sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv + fi + # this is a gross hack: normally we take the number of colors a table should have... + # for the builds_age table we only want one color, but different ones, so this hack: + COLORS=${COLOR[$1]} + if [ $1 -eq 2 ] ; then + case "$SUITE" in + testing) COLORS=40 ;; + unstable) COLORS=41 ;; + experimental) COLORS=42 ;; + esac + fi + # only generate graph if the query returned data + if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then + echo "Updating $2..." + DIR=$(dirname $2) + mkdir -p $DIR + echo "Generating $2." + /srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" + mv $2 $BASE/$DIR + [ "$DIR" = "." ] || rmdir $(dirname $2) + # create empty dummy png if there havent been any results ever + elif [ ! -f $BASE/$DIR/$(basename $2) ] ; then + DIR=$(dirname $2) + mkdir -p $DIR + echo "Creating $2 dummy." + convert -size 1920x960 xc:#aaaaaa -depth 8 $2 + if [ "$3" != "" ] ; then + local THUMB="${TABLE[1]}_${3}-thumbnail.png" + convert $2 -adaptive-resize 160x80 ${THUMB} + mv ${THUMB} $BASE/$DIR + fi + mv $2 $BASE/$DIR + [ "$DIR" = "." ] || rmdir $(dirname $2) + fi + rm ${TABLE[$1]}.csv +} + diff --git a/bin/reproducible_html_dashboards.sh b/bin/reproducible_html_dashboards.sh new file mode 100755 index 00000000..31ba1fcc --- /dev/null +++ b/bin/reproducible_html_dashboards.sh @@ -0,0 +1,430 @@ +#!/bin/bash + +# Copyright 2014-2015 Holger Levsen +# © 2015 Mattia Rizzolo +# released under the GPLv=2 + +DEBUG=false +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code defining db access +. /srv/jenkins/bin/reproducible_common.sh + +# +# init some variables +# +ARCH="amd64" # we only care about amd64 status here (for now) +# we only do stats up until yesterday... we also could do today too but not update the db yet... +DATE=$(date -d "1 day ago" '+%Y-%m-%d') +FORCE_DATE=$(date -d "2 day ago" '+%Y-%m-%d') +NOTES_GIT_PATH="/var/lib/jenkins/jobs/reproducible_html_notes/workspace" + +# variables related to the stats we do +TABLE[0]=stats_pkg_state +TABLE[1]=stats_builds_per_day +TABLE[2]=stats_builds_age +TABLE[3]=stats_bugs +TABLE[4]=stats_notes +TABLE[5]=stats_issues +#TABLE[6] +TABLE[7]=stats_bugs_state +FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested" +FIELDS[1]="datum" +for i in reproducible unreproducible FTBFS other ; do + for j in $SUITES ; do + FIELDS[1]="${FIELDS[1]}, ${i}_${j}" + done +done +FIELDS[2]="datum, oldest" +FIELDS[3]="datum " +for TAG in $USERTAGS ; do + FIELDS[3]="${FIELDS[3]}, open_$TAG, done_$TAG" +done +FIELDS[4]="datum, packages_with_notes" +FIELDS[5]="datum, known_issues" +#FIELDS[6] +FIELDS[7]="datum, done_bugs, open_bugs" +SUM_DONE="(0" +SUM_OPEN="(0" +for TAG in $USERTAGS ; do + SUM_DONE="$SUM_DONE+done_$TAG" + SUM_OPEN="$SUM_OPEN+open_$TAG" +done +SUM_DONE="$SUM_DONE)" +SUM_OPEN="$SUM_OPEN)" +COLOR[0]=5 +COLOR[1]=12 +COLOR[2]=1 +COLOR[3]=32 +COLOR[4]=1 +COLOR[5]=1 +#COLOR[6] +COLOR[7]=2 +MAINLABEL[1]="Amount of packages built each day" +MAINLABEL[3]="Usertags on bugs for user reproducible-builds@lists.alioth.debian.org" +MAINLABEL[4]="Packages which have notes" +MAINLABEL[5]="Identified issues" +MAINLABEL[7]="Open and closed bugs" +YLABEL[0]="Amount (total)" +YLABEL[1]="Amount (per day)" +YLABEL[2]="Age in days" +YLABEL[3]="Amount of bugs" +YLABEL[4]="Amount of packages" +YLABEL[5]="Amount of issues" +YLABEL[7]="Amount of bugs open / closed" + +# +# update package + build stats +# +update_suite_stats() { + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,suite from ${TABLE[0]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\"") + if [ -z $RESULT ] ; then + echo "Updating packages and builds stats for $SUITE on $DATE." + ALL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(name) FROM sources WHERE suite='${SUITE}'") + GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE';") + GOOAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)='$DATE';") + BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE';") + BAAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)='$DATE';") + UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE';") + UGLDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)='$DATE';") + REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND date(r.build_date)<='$DATE';") + RESDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND date(r.build_date)='$DATE';") + OLDESTG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.status = 'reproducible' AND s.suite='$SUITE' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;") + OLDESTB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;") + OLDESTU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;") + DIFFG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTG');") + if [ -z $DIFFG ] ; then DIFFG=0 ; fi + DIFFB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTB');") + if [ -z $DIFFB ] ; then DIFFB=0 ; fi + DIFFU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTU');") + if [ -z $DIFFU ] ; then DIFFU=0 ; fi + let "TOTAL=GOOD+BAD+UGLY+REST" || true # let FOO=0+0 returns error in bash... + if [ "$ALL" != "$TOTAL" ] ; then + let "UNTESTED=ALL-TOTAL" + else + UNTESTED=0 + fi + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[0]} VALUES (\"$DATE\", \"$SUITE\", $UNTESTED, $GOOD, $BAD, $UGLY, $REST)" + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)" + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")" + # we do 3 later and 6 is special anyway... + for i in 0 1 2 4 5 ; do + PREFIX="" + if [ $i -eq 0 ] ; then + PREFIX=$SUITE + fi + # force regeneration of the image if it exists + if [ -f $BASE/$PREFIX/${TABLE[$i]}.png ] ; then + echo "Touching $PREFIX/${TABLE[$i]}.png..." + touch -d "$FORCE_DATE 00:00" $BASE/$PREFIX/${TABLE[$i]}.png + fi + done + fi +} + +# +# update notes stats +# +update_notes_stats() { + if [ ! -d ${NOTES_GIT_PATH} ] ; then + echo "Warning: ${NOTES_GIT_PATH} does not exist, has the job been renamed???" + echo "Please investigate and fix!" + exit 1 + elif [ ! -f ${NOTES_GIT_PATH}/packages.yml ] || [ ! -f ${NOTES_GIT_PATH}/issues.yml ] ; then + # retry. sometimes these files vanish for a moment, probably when jenkins automatically updates the clones or such. + sleep 5 + if [ ! -f ${NOTES_GIT_PATH}/packages.yml ] || [ ! -f ${NOTES_GIT_PATH}/issues.yml ] ; then + echo "Warning: ${NOTES_GIT_PATH}/packages.yml or issues.yml does not exist, something has changed in notes.git it seems." + echo "Please investigate and fix!" + exit 1 + fi + fi + NOTES=$(grep -c -v "^ " ${NOTES_GIT_PATH}/packages.yml) + ISSUES=$(grep -c -v "^ " ${NOTES_GIT_PATH}/issues.yml) + COUNT_ISSUES=$(grep " -" ${NOTES_GIT_PATH}/packages.yml | egrep -v " - [0-9]+"|wc -l) + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum from ${TABLE[4]} WHERE datum = \"$DATE\"") + if [ -z $RESULT ] ; then + echo "Updating notes stats for $DATE." + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[4]} VALUES (\"$DATE\", \"$NOTES\")" + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[5]} VALUES (\"$DATE\", \"$ISSUES\")" + fi +} + +# +# gather suite stats +# +gather_suite_stats() { + AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(*) FROM sources WHERE suite=\"${SUITE}\"") + COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite=\"${SUITE}\"") + COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite=\"${SUITE}\" AND r.status=\"reproducible\"") + COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"unreproducible\"") + COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"FTBFS\"") + COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"404\"") + COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"not for us\"") + COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"blacklisted\"") + COUNT_OTHER=$(( $COUNT_SOURCELESS+$COUNT_NOTFORUS+$COUNT_BLACKLISTED )) + PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc) + PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc) + PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) + PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) + PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) + PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) + PERCENT_OTHER=$(echo "scale=1 ; ($COUNT_OTHER*100/$COUNT_TOTAL)" | bc) +} + +# +# update bug stats +# +update_bug_stats() { + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") + if [ -z $RESULT ] ; then + echo "Updating bug stats for $DATE." + declare -a DONE + declare -a OPEN + GOT_BTS_RESULTS=false + SQL="INSERT INTO ${TABLE[3]} VALUES (\"$DATE\" " + for TAG in $USERTAGS ; do + OPEN[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:open status:forwarded 2>/dev/null|wc -l) + DONE[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:done archive:both 2>/dev/null|wc -l) + # test if both values are integers + if ! ( [[ ${DONE[$TAG]} =~ ^-?[0-9]+$ ]] && [[ ${OPEN[$TAG]} =~ ^-?[0-9]+$ ]] ) ; then + echo "Non-integers value detected, exiting." + echo "Usertag: $TAG" + echo "Open: ${OPEN[$TAG]}" + echo "Done: ${DONE[$TAG]}" + exit 1 + elif [ ! "${DONE[$TAG]}" = "0" ] || [ ! "${OPEN[$TAG]}" = "0" ] ; then + GOT_BTS_RESULTS=true + fi + SQL="$SQL, ${OPEN[$TAG]}, ${DONE[$TAG]}" + done + SQL="$SQL)" + echo $SQL + if $GOT_BTS_RESULTS ; then + echo "Updating ${PACKAGES_DB} with bug stats for $DATE." + sqlite3 -init ${INIT} ${PACKAGES_DB} "$SQL" + # force regeneration of the image + echo "Touching ${TABLE[3]}.png..." + touch -d "$FORCE_DATE 00:00" $BASE/${TABLE[3]}.png + echo "Touching ${TABLE[7]}.png..." + touch -d "$FORCE_DATE 00:00" $BASE/${TABLE[7]}.png + fi + fi +} + +# +# gather bugs stats and generate html table +# +write_usertag_table() { + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") + if [ ! -z "$RESULT" ] ; then + COUNT=0 + TOPEN=0 ; TDONE=0 ; TTOTAL=0 + for FIELD in $(echo ${FIELDS[3]} | tr -d ,) ; do + let "COUNT+=1" + VALUE=$(echo $RESULT | cut -d "|" -f$COUNT) + if [ $COUNT -eq 1 ] ; then + write_page "" + elif [ $((COUNT%2)) -eq 0 ] ; then + write_page "" + TOTAL=$VALUE + let "TOPEN=TOPEN+VALUE" || TOPEN=0 + else + write_page "" + let "TOTAL=TOTAL+VALUE" || true # let FOO=0+0 returns error in bash... + let "TDONE=TDONE+VALUE" + write_page "" + let "TTOTAL=TTOTAL+TOTAL" + fi + done + write_page "" + write_page "
Usertagged bugsOpenDoneTotal
${FIELD:5}$VALUE$VALUE$TOTAL
Total number of usertags on $DATE
(this is not the number of bugs as bugs can have several tags)		$TOPEN$TDONE$TTOTAL
" + fi +} + +# +# create suite stats page +# +create_suite_stats_page() { + VIEW=suite_stats + PAGE=index_${VIEW}.html + MAINLABEL[0]="Reproducibility status for packages in '$SUITE'" + MAINLABEL[2]="Age in days of oldest reproducible build result in '$SUITE'" + echo "$(date) - starting to write $PAGE page." + write_page_header $VIEW "Overview of reproducible builds for packages in $SUITE" + if [ $(echo $PERCENT_TOTAL/1|bc) -lt 98 ] ; then + write_page "

$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE.

" + fi + write_page "

" + set_icon reproducible + write_icon + write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly in $SUITE/$ARCH." + set_icon unreproducible + write_icon + write_page "$COUNT_BAD packages ($PERCENT_BAD%) failed to build reproducibly." + set_icon FTBFS + write_icon + write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source.

" + write_page "

" + if [ $COUNT_SOURCELESS -gt 0 ] ; then + write_page "For " + set_icon 404 + write_icon + write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages sources could not be downloaded," + fi + set_icon not_for_us + write_icon + write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64' will not be build here" + write_page "and those " + set_icon blacklisted + write_icon + write_page "$COUNT_BLACKLISTED blacklisted packages neither.

" + write_page "

" + write_page " \"${MAINLABEL[0]}\"" + for i in 0 2 ; do + # recreate png once a day + if [ ! -f $BASE/$SUITE/${TABLE[$i]}.png ] || [ ! -z $(find $BASE/$SUITE -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then + create_png_from_table $i $SUITE/${TABLE[$i]}.png + fi + done + write_page "

" + write_page_footer + publish_page $SUITE +} + +# +# create main stats page +# +create_main_stats_page() { + VIEW=stats + PAGE=index_${VIEW}.html + echo "$(date) - starting to write $PAGE page." + write_page_header $VIEW "Overview of various statistics about reproducible builds" + # write suite table + write_page "

" + write_page "" + for SUITE in $SUITES ; do + gather_suite_stats + write_page "" + done + write_page "
suiteall sources packagesreproducible packagesunreproducible packagespackages failing to buildother packages
$SUITE$AMOUNT" + if [ $(echo $PERCENT_TOTAL/1|bc) -lt 98 ] ; then + write_page "($PERCENT_TOTAL% tested)" + fi + write_page "$COUNT_GOOD / $PERCENT_GOOD%$COUNT_BAD / $PERCENT_BAD%$COUNT_UGLY / $PERCENT_UGLY%$COUNT_OTHER / $PERCENT_OTHER%
" + # write suite graphs + write_page "

" + for SUITE in $SUITES ; do + write_page " \"$SUITE" + done + write_page "

" + # write meta pkg graphs per suite + for SUITE in $SUITES ; do + if [ "$SUITE" != "unstable" ] ; then + # only show pkg sets from unstable + continue + fi + for i in $(seq 1 ${#META_PKGSET[@]}) ; do + THUMB=${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png + LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" + write_page "\"$LABEL\"" + done + done + write_page "

" + # write inventory table + write_page "" + write_page "" + write_page "" + write_page "" + SUITE="unstable" + gather_suite_stats + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('unreproducible', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") + write_page "" + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='unreproducible' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") + write_page "" + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") + write_page "" + write_page "" + SUITE="testing" + gather_suite_stats + write_page "" + SUITE="unstable" + if [ -f ${NOTES_GIT_PATH}/packages.yml ] && [ -f ${NOTES_GIT_PATH}/issues.yml ] ; then + write_page "" + write_page "" + fi + RESULT=$(cat /srv/reproducible-results/modified_in_sid.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh + write_page "" + RESULT=$(cat /srv/reproducible-results/modified_in_exp.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh + write_page "" + write_page "
Various reproducibility statistics
identified distinct issues$ISSUES
total number of identified issues in packages$COUNT_ISSUES
packages with notes about these issues$NOTES
packages in $SUITE with issues but without identified ones$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%
  - unreproducible ones$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%
  - failing to build$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%
packages in $SUITE which need to be fixed$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%
  - in $SUITE$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%
committers to notes.git (in the last three months)$(cd ${NOTES_GIT_PATH} ; git log --since="3 months ago"|grep Author|sort -u |wc -l)
committers to notes.git (in total)$(cd ${NOTES_GIT_PATH} ; git log |grep Author|sort -u |wc -l)
packages modified in our toolchain (in unstable)$(echo $RESULT)
  - (in experimental)$(echo $RESULT)
" + # write bugs with usertags table + write_usertag_table + write_page "

" + # do other global graphs + for i in 3 7 4 5 ; do + write_page " \"${MAINLABEL[$i]}\"" + # redo pngs once a day + if [ ! -f $BASE/${TABLE[$i]}.png ] || [ ! -z $(find $BASE -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then + create_png_from_table $i ${TABLE[$i]}.png + fi + done + write_page "

" + # explain setup + write_explaination_table debian + # write build per day graph + write_page "

" + write_page " \"${MAINLABEL[$i]}\"" + # redo png once a day + if [ ! -f $BASE/${TABLE[1]}.png ] || [ ! -z $(find $BASE -maxdepth 1 -mtime +0 -name ${TABLE[1]}.png) ] ; then + create_png_from_table 1 ${TABLE[1]}.png + fi + # write suite builds age graphs + write_page "

" + for SUITE in $SUITES ; do + write_page " \"age" + done + # write build performace stats + write_page "" + AGE_TESTING=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='testing' AND datum='$DATE'") + AGE_UNSTABLE=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='unstable' AND datum='$DATE'") + AGE_EXPERIMENTAL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='experimental' AND datum='$DATE'") + write_page "" + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(AVG(r.build_duration) AS INTEGER) FROM results AS r WHERE r.build_duration!='' AND r.build_duration!='0' AND r.build_date LIKE '%$DATE%'") + MIN=$(echo $RESULT/60|bc) + SEC=$(echo "$RESULT-($MIN*60)"|bc) + write_page "" + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(AVG(r.build_duration) AS INTEGER) FROM results AS r WHERE r.build_duration!='' AND r.build_duration!='0' AND r.build_date > datetime('$DATE', '-28 days')") + MIN=$(echo $RESULT/60|bc) + SEC=$(echo "$RESULT-($MIN*60)"|bc) + write_page "" + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(r.build_date) FROM results AS r WHERE r.build_date LIKE '%$DATE%'") + write_page "" + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(r.build_date) FROM results AS r WHERE r.build_date > datetime('$DATE', '-28 days')") + RESULT="$(echo $RESULT/28|bc)" + write_page "" + write_page "
Build statistics
oldest build result in testing / unstable / experimental$AGE_TESTING / $AGE_UNSTABLE / $AGE_EXPERIMENTAL days
average test duration (on $DATE)$MIN minutes, $SEC seconds
average test duration (in the last 4 weeks)$MIN minutes, $SEC seconds
packages tested on $DATE$RESULT
packages tested on average per day in the last 4 weeks$RESULT
" + # link to index_breakages + write_page "

" + write_page "
There are some problems in this setup too. And there is documentation too, in case you missed the link at the top. More feedback is always welcome!

" + # the end + write_page_footer + cp $PAGE $BASE/reproducible.html + publish_page +} + +# +# main +# +SUITE="unstable" +update_bug_stats +update_notes_stats +for SUITE in $SUITES ; do + update_suite_stats + gather_suite_stats + create_suite_stats_page +done +SUITE="unstable" +create_main_stats_page + diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_graphs.sh deleted file mode 100755 index 2dc80128..00000000 --- a/bin/reproducible_html_graphs.sh +++ /dev/null @@ -1,673 +0,0 @@ -#!/bin/bash - -# Copyright 2014-2015 Holger Levsen -# © 2015 Mattia Rizzolo -# released under the GPLv=2 - -DEBUG=false -. /srv/jenkins/bin/common-functions.sh -common_init "$@" - -# common code defining db access -. /srv/jenkins/bin/reproducible_common.sh - -# -# init some variables -# -ARCH="amd64" # we only care about amd64 status here (for now) -# we only do stats up until yesterday... we also could do today too but not update the db yet... -DATE=$(date -d "1 day ago" '+%Y-%m-%d') -FORCE_DATE=$(date -d "2 day ago" '+%Y-%m-%d') -NOTES_GIT_PATH="/var/lib/jenkins/jobs/reproducible_html_notes/workspace" - -# variables related to the stats we do -TABLE[0]=stats_pkg_state -TABLE[1]=stats_builds_per_day -TABLE[2]=stats_builds_age -TABLE[3]=stats_bugs -TABLE[4]=stats_notes -TABLE[5]=stats_issues -TABLE[6]=stats_meta_pkg_state -TABLE[7]=stats_bugs_state -FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested" -FIELDS[1]="datum" -for i in reproducible unreproducible FTBFS other ; do - for j in $SUITES ; do - FIELDS[1]="${FIELDS[1]}, ${i}_${j}" - done -done -FIELDS[2]="datum, oldest" -FIELDS[3]="datum " -for TAG in $USERTAGS ; do - FIELDS[3]="${FIELDS[3]}, open_$TAG, done_$TAG" -done -FIELDS[4]="datum, packages_with_notes" -FIELDS[5]="datum, known_issues" -FIELDS[6]="datum, reproducible, unreproducible, FTBFS, other" -FIELDS[7]="datum, done_bugs, open_bugs" -SUM_DONE="(0" -SUM_OPEN="(0" -for TAG in $USERTAGS ; do - SUM_DONE="$SUM_DONE+done_$TAG" - SUM_OPEN="$SUM_OPEN+open_$TAG" -done -SUM_DONE="$SUM_DONE)" -SUM_OPEN="$SUM_OPEN)" -COLOR[0]=5 -COLOR[1]=12 -COLOR[2]=1 -COLOR[3]=32 -COLOR[4]=1 -COLOR[5]=1 -COLOR[6]=4 -COLOR[7]=2 -MAINLABEL[1]="Amount of packages built each day" -MAINLABEL[3]="Usertags on bugs for user reproducible-builds@lists.alioth.debian.org" -MAINLABEL[4]="Packages which have notes" -MAINLABEL[5]="Identified issues" -MAINLABEL[7]="Open and closed bugs" -YLABEL[0]="Amount (total)" -YLABEL[1]="Amount (per day)" -YLABEL[2]="Age in days" -YLABEL[3]="Amount of bugs" -YLABEL[4]="Amount of packages" -YLABEL[5]="Amount of issues" -YLABEL[7]="Amount of bugs open / closed" - -# -# update package + build stats -# -update_suite_stats() { - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,suite from ${TABLE[0]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\"") - if [ -z $RESULT ] ; then - echo "Updating packages and builds stats for $SUITE on $DATE." - ALL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(name) FROM sources WHERE suite='${SUITE}'") - GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE';") - GOOAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)='$DATE';") - BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE';") - BAAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)='$DATE';") - UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE';") - UGLDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)='$DATE';") - REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND date(r.build_date)<='$DATE';") - RESDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND date(r.build_date)='$DATE';") - OLDESTG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.status = 'reproducible' AND s.suite='$SUITE' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;") - OLDESTB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;") - OLDESTU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;") - DIFFG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTG');") - if [ -z $DIFFG ] ; then DIFFG=0 ; fi - DIFFB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTB');") - if [ -z $DIFFB ] ; then DIFFB=0 ; fi - DIFFU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTU');") - if [ -z $DIFFU ] ; then DIFFU=0 ; fi - let "TOTAL=GOOD+BAD+UGLY+REST" || true # let FOO=0+0 returns error in bash... - if [ "$ALL" != "$TOTAL" ] ; then - let "UNTESTED=ALL-TOTAL" - else - UNTESTED=0 - fi - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[0]} VALUES (\"$DATE\", \"$SUITE\", $UNTESTED, $GOOD, $BAD, $UGLY, $REST)" - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)" - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")" - # we do 3 later and 6 is special anyway... - for i in 0 1 2 4 5 ; do - PREFIX="" - if [ $i -eq 0 ] ; then - PREFIX=$SUITE - fi - # force regeneration of the image if it exists - if [ -f $BASE/$PREFIX/${TABLE[$i]}.png ] ; then - echo "Touching $PREFIX/${TABLE[$i]}.png..." - touch -d "$FORCE_DATE 00:00" $BASE/$PREFIX/${TABLE[$i]}.png - fi - done - fi -} - -# -# update notes stats -# -update_notes_stats() { - if [ ! -d ${NOTES_GIT_PATH} ] ; then - echo "Warning: ${NOTES_GIT_PATH} does not exist, has the job been renamed???" - echo "Please investigate and fix!" - exit 1 - elif [ ! -f ${NOTES_GIT_PATH}/packages.yml ] || [ ! -f ${NOTES_GIT_PATH}/issues.yml ] ; then - # retry. sometimes these files vanish for a moment, probably when jenkins automatically updates the clones or such. - sleep 5 - if [ ! -f ${NOTES_GIT_PATH}/packages.yml ] || [ ! -f ${NOTES_GIT_PATH}/issues.yml ] ; then - echo "Warning: ${NOTES_GIT_PATH}/packages.yml or issues.yml does not exist, something has changed in notes.git it seems." - echo "Please investigate and fix!" - exit 1 - fi - fi - NOTES=$(grep -c -v "^ " ${NOTES_GIT_PATH}/packages.yml) - ISSUES=$(grep -c -v "^ " ${NOTES_GIT_PATH}/issues.yml) - COUNT_ISSUES=$(grep " -" ${NOTES_GIT_PATH}/packages.yml | egrep -v " - [0-9]+"|wc -l) - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum from ${TABLE[4]} WHERE datum = \"$DATE\"") - if [ -z $RESULT ] ; then - echo "Updating notes stats for $DATE." - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[4]} VALUES (\"$DATE\", \"$NOTES\")" - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[5]} VALUES (\"$DATE\", \"$ISSUES\")" - fi -} - -# -# gather suite stats -# -gather_suite_stats() { - AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(*) FROM sources WHERE suite=\"${SUITE}\"") - COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite=\"${SUITE}\"") - COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite=\"${SUITE}\" AND r.status=\"reproducible\"") - COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"unreproducible\"") - COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"FTBFS\"") - COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"404\"") - COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"not for us\"") - COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"blacklisted\"") - COUNT_OTHER=$(( $COUNT_SOURCELESS+$COUNT_NOTFORUS+$COUNT_BLACKLISTED )) - PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc) - PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc) - PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) - PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) - PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) - PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) - PERCENT_OTHER=$(echo "scale=1 ; ($COUNT_OTHER*100/$COUNT_TOTAL)" | bc) -} - -# -# gather meta pkg stats -# -gather_meta_stats() { - PKGSET_PATH=/srv/reproducible-results/meta_pkgsets-$SUITE/${META_PKGSET[$1]}.pkgset - if [ -f $PKGSET_PATH ] ; then - META_LIST=$(cat $PKGSET_PATH) - if [ ! -z "$META_LIST" ] ; then - META_WHERE="" - # gather data about all packages we know about - # as a result, unknown packages in the package set - # are silently ignored - set +x - for PKG in $META_LIST ; do - if [ -z "$META_WHERE" ] ; then - META_WHERE="s.name in ('$PKG'" - else - META_WHERE="$META_WHERE, '$PKG'" - fi - done - if "$DEBUG" ; then set -x ; fi - META_WHERE="$META_WHERE)" - else - META_WHERE="name = 'meta-name-does-not-exist'" - fi - COUNT_META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;") - COUNT_META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;") - COUNT_META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE;") - COUNT_META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE;") - let META_ALL=COUNT_META_GOOD+COUNT_META_BAD+COUNT_META_UGLY+COUNT_META_REST || META_ALL=1 - PERCENT_META_GOOD=$(echo "scale=1 ; ($COUNT_META_GOOD*100/$META_ALL)" | bc) - PERCENT_META_BAD=$(echo "scale=1 ; ($COUNT_META_BAD*100/$META_ALL)" | bc) - PERCENT_META_UGLY=$(echo "scale=1 ; ($COUNT_META_UGLY*100/$META_ALL)" | bc) - PERCENT_META_REST=$(echo "scale=1 ; ($COUNT_META_REST*100/$META_ALL)" | bc) - # order reproducible packages by name, the rest by build_date - META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY s.name;") - META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") - META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") - META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name AS NAME FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") - else - META_RESULT=false - fi -} - -# -# update meta pkg stats -# -update_meta_pkg_stats() { - for i in $(seq 1 ${#META_PKGSET[@]}) ; do - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"${META_PKGSET[$i]}\"") - if [ -z $RESULT ] ; then - META_RESULT=true - gather_meta_stats $i - if $META_RESULT ; then - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)" - echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE on $DATE." - fi - echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..." - touch -d "$FORCE_DATE 00:00" $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png - fi - done -} - -# -# update bug stats -# -update_bug_stats() { - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") - if [ -z $RESULT ] ; then - echo "Updating bug stats for $DATE." - declare -a DONE - declare -a OPEN - GOT_BTS_RESULTS=false - SQL="INSERT INTO ${TABLE[3]} VALUES (\"$DATE\" " - for TAG in $USERTAGS ; do - OPEN[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:open status:forwarded 2>/dev/null|wc -l) - DONE[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:done archive:both 2>/dev/null|wc -l) - # test if both values are integers - if ! ( [[ ${DONE[$TAG]} =~ ^-?[0-9]+$ ]] && [[ ${OPEN[$TAG]} =~ ^-?[0-9]+$ ]] ) ; then - echo "Non-integers value detected, exiting." - echo "Usertag: $TAG" - echo "Open: ${OPEN[$TAG]}" - echo "Done: ${DONE[$TAG]}" - exit 1 - elif [ ! "${DONE[$TAG]}" = "0" ] || [ ! "${OPEN[$TAG]}" = "0" ] ; then - GOT_BTS_RESULTS=true - fi - SQL="$SQL, ${OPEN[$TAG]}, ${DONE[$TAG]}" - done - SQL="$SQL)" - echo $SQL - if $GOT_BTS_RESULTS ; then - echo "Updating ${PACKAGES_DB} with bug stats for $DATE." - sqlite3 -init ${INIT} ${PACKAGES_DB} "$SQL" - # force regeneration of the image - echo "Touching ${TABLE[3]}.png..." - touch -d "$FORCE_DATE 00:00" $BASE/${TABLE[3]}.png - echo "Touching ${TABLE[7]}.png..." - touch -d "$FORCE_DATE 00:00" $BASE/${TABLE[7]}.png - fi - fi -} - -# -# create the png (and query the db to populate a csv file...) -# -create_png_from_table() { - echo "Checking whether to update $2..." - # $1 = id of the stats table - # $2 = image file name - # $3 = meta package set, only sensible if $1=6 - echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv - # prepare query - WHERE_EXTRA="WHERE suite = '$SUITE'" - if [ $1 -eq 3 ] || [ $1 -eq 4 ] || [ $1 -eq 5 ] ; then - # TABLE[3+4+5] don't have a suite column: - WHERE_EXTRA="" - elif [ $1 -eq 6 ] ; then - # 6 is special too: - WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$3'" - fi - # run query - if [ $1 -eq 1 ] ; then - # not sure if it's worth to generate the following query... - sqlite3 -init ${INIT} --nullvalue 0 -csv ${PACKAGES_DB} "SELECT s.datum, - COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='testing'),0) as 'reproducible_testing', - COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='unstable'),0) as 'reproducible_unstable', - COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='experimental'),0) as 'reproducible_experimental', - (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS unreproducible_testing, - (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS unreproducible_unstable, - (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS unreproducible_experimental, - (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS FTBFS_testing, - (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS FTBFS_unstable, - (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS FTBFS_experimental, - (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS other_testing, - (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS other_unstable, - (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS other_experimental - FROM stats_builds_per_day AS s GROUP BY s.datum" >> ${TABLE[$1]}.csv - elif [ $1 -eq 2 ] ; then - # just make a graph of the oldest reproducible build (ignore FTBFS and unreproducible) - sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv - elif [ $1 -eq 7 ] ; then - sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, $SUM_DONE, $SUM_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv - else - sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv - fi - # this is a gross hack: normally we take the number of colors a table should have... - # for the builds_age table we only want one color, but different ones, so this hack: - COLORS=${COLOR[$1]} - if [ $1 -eq 2 ] ; then - case "$SUITE" in - testing) COLORS=40 ;; - unstable) COLORS=41 ;; - experimental) COLORS=42 ;; - esac - fi - # only generate graph if the query returned data - if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then - echo "Updating $2..." - DIR=$(dirname $2) - mkdir -p $DIR - echo "Generating $2." - /srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" - mv $2 $BASE/$DIR - [ "$DIR" = "." ] || rmdir $(dirname $2) - # create empty dummy png if there havent been any results ever - elif [ ! -f $BASE/$DIR/$(basename $2) ] ; then - DIR=$(dirname $2) - mkdir -p $DIR - echo "Creating $2 dummy." - convert -size 1920x960 xc:#aaaaaa -depth 8 $2 - if [ "$3" != "" ] ; then - local THUMB="${TABLE[1]}_${3}-thumbnail.png" - convert $2 -adaptive-resize 160x80 ${THUMB} - mv ${THUMB} $BASE/$DIR - fi - mv $2 $BASE/$DIR - [ "$DIR" = "." ] || rmdir $(dirname $2) - fi - rm ${TABLE[$1]}.csv -} - -# -# gather bugs stats and generate html table -# -write_usertag_table() { - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") - if [ ! -z "$RESULT" ] ; then - COUNT=0 - TOPEN=0 ; TDONE=0 ; TTOTAL=0 - for FIELD in $(echo ${FIELDS[3]} | tr -d ,) ; do - let "COUNT+=1" - VALUE=$(echo $RESULT | cut -d "|" -f$COUNT) - if [ $COUNT -eq 1 ] ; then - write_page "" - elif [ $((COUNT%2)) -eq 0 ] ; then - write_page "" - TOTAL=$VALUE - let "TOPEN=TOPEN+VALUE" || TOPEN=0 - else - write_page "" - let "TOTAL=TOTAL+VALUE" || true # let FOO=0+0 returns error in bash... - let "TDONE=TDONE+VALUE" - write_page "" - let "TTOTAL=TTOTAL+TOTAL" - fi - done - write_page "" - write_page "
Usertagged bugsOpenDoneTotal
${FIELD:5}$VALUE$VALUE$TOTAL
Total number of usertags on $DATE
(this is not the number of bugs as bugs can have several tags)		$TOPEN$TDONE$TTOTAL
" - fi -} - -# -# create suite stats page -# -create_suite_stats_page() { - VIEW=suite_stats - PAGE=index_${VIEW}.html - MAINLABEL[0]="Reproducibility status for packages in '$SUITE'" - MAINLABEL[2]="Age in days of oldest reproducible build result in '$SUITE'" - echo "$(date) - starting to write $PAGE page." - write_page_header $VIEW "Overview of reproducible builds for packages in $SUITE" - if [ $(echo $PERCENT_TOTAL/1|bc) -lt 98 ] ; then - write_page "

$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE.

" - fi - write_page "

" - set_icon reproducible - write_icon - write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly in $SUITE/$ARCH." - set_icon unreproducible - write_icon - write_page "$COUNT_BAD packages ($PERCENT_BAD%) failed to build reproducibly." - set_icon FTBFS - write_icon - write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source.

" - write_page "

" - if [ $COUNT_SOURCELESS -gt 0 ] ; then - write_page "For " - set_icon 404 - write_icon - write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages sources could not be downloaded," - fi - set_icon not_for_us - write_icon - write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64' will not be build here" - write_page "and those " - set_icon blacklisted - write_icon - write_page "$COUNT_BLACKLISTED blacklisted packages neither.

" - write_page "

" - write_page " \"${MAINLABEL[0]}\"" - for i in 0 2 ; do - # recreate png once a day - if [ ! -f $BASE/$SUITE/${TABLE[$i]}.png ] || [ ! -z $(find $BASE/$SUITE -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then - create_png_from_table $i $SUITE/${TABLE[$i]}.png - fi - done - write_page "

" - write_page_footer - publish_page $SUITE -} - -# -# create pkg set navigation -# -create_pkg_sets_navigation() { - local i - write_page "
  • Tracked package sets in $SUITE:
    • " - for i in $(seq 1 ${#META_PKGSET[@]}) ; do - if [ -f $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png ] ; then - THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" - LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" - write_page "\"$LABEL\"" - write_page "
  • " - write_page "${META_PKGSET[$i]}" - write_page "
    • " - fi - done - write_page "
" -} - -# -# create pkg sets pages -# -create_pkg_sets_pages() { - # - # create index page - # - VIEW=pkg_sets - PAGE=index_${VIEW}.html - echo "$(date) - starting to write $PAGE page." - write_page_header $VIEW "Overview about reproducible builds of specific package sets in $SUITE/$ARCH" - create_pkg_sets_navigation - write_page_footer - publish_page $SUITE/$ARCH - # - # create individual pages for all the sets - # - local i - for i in $(seq 1 ${#META_PKGSET[@]}) ; do - PAGE="pkg_set_${META_PKGSET[$i]}.html" - echo "$(date) - starting to write $PAGE page." - write_page_header $VIEW "Overview about reproducible builds for the ${META_PKGSET[$i]} package set in $SUITE/$ARCH" - create_pkg_sets_navigation - write_page "
" - META_RESULT=true - gather_meta_stats $i - if $META_RESULT ; then - MAINLABEL[6]="Reproducibility status for packages in $SUITE from '${META_PKGSET[$i]}'" - YLABEL[6]="Amount (${META_PKGSET[$i]} packages)" - PNG=${TABLE[6]}_${META_PKGSET[$i]}.png - THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" - # redo pngs once a day - if [ ! -f $BASE/$SUITE/$ARCH/$PNG ] || [ ! -z $(find $BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then - create_png_from_table 6 $SUITE/$ARCH/$PNG ${META_PKGSET[$i]} - convert $BASE/$SUITE/$ARCH/$PNG -adaptive-resize 160x80 $BASE/$SUITE/$ARCH/$THUMB - fi - LABEL="package set '${META_PKGSET[$j]}' in $SUITE/$ARCH" - write_page "

\"$LABEL\"" - write_page "
The package set '${META_PKGSET[$i]}' in $SUITE/$ARCH consists of:
 
" - set_icon unreproducible - write_icon - write_page "$COUNT_META_BAD ($PERCENT_META_BAD%) packages failed to build reproducibly:" - link_packages $META_BAD - write_page "
" - if [ $COUNT_META_UGLY -gt 0 ] ; then - set_icon FTBFS - write_icon - write_page "$COUNT_META_UGLY ($PERCENT_META_UGLY%) packages failed to build from source:" - link_packages $META_UGLY - write_page "
" - fi - if [ $COUNT_META_REST -gt 0 ] ; then - set_icon not_for_us - write_icon - set_icon blacklisted - write_icon - set_icon 404 - write_icon - write_page "$COUNT_META_REST ($PERCENT_META_REST%) packages are either blacklisted, not for us or cannot be downloaded:" - link_packages $META_REST - write_page "
" - fi - write_page " 
" - set_icon reproducible - write_icon - write_page "$COUNT_META_GOOD packages ($PERCENT_META_GOOD%) successfully built reproducibly:" - link_packages $META_GOOD - write_page "
" - write_page "

" - write_page_meta_sign - fi - write_page_footer - publish_page $SUITE/$ARCH - done -} - -# -# create main stats page -# -create_main_stats_page() { - VIEW=stats - PAGE=index_${VIEW}.html - echo "$(date) - starting to write $PAGE page." - write_page_header $VIEW "Overview of various statistics about reproducible builds" - # write suite table - write_page "

" - write_page "" - for SUITE in $SUITES ; do - gather_suite_stats - write_page "" - done - write_page "
suiteall sources packagesreproducible packagesunreproducible packagespackages failing to buildother packages
$SUITE$AMOUNT" - if [ $(echo $PERCENT_TOTAL/1|bc) -lt 98 ] ; then - write_page "($PERCENT_TOTAL% tested)" - fi - write_page "$COUNT_GOOD / $PERCENT_GOOD%$COUNT_BAD / $PERCENT_BAD%$COUNT_UGLY / $PERCENT_UGLY%$COUNT_OTHER / $PERCENT_OTHER%
" - # write suite graphs - write_page "

" - for SUITE in $SUITES ; do - write_page " \"$SUITE" - done - write_page "

" - # write meta pkg graphs per suite - for SUITE in $SUITES ; do - if [ "$SUITE" != "unstable" ] ; then - # only show pkg sets from unstable - continue - fi - for i in $(seq 1 ${#META_PKGSET[@]}) ; do - THUMB=${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png - LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" - write_page "\"$LABEL\"" - done - done - write_page "

" - # write inventory table - write_page "" - write_page "" - write_page "" - write_page "" - SUITE="unstable" - gather_suite_stats - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('unreproducible', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") - write_page "" - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='unreproducible' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") - write_page "" - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status='FTBFS' AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") - write_page "" - write_page "" - SUITE="testing" - gather_suite_stats - write_page "" - SUITE="unstable" - if [ -f ${NOTES_GIT_PATH}/packages.yml ] && [ -f ${NOTES_GIT_PATH}/issues.yml ] ; then - write_page "" - write_page "" - fi - RESULT=$(cat /srv/reproducible-results/modified_in_sid.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh - write_page "" - RESULT=$(cat /srv/reproducible-results/modified_in_exp.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh - write_page "" - write_page "
Various reproducibility statistics
identified distinct issues$ISSUES
total number of identified issues in packages$COUNT_ISSUES
packages with notes about these issues$NOTES
packages in $SUITE with issues but without identified ones$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%
  - unreproducible ones$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%
  - failing to build$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%
packages in $SUITE which need to be fixed$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%
  - in $SUITE$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%
committers to notes.git (in the last three months)$(cd ${NOTES_GIT_PATH} ; git log --since="3 months ago"|grep Author|sort -u |wc -l)
committers to notes.git (in total)$(cd ${NOTES_GIT_PATH} ; git log |grep Author|sort -u |wc -l)
packages modified in our toolchain (in unstable)$(echo $RESULT)
  - (in experimental)$(echo $RESULT)
" - # write bugs with usertags table - write_usertag_table - write_page "

" - # do other global graphs - for i in 3 7 4 5 ; do - write_page " \"${MAINLABEL[$i]}\"" - # redo pngs once a day - if [ ! -f $BASE/${TABLE[$i]}.png ] || [ ! -z $(find $BASE -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then - create_png_from_table $i ${TABLE[$i]}.png - fi - done - write_page "

" - # explain setup - write_explaination_table debian - # write build per day graph - write_page "

" - write_page " \"${MAINLABEL[$i]}\"" - # redo png once a day - if [ ! -f $BASE/${TABLE[1]}.png ] || [ ! -z $(find $BASE -maxdepth 1 -mtime +0 -name ${TABLE[1]}.png) ] ; then - create_png_from_table 1 ${TABLE[1]}.png - fi - # write suite builds age graphs - write_page "

" - for SUITE in $SUITES ; do - write_page " \"age" - done - # write build performace stats - write_page "" - AGE_TESTING=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='testing' AND datum='$DATE'") - AGE_UNSTABLE=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='unstable' AND datum='$DATE'") - AGE_EXPERIMENTAL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='experimental' AND datum='$DATE'") - write_page "" - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(AVG(r.build_duration) AS INTEGER) FROM results AS r WHERE r.build_duration!='' AND r.build_duration!='0' AND r.build_date LIKE '%$DATE%'") - MIN=$(echo $RESULT/60|bc) - SEC=$(echo "$RESULT-($MIN*60)"|bc) - write_page "" - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT CAST(AVG(r.build_duration) AS INTEGER) FROM results AS r WHERE r.build_duration!='' AND r.build_duration!='0' AND r.build_date > datetime('$DATE', '-28 days')") - MIN=$(echo $RESULT/60|bc) - SEC=$(echo "$RESULT-($MIN*60)"|bc) - write_page "" - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(r.build_date) FROM results AS r WHERE r.build_date LIKE '%$DATE%'") - write_page "" - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT COUNT(r.build_date) FROM results AS r WHERE r.build_date > datetime('$DATE', '-28 days')") - RESULT="$(echo $RESULT/28|bc)" - write_page "" - write_page "
Build statistics
oldest build result in testing / unstable / experimental$AGE_TESTING / $AGE_UNSTABLE / $AGE_EXPERIMENTAL days
average test duration (on $DATE)$MIN minutes, $SEC seconds
average test duration (in the last 4 weeks)$MIN minutes, $SEC seconds
packages tested on $DATE$RESULT
packages tested on average per day in the last 4 weeks$RESULT
" - # link to index_breakages - write_page "

" - write_page "
There are some problems in this setup too. And there is documentation too, in case you missed the link at the top. More feedback is always welcome!

" - # the end - write_page_footer - cp $PAGE $BASE/reproducible.html - publish_page -} - -# -# main -# -SUITE="unstable" -update_bug_stats -update_notes_stats -for SUITE in $SUITES ; do - update_suite_stats - gather_suite_stats - create_suite_stats_page - if [ "$SUITE" = "experimental" ] ; then - # no pkg sets in experimental - continue - fi - update_meta_pkg_stats - create_pkg_sets_pages -done -SUITE="unstable" -create_main_stats_page - diff --git a/bin/reproducible_html_pkg_sets.sh b/bin/reproducible_html_pkg_sets.sh new file mode 100755 index 00000000..8196d3eb --- /dev/null +++ b/bin/reproducible_html_pkg_sets.sh @@ -0,0 +1,196 @@ +#!/bin/bash + +# Copyright 2014-2015 Holger Levsen +# © 2015 Mattia Rizzolo +# released under the GPLv=2 + +DEBUG=false +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code defining db access +. /srv/jenkins/bin/reproducible_common.sh + +# +# init some variables +# +ARCH="amd64" # we only care about amd64 status here (for now) +# we only do stats up until yesterday... we also could do today too but not update the db yet... +DATE=$(date -d "1 day ago" '+%Y-%m-%d') +FORCE_DATE=$(date -d "2 day ago" '+%Y-%m-%d') + +# variables related to the stats we do +TABLE[6]=stats_meta_pkg_state +FIELDS[6]="datum, reproducible, unreproducible, FTBFS, other" +COLOR[6]=4 + +# +# gather meta pkg stats +# +gather_meta_stats() { + PKGSET_PATH=/srv/reproducible-results/meta_pkgsets-$SUITE/${META_PKGSET[$1]}.pkgset + if [ -f $PKGSET_PATH ] ; then + META_LIST=$(cat $PKGSET_PATH) + if [ ! -z "$META_LIST" ] ; then + META_WHERE="" + # gather data about all packages we know about + # as a result, unknown packages in the package set + # are silently ignored + set +x + for PKG in $META_LIST ; do + if [ -z "$META_WHERE" ] ; then + META_WHERE="s.name in ('$PKG'" + else + META_WHERE="$META_WHERE, '$PKG'" + fi + done + if "$DEBUG" ; then set -x ; fi + META_WHERE="$META_WHERE)" + else + META_WHERE="name = 'meta-name-does-not-exist'" + fi + COUNT_META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;") + COUNT_META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;") + COUNT_META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE;") + COUNT_META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE;") + let META_ALL=COUNT_META_GOOD+COUNT_META_BAD+COUNT_META_UGLY+COUNT_META_REST || META_ALL=1 + PERCENT_META_GOOD=$(echo "scale=1 ; ($COUNT_META_GOOD*100/$META_ALL)" | bc) + PERCENT_META_BAD=$(echo "scale=1 ; ($COUNT_META_BAD*100/$META_ALL)" | bc) + PERCENT_META_UGLY=$(echo "scale=1 ; ($COUNT_META_UGLY*100/$META_ALL)" | bc) + PERCENT_META_REST=$(echo "scale=1 ; ($COUNT_META_REST*100/$META_ALL)" | bc) + # order reproducible packages by name, the rest by build_date + META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY s.name;") + META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") + META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") + META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name AS NAME FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") + else + META_RESULT=false + fi +} + +# +# update meta pkg stats +# +update_meta_pkg_stats() { + for i in $(seq 1 ${#META_PKGSET[@]}) ; do + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"${META_PKGSET[$i]}\"") + if [ -z $RESULT ] ; then + META_RESULT=true + gather_meta_stats $i + if $META_RESULT ; then + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)" + echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE on $DATE." + fi + echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..." + touch -d "$FORCE_DATE 00:00" $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png + fi + done +} + +# +# create pkg set navigation +# +create_pkg_sets_navigation() { + local i + write_page "
  • Tracked package sets in $SUITE:
    • " + for i in $(seq 1 ${#META_PKGSET[@]}) ; do + if [ -f $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png ] ; then + THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" + LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" + write_page "\"$LABEL\"" + write_page "
  • " + write_page "${META_PKGSET[$i]}" + write_page "
    • " + fi + done + write_page "
" +} + +# +# create pkg sets pages +# +create_pkg_sets_pages() { + # + # create index page + # + VIEW=pkg_sets + PAGE=index_${VIEW}.html + echo "$(date) - starting to write $PAGE page." + write_page_header $VIEW "Overview about reproducible builds of specific package sets in $SUITE/$ARCH" + create_pkg_sets_navigation + write_page_footer + publish_page $SUITE/$ARCH + # + # create individual pages for all the sets + # + local i + for i in $(seq 1 ${#META_PKGSET[@]}) ; do + PAGE="pkg_set_${META_PKGSET[$i]}.html" + echo "$(date) - starting to write $PAGE page." + write_page_header $VIEW "Overview about reproducible builds for the ${META_PKGSET[$i]} package set in $SUITE/$ARCH" + create_pkg_sets_navigation + write_page "
" + META_RESULT=true + gather_meta_stats $i + if $META_RESULT ; then + MAINLABEL[6]="Reproducibility status for packages in $SUITE from '${META_PKGSET[$i]}'" + YLABEL[6]="Amount (${META_PKGSET[$i]} packages)" + PNG=${TABLE[6]}_${META_PKGSET[$i]}.png + THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" + # redo pngs once a day + if [ ! -f $BASE/$SUITE/$ARCH/$PNG ] || [ ! -z $(find $BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then + create_png_from_table 6 $SUITE/$ARCH/$PNG ${META_PKGSET[$i]} + convert $BASE/$SUITE/$ARCH/$PNG -adaptive-resize 160x80 $BASE/$SUITE/$ARCH/$THUMB + fi + LABEL="package set '${META_PKGSET[$j]}' in $SUITE/$ARCH" + write_page "

\"$LABEL\"" + write_page "
The package set '${META_PKGSET[$i]}' in $SUITE/$ARCH consists of:
 
" + set_icon unreproducible + write_icon + write_page "$COUNT_META_BAD ($PERCENT_META_BAD%) packages failed to build reproducibly:" + link_packages $META_BAD + write_page "
" + if [ $COUNT_META_UGLY -gt 0 ] ; then + set_icon FTBFS + write_icon + write_page "$COUNT_META_UGLY ($PERCENT_META_UGLY%) packages failed to build from source:" + link_packages $META_UGLY + write_page "
" + fi + if [ $COUNT_META_REST -gt 0 ] ; then + set_icon not_for_us + write_icon + set_icon blacklisted + write_icon + set_icon 404 + write_icon + write_page "$COUNT_META_REST ($PERCENT_META_REST%) packages are either blacklisted, not for us or cannot be downloaded:" + link_packages $META_REST + write_page "
" + fi + write_page " 
" + set_icon reproducible + write_icon + write_page "$COUNT_META_GOOD packages ($PERCENT_META_GOOD%) successfully built reproducibly:" + link_packages $META_GOOD + write_page "
" + write_page "

" + write_page_meta_sign + fi + write_page_footer + publish_page $SUITE/$ARCH + done +} + +# +# main +# +for SUITE in $SUITES ; do + if [ "$SUITE" = "experimental" ] ; then + # no pkg sets in experimental + continue + fi + update_meta_pkg_stats + create_pkg_sets_pages +done + -- cgit v1.2.3-70-g09d2