#!/bin/bash # Copyright 2014-2015 Holger Levsen # © 2015 Mattia Rizzolo # released under the GPLv=2 DEBUG=true . /srv/jenkins/bin/common-functions.sh common_init "$@" # common code defining db access . /srv/jenkins/bin/reproducible_common.sh # # init some variables # ARCH="amd64" # we only care about amd64 status here (for now) # we only do stats up until yesterday... we also could do today too but not update the db yet... DATE=$(date -d "1 day ago" '+%Y-%m-%d') FORCE_DATE=$(date -d "2 day ago" '+%Y-%m-%d') NOTES_GIT_PATH="/var/lib/jenkins/jobs/reproducible_html_notes/workspace" # variables related to the stats we do TABLE[0]=stats_pkg_state TABLE[1]=stats_builds_per_day TABLE[2]=stats_builds_age TABLE[3]=stats_bugs TABLE[4]=stats_notes TABLE[5]=stats_issues TABLE[6]=stats_meta_pkg_state TABLE[7]=stats_bugs_state FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested" FIELDS[1]="datum" for i in reproducible unreproducible FTBFS other ; do for j in $SUITES ; do FIELDS[1]="${FIELDS[1]}, ${i}_${j}" done done FIELDS[2]="datum, oldest" FIELDS[3]="datum " for TAG in $USERTAGS ; do FIELDS[3]="${FIELDS[3]}, open_$TAG, done_$TAG" done FIELDS[4]="datum, packages_with_notes" FIELDS[5]="datum, known_issues" FIELDS[6]="datum, reproducible, unreproducible, FTBFS, other" FIELDS[7]="datum, done_bugs, open_bugs" SUM_DONE="(0" SUM_OPEN="(0" for TAG in $USERTAGS ; do SUM_DONE="$SUM_DONE+done_$TAG" SUM_OPEN="$SUM_OPEN+open_$TAG" done SUM_DONE="$SUM_DONE)" SUM_OPEN="$SUM_OPEN)" COLOR[0]=5 COLOR[1]=12 COLOR[2]=1 COLOR[3]=28 COLOR[4]=1 COLOR[5]=1 COLOR[6]=4 COLOR[7]=2 MAINLABEL[1]="Amount of packages built each day" MAINLABEL[3]="Usertags on bugs for user reproducible-builds@lists.alioth.debian.org" MAINLABEL[4]="Packages which have notes" MAINLABEL[5]="Identified issues" MAINLABEL[7]="Open and closed bugs" YLABEL[0]="Amount (total)" YLABEL[1]="Amount (per day)" YLABEL[2]="Age in days" YLABEL[3]="Amount of bugs" YLABEL[4]="Amount of packages" YLABEL[5]="Amount of issues" YLABEL[7]="Amount of bugs open / closed" # # update package + build stats # update_suite_stats() { RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,suite from ${TABLE[0]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\"") if [ -z $RESULT ] ; then echo "Updating packages and builds stats for $SUITE on $DATE." ALL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(name) FROM sources WHERE suite='${SUITE}'") GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE';") GOOAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)='$DATE';") BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE';") BAAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)='$DATE';") UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE';") UGLDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)='$DATE';") REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND date(r.build_date)<='$DATE';") RESDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND s.suite='$SUITE' AND date(r.build_date)='$DATE';") OLDESTG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.status = 'reproducible' AND s.suite='$SUITE' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;") OLDESTB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;") OLDESTU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT r.build_date FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND NOT date(r.build_date)>='$DATE' ORDER BY r.build_date LIMIT 1;") DIFFG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTG');") if [ -z $DIFFG ] ; then DIFFG=0 ; fi DIFFB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTB');") if [ -z $DIFFB ] ; then DIFFB=0 ; fi DIFFU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTU');") if [ -z $DIFFU ] ; then DIFFU=0 ; fi let "TOTAL=GOOD+BAD+UGLY+REST" || true # let FOO=0+0 returns error in bash... if [ "$ALL" != "$TOTAL" ] ; then let "UNTESTED=ALL-TOTAL" else UNTESTED=0 fi sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[0]} VALUES (\"$DATE\", \"$SUITE\", $UNTESTED, $GOOD, $BAD, $UGLY, $REST)" sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)" sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")" # we do 3 later and 6 is special anyway... for i in 0 1 2 4 5 ; do PREFIX="" if [ $i -eq 0 ] ; then PREFIX=$SUITE fi # force regeneration of the image if it exists if [ -f $BASE/$PREFIX/${TABLE[$i]}.png ] ; then echo "Touching $PREFIX/${TABLE[$i]}.png..." touch -d "$FORCE_DATE 00:00" $BASE/$PREFIX/${TABLE[$i]}.png fi done fi } # # update notes stats # update_notes_stats() { if [ ! -d ${NOTES_GIT_PATH} ] ; then echo "Warning: ${NOTES_GIT_PATH} does not exist, has the job been renamed???" echo "Please investigate and fix!" exit 1 elif [ ! -f ${NOTES_GIT_PATH}/packages.yml ] || [ ! -f ${NOTES_GIT_PATH}/issues.yml ] ; then # retry. sometimes these files vanish for a moment, probably when jenkins automatically updates the clones or such. sleep 5 if [ ! -f ${NOTES_GIT_PATH}/packages.yml ] || [ ! -f ${NOTES_GIT_PATH}/issues.yml ] ; then echo "Warning: ${NOTES_GIT_PATH}/packages.yml or issues.yml does not exist, something has changed in notes.git it seems." echo "Please investigate and fix!" exit 1 fi fi NOTES=$(grep -c -v "^ " ${NOTES_GIT_PATH}/packages.yml) ISSUES=$(grep -c -v "^ " ${NOTES_GIT_PATH}/issues.yml) COUNT_ISSUES=$(grep " -" ${NOTES_GIT_PATH}/packages.yml | egrep -v " - [0-9]+"|wc -l) RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum from ${TABLE[4]} WHERE datum = \"$DATE\"") if [ -z $RESULT ] ; then echo "Updating notes stats for $DATE." sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[4]} VALUES (\"$DATE\", \"$NOTES\")" sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[5]} VALUES (\"$DATE\", \"$ISSUES\")" fi } # # gather suite stats # gather_suite_stats() { AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(*) FROM sources WHERE suite=\"${SUITE}\"") COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite=\"${SUITE}\"") COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite=\"${SUITE}\" AND r.status=\"reproducible\"") COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"unreproducible\"") COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"FTBFS\"") COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"404\"") COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"not for us\"") COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"blacklisted\"") COUNT_OTHER=$(( $COUNT_SOURCELESS+$COUNT_NOTFORUS+$COUNT_BLACKLISTED )) PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc) PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc) PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) PERCENT_OTHER=$(echo "scale=1 ; ($COUNT_OTHER*100/$COUNT_TOTAL)" | bc) } # # gather meta pkg stats # gather_meta_stats() { PKGSET_PATH=/srv/reproducible-results/meta_pkgsets-$SUITE/${META_PKGSET[$1]}.pkgset if [ -f $PKGSET_PATH ] ; then META_LIST=$(cat $PKGSET_PATH) if [ ! -z "$META_LIST" ] ; then META_WHERE="" # gather data about all packages we know about # as a result, unknown packages in the package set # are silently ignored set +x for PKG in $META_LIST ; do if [ -z "$META_WHERE" ] ; then META_WHERE="s.name in ('$PKG'" else META_WHERE="$META_WHERE, '$PKG'" fi done if "$DEBUG" ; then set -x ; fi META_WHERE="$META_WHERE)" else META_WHERE="name = 'meta-name-does-not-exist'" fi COUNT_META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;") COUNT_META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;") COUNT_META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE;") COUNT_META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE;") let META_ALL=COUNT_META_GOOD+COUNT_META_BAD+COUNT_META_UGLY+COUNT_META_REST || META_ALL=1 PERCENT_META_GOOD=$(echo "scale=1 ; ($COUNT_META_GOOD*100/$META_ALL)" | bc) PERCENT_META_BAD=$(echo "scale=1 ; ($COUNT_META_BAD*100/$META_ALL)" | bc) PERCENT_META_UGLY=$(echo "scale=1 ; ($COUNT_META_UGLY*100/$META_ALL)" | bc) PERCENT_META_REST=$(echo "scale=1 ; ($COUNT_META_REST*100/$META_ALL)" | bc) # order reproducible packages by name, the rest by build_date META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY s.name;") META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name AS NAME FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") else META_RESULT=false fi } # # update meta pkg stats # update_meta_pkg_stats() { for i in $(seq 1 ${#META_PKGSET[@]}) ; do RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"${META_PKGSET[$i]}\"") if [ -z $RESULT ] ; then META_RESULT=true gather_meta_stats $i if $META_RESULT ; then sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)" echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE on $DATE." fi echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..." touch -d "$FORCE_DATE 00:00" $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png fi done } # # update bug stats # update_bug_stats() { RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") if [ -z $RESULT ] ; then echo "Updating bug stats for $DATE." declare -a DONE declare -a OPEN GOT_BTS_RESULTS=false SQL="INSERT INTO ${TABLE[3]} VALUES (\"$DATE\" " for TAG in $USERTAGS ; do OPEN[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:open status:forwarded 2>/dev/null|wc -l) DONE[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:done archive:both 2>/dev/null|wc -l) # test if both values are integers if ! ( [[ ${DONE[$TAG]} =~ ^-?[0-9]+$ ]] && [[ ${OPEN[$TAG]} =~ ^-?[0-9]+$ ]] ) ; then echo "Non-integers value detected, exiting." echo "Usertag: $TAG" echo "Open: ${OPEN[$TAG]}" echo "Done: ${DONE[$TAG]}" exit 1 elif [ ! "${DONE[$TAG]}" = "0" ] || [ ! "${OPEN[$TAG]}" = "0" ] ; then GOT_BTS_RESULTS=true fi SQL="$SQL, ${OPEN[$TAG]}, ${DONE[$TAG]}" done SQL="$SQL)" echo $SQL if $GOT_BTS_RESULTS ; then echo "Updating ${PACKAGES_DB} with bug stats for $DATE." sqlite3 -init ${INIT} ${PACKAGES_DB} "$SQL" # force regeneration of the image echo "Touching ${TABLE[3]}.png..." touch -d "$FORCE_DATE 00:00" $BASE/${TABLE[3]}.png echo "Touching ${TABLE[7]}.png..." touch -d "$FORCE_DATE 00:00" $BASE/${TABLE[7]}.png fi fi } # # create the png (and query the db to populate a csv file...) # create_png_from_table() { echo "Checking whether to update $2..." # $1 = id of the stats table # $2 = image file name # $3 = meta package set, only sensible if $1=6 echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv # prepare query WHERE_EXTRA="WHERE suite = '$SUITE'" if [ $1 -eq 3 ] || [ $1 -eq 4 ] || [ $1 -eq 5 ] ; then # TABLE[3+4+5] don't have a suite column: WHERE_EXTRA="" elif [ $1 -eq 6 ] ; then # 6 is special too: WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$3'" fi # run query if [ $1 -eq 1 ] ; then # not sure if it's worth to generate the following query... sqlite3 -init ${INIT} --nullvalue 0 -csv ${PACKAGES_DB} "SELECT s.datum, COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='testing'),0) as 'reproducible_testing', COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='unstable'),0) as 'reproducible_unstable', COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='experimental'),0) as 'reproducible_experimental', (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS unreproducible_testing, (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS unreproducible_unstable, (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS unreproducible_experimental, (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS FTBFS_testing, (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS FTBFS_unstable, (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS FTBFS_experimental, (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS other_testing, (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS other_unstable, (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS other_experimental FROM stats_builds_per_day AS s GROUP BY s.datum" >> ${TABLE[$1]}.csv elif [ $1 -eq 2 ] ; then # just make a graph of the oldest reproducible build (ignore FTBFS and unreproducible) sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv elif [ $1 -eq 7 ] ; then sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, $SUM_DONE, $SUM_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv else sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv fi # this is a gross hack: normally we take the number of colors a table should have... # for the builds_age table we only want one color, but different ones, so this hack: COLORS=${COLOR[$1]} if [ $1 -eq 2 ] ; then case "$SUITE" in testing) COLORS=40 ;; unstable) COLORS=41 ;; experimental) COLORS=42 ;; esac fi # only generate graph if the query returned data if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then echo "Updating $2..." DIR=$(dirname $2) mkdir -p $DIR echo "Generating $2." /srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" mv $2 $BASE/$DIR [ "$DIR" = "." ] || rmdir $(dirname $2) # create empty dummy png if there havent been any results ever elif [ ! -f $BASE/$DIR/$(basename $2) ] ; then DIR=$(dirname $2) mkdir -p $DIR echo "Creating $2 dummy." convert -size 1600x800 xc:#aaaaaa -depth 8 $2 if [ "$3" != "" ] ; then local THUMB="${TABLE[1]}_${3}-thumbnail.png" convert $2 -adaptive-resize 160x80 ${THUMB} mv ${THUMB} $BASE/$DIR fi mv $2 $BASE/$DIR [ "$DIR" = "." ] || rmdir $(dirname $2) fi rm ${TABLE[$1]}.csv } # # gather bugs stats and generate html table # write_usertag_table() { RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") if [ ! -z "$RESULT" ] ; then COUNT=0 TOPEN=0 ; TDONE=0 ; TTOTAL=0 for FIELD in $(echo ${FIELDS[3]} | tr -d ,) ; do let "COUNT+=1" VALUE=$(echo $RESULT | cut -d "|" -f$COUNT) if [ $COUNT -eq 1 ] ; then write_page "" elif [ $((COUNT%2)) -eq 0 ] ; then write_page "" TOTAL=$VALUE let "TOPEN=TOPEN+VALUE" || TOPEN=0 else write_page "" let "TOTAL=TOTAL+VALUE" || true # let FOO=0+0 returns error in bash... let "TDONE=TDONE+VALUE" write_page "" let "TTOTAL=TTOTAL+TOTAL" fi done write_page "" write_page "
Usertagged bugsOpenDoneTotal
${FIELD:5}$VALUE$VALUE$TOTAL
Total number of usertags on $DATE
(this is not the number of bugs as bugs can have several tags)		$TOPEN$TDONE$TTOTAL
" fi } # # create suite stats page # create_suite_stats_page() { VIEW=suite_stats PAGE=index_${VIEW}.html MAINLABEL[0]="Reproducibility status for packages in '$SUITE'" MAINLABEL[2]="Age in days of oldest reproducible build result in '$SUITE'" echo "$(date) - starting to write $PAGE page." write_page_header $VIEW "Overview of reproducible builds for packages in $SUITE" if [ $(echo $PERCENT_TOTAL/1|bc) -lt 98 ] ; then write_page "

$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE.

" fi write_page "

" set_icon reproducible write_icon write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly in $SUITE/$ARCH." set_icon unreproducible write_icon write_page "$COUNT_BAD packages ($PERCENT_BAD%) failed to built reproducibly." set_icon FTBFS write_icon write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source.

" write_page "

" if [ $COUNT_SOURCELESS -gt 0 ] ; then write_page "For " set_icon 404 write_icon write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages sources could not be downloaded," fi set_icon not_for_us write_icon write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64' will not be build here" write_page "and those " set_icon blacklisted write_icon write_page "$COUNT_BLACKLISTED blacklisted packages neither.

" write_page "

" write_page " \"${MAINLABEL[0]}\"" for i in 0 2 ; do # recreate png once a day if [ ! -f $BASE/$SUITE/${TABLE[$i]}.png ] || [ ! -z $(find $BASE/$SUITE -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then create_png_from_table $i $SUITE/${TABLE[$i]}.png fi done write_page "

" write_page_footer publish_page $SUITE } # # create pkg set navigation # create_pkg_sets_navigation() { local i write_page "" } # # create pkg sets pages # create_pkg_sets_pages() { # # create index page # VIEW=pkg_sets PAGE=index_${VIEW}.html echo "$(date) - starting to write $PAGE page." write_page_header $VIEW "Overview about reproducible builds of specific package sets in $SUITE/$ARCH" create_pkg_sets_navigation write_page_footer publish_page $SUITE/$ARCH # # create individual pages for all the sets # local i for i in $(seq 1 ${#META_PKGSET[@]}) ; do PAGE="pkg_set_${META_PKGSET[$i]}.html" echo "$(date) - starting to write $PAGE page." write_page_header $VIEW "Overview about reproducible builds for the ${META_PKGSET[$i]} package set in $SUITE/$ARCH" create_pkg_sets_navigation write_page "
" META_RESULT=true gather_meta_stats $i if $META_RESULT ; then MAINLABEL[6]="Reproducibility status for packages in $SUITE from '${META_PKGSET[$i]}'" YLABEL[6]="Amount (${META_PKGSET[$i]} packages)" PNG=${TABLE[6]}_${META_PKGSET[$i]}.png THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" # redo pngs once a day if [ ! -f $BASE/$SUITE/$ARCH/$PNG ] || [ ! -z $(find $BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then create_png_from_table 6 $SUITE/$ARCH/$PNG ${META_PKGSET[$i]} convert $BASE/$SUITE/$ARCH/$PNG -adaptive-resize 160x80 $BASE/$SUITE/$ARCH/$THUMB fi LABEL="package set '${META_PKGSET[$j]}' in $SUITE/$ARCH" write_page "

\"$LABEL\"" write_page "
The package set '${META_PKGSET[$i]}' in $SUITE/$ARCH consists of:
 
" set_icon unreproducible write_icon write_page "$COUNT_META_BAD ($PERCENT_META_BAD%) packages failed to built reproducibly:" link_packages $META_BAD write_page "
" if [ $COUNT_META_UGLY -gt 0 ] ; then set_icon FTBFS write_icon write_page "$COUNT_META_UGLY ($PERCENT_META_UGLY%) packages failed to build from source:" link_packages $META_UGLY write_page "
" fi if [ $COUNT_META_REST -gt 0 ] ; then set_icon not_for_us write_icon set_icon blacklisted write_icon set_icon 404 write_icon write_page "$COUNT_META_REST ($PERCENT_META_REST%) packages are either blacklisted, not for us or cannot be downloaded:" link_packages $META_REST write_page "
" fi write_page " 
" set_icon reproducible write_icon write_page "$COUNT_META_GOOD packages ($PERCENT_META_GOOD%) successfully built reproducibly:" link_packages $META_GOOD write_page "
" write_page "

" write_page_meta_sign fi write_page_footer publish_page $SUITE/$ARCH done } # # create main stats page # create_main_stats_page() { VIEW=stats PAGE=index_${VIEW}.html echo "$(date) - starting to write $PAGE page." write_page_header $VIEW "Overview of various statistics about reproducible builds" # write suite table write_page "

" write_page "" for SUITE in $SUITES ; do gather_suite_stats write_page "" done write_page "
suiteall sources packagesreproducible packagesunreproducible packagespackages failing to buildother packages
$SUITE$AMOUNT" if [ $(echo $PERCENT_TOTAL/1|bc) -lt 98 ] ; then write_page "($PERCENT_TOTAL% tested)" fi write_page "$COUNT_GOOD / $PERCENT_GOOD%$COUNT_BAD / $PERCENT_BAD%$COUNT_UGLY / $PERCENT_UGLY%$COUNT_OTHER / $PERCENT_OTHER%
" # write suite graphs write_page "

" for SUITE in $SUITES ; do write_page " \"$SUITE" done write_page "

" # write meta pkg graphs per suite for SUITE in $SUITES ; do if [ "$SUITE" != "unstable" ] ; then # only show pkg sets from unstable continue fi for i in $(seq 1 ${#META_PKGSET[@]}) ; do THUMB=${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" write_page "\"$LABEL\"" done done write_page "

" # write inventory table write_page "" write_page "" write_page "" write_page "" SUITE="unstable" gather_suite_stats RESULT=$(sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ('unreproducible', 'FTBFS', 'blacklisted') AND s.id NOT IN (SELECT package_id FROM notes) AND s.suite='$SUITE' AND s.architecture='$ARCH')") write_page "" write_page "" if [ -f ${NOTES_GIT_PATH}/packages.yml ] && [ -f ${NOTES_GIT_PATH}/issues.yml ] ; then write_page "" write_page "" fi RESULT=$(cat /srv/reproducible-results/modified_in_sid.txt || echo "unknown") # written by reproducible_html_repository_comparison.sh write_page "" write_page "
 amount
identified distinct issues$ISSUES
total number of identified issues in packages$COUNT_ISSUES
packages with notes about these issues$NOTES
packages in $SUITE with issues but without identified ones$RESULT / $(echo "scale=1 ; ($RESULT*100/$COUNT_TOTAL)" | bc)%
packages in $SUITE which need to be fixed$(echo $COUNT_BAD + $COUNT_UGLY |bc) / $(echo $PERCENT_BAD + $PERCENT_UGLY|bc)%
committers to notes.git (in the last three months)$(cd ${NOTES_GIT_PATH} ; git log --since="3 months ago"|grep Author|sort -u |wc -l)
committers to notes.git (in total)$(cd ${NOTES_GIT_PATH} ; git log |grep Author|sort -u |wc -l)
packages modified in our toolchain$(echo $RESULT)
" # write bugs with usertags table write_usertag_table write_page "

" # do other global graphs for i in 3 7 4 5 ; do write_page " \"${MAINLABEL[$i]}\"" # redo pngs once a day if [ ! -f $BASE/${TABLE[$i]}.png ] || [ ! -z $(find $BASE -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then create_png_from_table $i ${TABLE[$i]}.png fi done # explain setup write_page "

" write_page "" write_page "" write_page "" write_page "" write_page "" write_page "" write_page "" write_page "" local NUM_CPU=$(cat /proc/cpuinfo |grep ^processor|wc -l) write_page "" write_page "" write_page "" write_page "" write_page "" write_page "" write_page "" write_page "" write_page "
variationfirst buildsecond build
hostname$(hostname)i-capture-the-hostname
domainname$(hostname -d)i-capture-the-domainname
env BUILDUSERIDBUILDUSERID=\"1111\"BUILDUSERID=\"2222\"
env BUILDUSERNAMEBUILDUSERNAME=\"pbuilder1\"BUILDUSERNAME=\"pbuilder2\"
env TZTZ=\"/usr/share/zoneinfo/Etc/GMT+12\"TZ=\"/usr/share/zoneinfo/Etc/GMT-14\"
env LANGLANG=\"en_GB.UTF-8\"LANG=\"fr_CH.UTF-8\"
env LC_ALLunsetLC_ALL=\"fr_CH.UTF-8\"
env DEB_BUILD_OPTIONSDEB_BUILD_OPTIONS=\"parallel=$NUM_CPU\"DEB_BUILD_OPTIONS=\"parallel=$(echo $NUM_CPU-1|bc)\"
(using a different number of cores is on the agenda)
UTS namespaceshared with the hostmodified using /usr/bin/unshare --uts
kernel version, modified using /usr/bin/linux64 --uname-2.6$(uname -sr)$(/usr/bin/linux64 --uname-2.6 uname -sr)
umask00220002
CPU type$(cat /proc/cpuinfo|grep 'model name'|head -1|cut -d ":" -f2-)same for both builds (currently, work in progress)
year, month, datetoday ($DATE)same for both builds (currently, work in progress)
hour, minutehour is usually the same...usually, the minute differs... (currently, work in progress)
everything else...is likely the same. So far, this is just about the prospects of reproducible builds of Debian - there will be more variations in the wild.
" # write build per day graph write_page "

" write_page " \"${MAINLABEL[$i]}\"" # redo png once a day if [ ! -f $BASE/${TABLE[1]}.png ] || [ ! -z $(find $BASE -maxdepth 1 -mtime +0 -name ${TABLE[1]}.png) ] ; then create_png_from_table 1 ${TABLE[1]}.png fi # write suite builds age graphs write_page "

" for SUITE in $SUITES ; do write_page " \"age" done # write build performace stats write_page "" AGE_TESTING=$(sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='testing' AND datum='$DATE'") AGE_UNSTABLE=$(sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='unstable' AND datum='$DATE'") AGE_EXPERIMENTAL=$(sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT CAST(max(oldest_reproducible, oldest_unreproducible, oldest_FTBFS) AS INTEGER) FROM ${TABLE[2]} WHERE suite='experimental' AND datum='$DATE'") write_page "" RESULT=$(sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT CAST(AVG(r.build_duration) AS INTEGER) FROM results AS r WHERE r.build_duration!='' AND r.build_duration!='0' AND r.build_date LIKE '%$DATE%'") MIN=$(echo $RESULT/60|bc) SEC=$(echo "$RESULT-($MIN*60)"|bc) write_page "" RESULT=$(sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT CAST(AVG(r.build_duration) AS INTEGER) FROM results AS r WHERE r.build_duration!='' AND r.build_duration!='0' AND r.build_date > datetime('$DATE', '-28 days')") MIN=$(echo $RESULT/60|bc) SEC=$(echo "$RESULT-($MIN*60)"|bc) write_page "" RESULT=$(sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT COUNT(r.build_date) FROM results AS r WHERE r.build_date LIKE '%$DATE%'") write_page "" RESULT=$(sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT COUNT(r.build_date) FROM results AS r WHERE r.build_date > datetime('$DATE', '-28 days')") RESULT="$(echo $RESULT/28|bc)" write_page "" write_page "
 amount
oldest build result in testing / unstable / experimental$AGE_TESTING / $AGE_UNSTABLE / $AGE_EXPERIMENTAL days
average test duration (on $DATE)$MIN minutes, $SEC seconds
average test duration (in the last 4 weeks)$MIN minutes, $SEC seconds
packages tested on $DATE$RESULT
packages tested on average per day in the last 4 weeks$RESULT
" # link to index_breakages write_page "

" write_page "
There are some problems in this setup too. And there is documentation too, in case you missed the link at the top. More feedback is always welcome!

" # the end write_page_footer publish_page } # # main # SUITE="unstable" update_bug_stats update_notes_stats create_main_stats_page for SUITE in $SUITES ; do update_suite_stats gather_suite_stats create_suite_stats_page if [ "$SUITE" = "experimental" ] ; then # no pkg sets in experimental continue fi update_meta_pkg_stats create_pkg_sets_pages done