From 343bfa89524e42ddf836cbcd3dd7d67687ba379a Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Wed, 15 Oct 2014 19:49:53 +0200 Subject: reproducible: add stats page with graphs --- bin/make_graph.py | 42 +++++++++++++++++++++++ bin/reproducible_common.sh | 29 +++++++++++++++- bin/reproducible_stats.sh | 83 +++++++++++++++++++++++++++++++++++++++++++++- update_jdn.sh | 4 +-- 4 files changed, 154 insertions(+), 4 deletions(-) create mode 100755 bin/make_graph.py diff --git a/bin/make_graph.py b/bin/make_graph.py new file mode 100755 index 00000000..c87bd5c0 --- /dev/null +++ b/bin/make_graph.py @@ -0,0 +1,42 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# Copyright 2009-2014 Holger Levsen (holger@layer-acht.org) +# +# based on similar code taken from piuparts-reports.py written by me + +import os +import sys +import string +from rpy2 import robjects +from rpy2.robjects.packages import importr + +def main(): + if len(sys.argv) != 6: + print "we need exactly five params: csvfilein, pngoutfile, color, mainlabel, ylabl" + return + filein = sys.argv[1] + fileout = sys.argv[2] + colors = sys.argv[3] + columns = str(int(colors)+1) + mainlabel = sys.argv[4] + ylabel = sys.argv[5] + countsfile = os.path.join(filein) + pngfile = os.path.join(fileout) + grdevices = importr('grDevices') + grdevices.png(file=pngfile, width=1600, height=800, pointsize=10, res=100, antialias="none") + r = robjects.r + r('t <- (read.table("'+countsfile+'",sep=",",header=1,row.names=1))') + r('cname <- c("date",rep(colnames(t)))') + # thanks to http://tango.freedesktop.org/Generic_Icon_Theme_Guidelines for those nice colors + r('palette(c("#4e9a06", "#f57900", "#cc0000", "#2e3436", "#888a85"))') + r('v <- t[0:nrow(t),0:'+colors+']') + # make graph since day 1 + r('barplot(t(v),col = 1:'+columns+', main="'+mainlabel+'", xlab="", ylab="'+ylabel+'", space=0, border=NA)') + r('legend(x="bottom",legend=colnames(t), ncol=2,fill=1:'+columns+',xjust=0.5,yjust=0,bty="n")') + grdevices.dev_off() + +if __name__ == "__main__": + main() + +# vi:set et ts=4 sw=4 : diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index b78ecf07..3e77542a 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -42,7 +42,34 @@ elif [ ! -f ${PACKAGES_DB} ] ; then CREATE TABLE sources (name TEXT NOT NULL, version TEXT NOT NULL)' - # 30 seconds timeout when trying to get a lock + sqlite3 ${PACKAGES_DB} ' + CREATE TABLE stats_pkg_state + (datum TEXT NOT NULL, + suite TEXT NOT NULL, + untested INTEGER, + reproducible INTEGER, + unreproducible INTEGER, + FTBFS INTEGER, + other INTEGER, + PRIMARY KEY (datum))' + sqlite3 ${PACKAGES_DB} ' + CREATE TABLE stats_builds_per_day + (datum TEXT NOT NULL, + suite TEXT NOT NULL, + reproducible INTEGER, + unreproducible INTEGER, + FTBFS INTEGER, + other INTEGER, + PRIMARY KEY (datum))' + sqlite3 ${PACKAGES_DB} ' + CREATE TABLE stats_builds_age + (datum TEXT NOT NULL, + suite TEXT NOT NULL, + oldest_reproducible REAL, + oldest_unreproducible REAL, + oldest_FTBFS REAL, + PRIMARY KEY (datum))' + # 60 seconds timeout when trying to get a lock cat >/var/lib/jenkins/reproducible.init <<-EOF .timeout 60000 EOF diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh index 434cd885..02d17c94 100755 --- a/bin/reproducible_stats.sh +++ b/bin/reproducible_stats.sh @@ -60,6 +60,7 @@ SPOKENTARGET["last_24h"]="packages tested in the last 24h" SPOKENTARGET["last_48h"]="packages tested in the last 48h" SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" +SPOKENTARGET["stats"]="various statistics about reproducible builds" SPOKENTARGET["notes"]="packages with notes" SPOKENTARGET["issues"]="known issues related to reproducible builds" SPOKENTARGET["reproducible"]="packages which built reproducibly" @@ -510,7 +511,7 @@ write_page_header() { set_icon $MY_STATE $WITH # sets ICON and STATE_TARGET_NAME write_page "
  • \"${STATE_TARGET_NAME}
    • " done - for TARGET in issues notes $ALLVIEWS dd-list ; do + for TARGET in issues notes $ALLVIEWS dd-list stats ; do if [ "$TARGET" = "$1" ] ; then continue elif [ "$TARGET" = "issues" ] ; then @@ -721,4 +722,84 @@ rm $TMPFILE write_page_footer publish_page +# +# create stats +# +# FIXME: we only do stats up until yesterday... we also could do today too but not update the db yet... +DATE=$(date -d "1 day ago" '+%Y-%m-%d') +TABLE[0]=stats_pkg_state +TABLE[1]=stats_builds_per_day +TABLE[2]=stats_builds_age +RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,suite from ${TABLE[0]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\"") +if [ -z $RESULT ] ; then + ALL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(name) from sources") + GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)<='$DATE';") + GOOAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)='$DATE';") + BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)<='$DATE';") + BAAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)='$DATE';") + UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)<='$DATE';") + UGLDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)='$DATE';") + REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)<='$DATE';") + RESDAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)='$DATE';") + OLDESTG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT build_date FROM source_packages WHERE status = 'reproducible' AND NOT date(build_date)>='$DATE' ORDER BY build_date LIMIT 1;") + OLDESTB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT build_date FROM source_packages WHERE status = 'unreproducible' AND NOT date(build_date)>='$DATE' ORDER BY build_date LIMIT 1;") + OLDESTU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT build_date FROM source_packages WHERE status = 'FTBFS' AND NOT date(build_date)>='$DATE' ORDER BY build_date LIMIT 1;") + DIFFG=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTG');") + if [ -z $DIFFG ] ; then DIFFG=0 ; fi + DIFFB=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTB');") + if [ -z $DIFFB ] ; then DIFFB=0 ; fi + DIFFU=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT julianday('$DATE') - julianday('$OLDESTU');") + if [ -z $DIFFU ] ; then DIFFU=0 ; fi + let "TOTAL=GOOD+BAD+UGLY+REST" + let "UNTESTED=ALL-TOTAL" + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[0]} VALUES (\"$DATE\", \"$SUITE\", $UNTESTED, $GOOD, $BAD, $UGLY, $REST)" + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)" + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")" +fi + +redo_png() { + FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested" + FIELDS[1]="datum, reproducible, unreproducible, FTBFS, other" + FIELDS[2]="datum, oldest_reproducible, oldest_unreproducible, oldest_FTBFS" + COLOR[0]=5 + COLOR[1]=4 + COLOR[2]=3 + MAINLABEL[0]="Package reproducibility status" + MAINLABEL[1]="Amout of packages build each day" + MAINLABEL[2]="Age in days of oldest kind of logfile" + YLABEL[0]="Amount (total)" + YLABEL[1]="Amount (per day)" + YLABEL[2]="Age in days" + echo "${FIELDS[$i]}" > ${TABLE[$i]}.csv + sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$i]} from ${TABLE[$i]} WHERE suite = '$SUITE' ORDER BY datum" >> ${TABLE[$i]}.csv + /srv/jenkins/bin/make_graph.py ${TABLE[$i]}.csv ${TABLE[$i]}.png ${COLOR[$i]} "${MAINLABEL[$i]}" "${YLABEL[$i]}" + rm ${TABLE[$i]}.csv + mv ${TABLE[$i]}.png /var/lib/jenkins/userContent/ +} + +VIEW=stats +PAGE=index_${VIEW}.html +echo "Starting to write $PAGE page." +write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}" +write_page "

    $COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly.

    " +write_page "

    $COUNT_BAD packages ($PERCENT_BAD%) failed to built reproducibly.

    " +write_page "

    $COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source.

    " +if [ $COUNT_SOURCELESS -gt 0 ] ; then + write_page "

    For $COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages sources could not be downloaded," +fi +write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64' will never be build here" +write_page "and those $COUNT_BLACKLISTED blacklisted packages neither.

    " +write_page "

    " +# FIXME: we don't do stats_builds_age.png yet :/ +for i in 0 1 ; do + # redo pngs once a day + if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime -1 -name ${TABLE[$i]}.png) ] ; then + redo_png + fi + write_page " \"${MAINLABEL[$i]}\"" +done +write_page "

    " +write_page_footer +publish_page + echo "Enjoy $JENKINS_URL/userContent/reproducible.html" diff --git a/update_jdn.sh b/update_jdn.sh index 7ebb6621..f2bf6f46 100755 --- a/update_jdn.sh +++ b/update_jdn.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2012 Holger Levsen +# Copyright 2012-2014 Holger Levsen # released under the GPLv=2 BASEDIR=/root/jenkins.debian.net @@ -60,7 +60,7 @@ fi sudo apt-get install vim screen less etckeeper moreutils curl mtr-tiny dstat devscripts bash-completion shorewall shorewall6 cron-apt apt-listchanges munin munin-plugins-extra calamaris visitors procmail libjson-rpc-perl libfile-touch-perl zutils ip2host pigz \ build-essential python-setuptools \ debootstrap sudo figlet graphviz apache2 python-yaml python-pip mr subversion subversion-tools vnstat webcheck poxml vncsnapshot imagemagick ffmpeg2theora python-twisted python-imaging gocr guestmount schroot sqlite3\ - unzip python-hachoir-metadata ghc + unzip python-hachoir-metadata ghc python-rpy2 sudo apt-get install -t wheezy-backports qemu explain "Packages installed." -- cgit v1.2.3-70-g09d2