diff options
| -rwxr-xr-x | bin/make_graph.py | 14 | ||||
| -rwxr-xr-x | bin/reproducible_common.sh | 22 | ||||
| -rwxr-xr-x | bin/reproducible_stats.sh | 94 | ||||
| -rwxr-xr-x | update_jdn.sh | 2 |
4 files changed, 113 insertions, 19 deletions
diff --git a/bin/make_graph.py b/bin/make_graph.py index c87bd5c0..a438820c 100755 --- a/bin/make_graph.py +++ b/bin/make_graph.py @@ -29,11 +29,21 @@ def main(): r('t <- (read.table("'+countsfile+'",sep=",",header=1,row.names=1))') r('cname <- c("date",rep(colnames(t)))') # thanks to http://tango.freedesktop.org/Generic_Icon_Theme_Guidelines for those nice colors - r('palette(c("#4e9a06", "#f57900", "#cc0000", "#2e3436", "#888a85"))') + if int(colors) < 10: + r('palette(c("#4e9a06", "#f57900", "#cc0000", "#2e3436", "#888a85"))') + else: + r('palette(c("#fce94f", "#c4a000", "#fcaf3e", "#ce5c00", \ + "#e9b96e", "#8f5902", "#8ae234", "#4e9a06", \ + "#729fcf", "#204a87", "#ad7fa8", "#5c3566", \ + "#ef2929", "#a40000", "#eeeeec", "#babdb6", \ + "#888a85", "#2e3436"))') r('v <- t[0:nrow(t),0:'+colors+']') # make graph since day 1 r('barplot(t(v),col = 1:'+columns+', main="'+mainlabel+'", xlab="", ylab="'+ylabel+'", space=0, border=NA)') - r('legend(x="bottom",legend=colnames(t), ncol=2,fill=1:'+columns+',xjust=0.5,yjust=0,bty="n")') + if int(colors) < 10: + r('legend(x="bottom",legend=colnames(t), ncol=2,fill=1:'+columns+',xjust=0.5,yjust=0,bty="n")') + else: + r('legend(x="bottom",legend=colnames(t), ncol=9,fill=1:'+columns+',xjust=0.5,yjust=0,bty="n")') grdevices.dev_off() if __name__ == "__main__": diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 3e77542a..4f58569b 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -69,6 +69,28 @@ elif [ ! -f ${PACKAGES_DB} ] ; then oldest_unreproducible REAL, oldest_FTBFS REAL, PRIMARY KEY (datum))' + sqlite3 ${PACKAGES_DB} ' + CREATE TABLE stats_bugs + (datum TEXT NOT NULL, + open_toolchain INTEGER, + done_toolchain INTEGER, + open_infrastructure INTEGER, + done_infrastructure INTEGER, + open_timestamps INTEGER, + done_timestamps INTEGER, + open_fileordering INTEGER, + done_fileordering INTEGER, + open_buildpath INTEGER, + done_buildpath INTEGER, + open_username INTEGER, + done_username INTEGER, + open_hostname INTEGER, + done_hostname INTEGER, + open_uname INTEGER, + done_uname INTEGER, + open_randomness INTEGER, + done_randomness INTEGER, + PRIMARY KEY (datum))' # 60 seconds timeout when trying to get a lock cat >/var/lib/jenkins/reproducible.init <<-EOF .timeout 60000 diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh index c739bb93..825d15ef 100755 --- a/bin/reproducible_stats.sh +++ b/bin/reproducible_stats.sh @@ -757,6 +757,7 @@ DATE=$(date -d "1 day ago" '+%Y-%m-%d') TABLE[0]=stats_pkg_state TABLE[1]=stats_builds_per_day TABLE[2]=stats_builds_age +TABLE[3]=stats_bugs RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,suite from ${TABLE[0]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\"") if [ -z $RESULT ] ; then ALL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(name) from sources") @@ -784,26 +785,84 @@ if [ -z $RESULT ] ; then sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")" fi +# query bts +USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness" +RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") +if [ -z $RESULT ] ; then + declare -a DONE + declare -a OPEN + SQL="INSERT INTO ${TABLE[3]} VALUES (\"$DATE\" " + for TAG in $USERTAGS ; do + OPEN[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:open status:forwarded 2>/dev/null|wc -l) + DONE[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:done archive:both 2>/dev/null|wc -l) + # test if both values are integers + if ! ( [[ ${DONE[$TAG]} =~ ^-?[0-9]+$ ]] && [[ ${OPEN[$TAG]} =~ ^-?[0-9]+$ ]] ) ; then + echo "Non-integers value detected, exiting." + echo "Usertag: $TAG" + echo "Open: ${OPEN[$TAG]}" + echo "Done: ${DONE[$TAG]}" + exit 1 + fi + SQL="$SQL, ${OPEN[$TAG]}, ${DONE[$TAG]}" + done + SQL="$SQL)" + echo $SQL + sqlite3 -init ${INIT} ${PACKAGES_DB} "$SQL" +fi + +# used for redo_png (but only needed to define once) +FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested" +FIELDS[1]="datum, reproducible, unreproducible, FTBFS, other" +FIELDS[2]="datum, oldest_reproducible, oldest_unreproducible, oldest_FTBFS" +FIELDS[3]="datum " +for TAG in $USERTAGS ; do + FIELDS[3]="${FIELDS[3]}, open_$TAG, done_$TAG" +done +COLOR[0]=5 +COLOR[1]=4 +COLOR[2]=3 +COLOR[3]=18 +MAINLABEL[0]="Package reproducibility status" +MAINLABEL[1]="Amout of packages build each day" +MAINLABEL[2]="Age in days of oldest kind of logfile" +MAINLABEL[3]="Bugs with usertags for user reproducible-builds@lists.alioth.debian.org" +YLABEL[0]="Amount (total)" +YLABEL[1]="Amount (per day)" +YLABEL[2]="Age in days" +YLABEL[3]="Amount of bugs" redo_png() { - FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested" - FIELDS[1]="datum, reproducible, unreproducible, FTBFS, other" - FIELDS[2]="datum, oldest_reproducible, oldest_unreproducible, oldest_FTBFS" - COLOR[0]=5 - COLOR[1]=4 - COLOR[2]=3 - MAINLABEL[0]="Package reproducibility status" - MAINLABEL[1]="Amout of packages build each day" - MAINLABEL[2]="Age in days of oldest kind of logfile" - YLABEL[0]="Amount (total)" - YLABEL[1]="Amount (per day)" - YLABEL[2]="Age in days" echo "${FIELDS[$i]}" > ${TABLE[$i]}.csv - sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$i]} from ${TABLE[$i]} WHERE suite = '$SUITE' ORDER BY datum" >> ${TABLE[$i]}.csv + # TABLE[3] doesn't have a suite column... + if [ $i -ne 3 ] ; then + WHERE_SUITE="WHERE suite = '$SUITE'" + else + WHERE_SUITE="" + fi + sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$i]} from ${TABLE[$i]} ${WHERE_SUITE} ORDER BY datum" >> ${TABLE[$i]}.csv /srv/jenkins/bin/make_graph.py ${TABLE[$i]}.csv ${TABLE[$i]}.png ${COLOR[$i]} "${MAINLABEL[$i]}" "${YLABEL[$i]}" rm ${TABLE[$i]}.csv mv ${TABLE[$i]}.png /var/lib/jenkins/userContent/ } +write_usertag_table() { + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"") + if [ -z "$RESULTS" ] ; then + COUNT=0 + for FIELD in $(echo ${FIELDS[3]} | tr -d ,) ; do + let "COUNT+=1" + VALUE=$(echo $RESULT | cut -d "|" -f$COUNT) + if [ $COUNT -eq 1 ] ; then + write_page "<table><tr><td colspan=\"4\"><em>Bugs with usertags for reproducible-builds@lists.alioth.debian.org on $VALUE</em></td></tr>" + elif [ $((COUNT%2)) -eq 0 ] ; then + write_page "<tr><td> </td><td><a href=\"https://bugs.debian.org/cgi-bin/pkgreport.cgi?tag=${FIELD:5};users=reproducible-builds@lists.alioth.debian.org&archive=both\">${FIELD:5}</a></td><td>Open: $VALUE</td>" + else + write_page "<td>Done: $VALUE</td></tr>" + fi + done + write_page "</table>" + fi +} + VIEW=stats PAGE=index_${VIEW}.html echo "Starting to write $PAGE page." @@ -835,13 +894,16 @@ set_icon blacklisted write_icon write_page "$COUNT_BLACKLISTED blacklisted packages neither.</p>" write_page "<p>" -# FIXME: we don't do stats_builds_age.png yet :/ -for i in 0 1 ; do +# FIXME: we don't do 2 / stats_builds_age.png yet :/ +for i in 0 1 3 ; do + if [ "$i" = "3" ] ; then + write_usertag_table + fi + write_page " <a href=\"$JENKINS_URL/userContent/${TABLE[$i]}.png\"><img src=\"$JENKINS_URL/userContent/${TABLE[$i]}.png\" width=\"50%\" height=\"50%\" alt=\"${MAINLABEL[$i]}\"></a>" # redo pngs once a day if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime -1 -name ${TABLE[$i]}.png) ] ; then redo_png fi - write_page " <a href=\"$JENKINS_URL/userContent/${TABLE[$i]}.png\"><img src=\"$JENKINS_URL/userContent/${TABLE[$i]}.png\" width=\"50%\" height=\"50%\" alt=\"${MAINLABEL[$i]}\"></a>" done write_page "</p>" write_page_footer diff --git a/update_jdn.sh b/update_jdn.sh index f2bf6f46..eee5c280 100755 --- a/update_jdn.sh +++ b/update_jdn.sh @@ -60,7 +60,7 @@ fi sudo apt-get install vim screen less etckeeper moreutils curl mtr-tiny dstat devscripts bash-completion shorewall shorewall6 cron-apt apt-listchanges munin munin-plugins-extra calamaris visitors procmail libjson-rpc-perl libfile-touch-perl zutils ip2host pigz \ build-essential python-setuptools \ debootstrap sudo figlet graphviz apache2 python-yaml python-pip mr subversion subversion-tools vnstat webcheck poxml vncsnapshot imagemagick ffmpeg2theora python-twisted python-imaging gocr guestmount schroot sqlite3\ - unzip python-hachoir-metadata ghc python-rpy2 + unzip python-hachoir-metadata ghc python-rpy2 libsoap-lite-perl sudo apt-get install -t wheezy-backports qemu explain "Packages installed." |