reproducible: include bugs with usertags in stats

4 files changed, 113 insertions, 19 deletions

diff --git a/bin/make_graph.py b/bin/make_graph.py
index c87bd5c0..a438820c 100755
--- a/bin/make_graph.py
+++ b/bin/make_graph.py
@@ -29,11 +29,21 @@ def main():
     r('t <- (read.table("'+countsfile+'",sep=",",header=1,row.names=1))')
     r('cname <- c("date",rep(colnames(t)))')
     # thanks to http://tango.freedesktop.org/Generic_Icon_Theme_Guidelines for those nice colors
-    r('palette(c("#4e9a06", "#f57900", "#cc0000", "#2e3436", "#888a85"))')
+    if int(colors) < 10:
+        r('palette(c("#4e9a06", "#f57900", "#cc0000", "#2e3436", "#888a85"))')
+    else:
+        r('palette(c("#fce94f", "#c4a000", "#fcaf3e", "#ce5c00", \
+                     "#e9b96e", "#8f5902", "#8ae234", "#4e9a06", \
+                     "#729fcf", "#204a87", "#ad7fa8", "#5c3566", \
+                     "#ef2929", "#a40000", "#eeeeec", "#babdb6", \
+                     "#888a85", "#2e3436"))')
     r('v <- t[0:nrow(t),0:'+colors+']')
     # make graph since day 1
     r('barplot(t(v),col = 1:'+columns+', main="'+mainlabel+'", xlab="", ylab="'+ylabel+'", space=0, border=NA)')
-    r('legend(x="bottom",legend=colnames(t), ncol=2,fill=1:'+columns+',xjust=0.5,yjust=0,bty="n")')
+    if int(colors) < 10:
+        r('legend(x="bottom",legend=colnames(t), ncol=2,fill=1:'+columns+',xjust=0.5,yjust=0,bty="n")')
+    else:
+        r('legend(x="bottom",legend=colnames(t), ncol=9,fill=1:'+columns+',xjust=0.5,yjust=0,bty="n")')
     grdevices.dev_off()
 
 if __name__ == "__main__":
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index 3e77542a..4f58569b 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -69,6 +69,28 @@ elif [ ! -f ${PACKAGES_DB} ] ; then
 		oldest_unreproducible REAL,
 		oldest_FTBFS REAL,
 		PRIMARY KEY (datum))'
+	sqlite3 ${PACKAGES_DB} '
+		CREATE TABLE stats_bugs
+		(datum TEXT NOT NULL,
+		open_toolchain INTEGER,
+		done_toolchain INTEGER,
+		open_infrastructure INTEGER,
+		done_infrastructure INTEGER,
+		open_timestamps INTEGER,
+		done_timestamps INTEGER,
+		open_fileordering INTEGER,
+		done_fileordering INTEGER,
+		open_buildpath INTEGER,
+		done_buildpath INTEGER,
+		open_username INTEGER,
+		done_username INTEGER,
+		open_hostname INTEGER,
+		done_hostname INTEGER,
+		open_uname INTEGER,
+		done_uname INTEGER,
+		open_randomness INTEGER,
+		done_randomness INTEGER,
+		PRIMARY KEY (datum))'
 	# 60 seconds timeout when trying to get a lock
 	cat >/var/lib/jenkins/reproducible.init <<-EOF
 .timeout 60000
diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh
index c739bb93..825d15ef 100755
--- a/bin/reproducible_stats.sh
+++ b/bin/reproducible_stats.sh
@@ -757,6 +757,7 @@ DATE=$(date -d "1 day ago" '+%Y-%m-%d')
 TABLE[0]=stats_pkg_state
 TABLE[1]=stats_builds_per_day
 TABLE[2]=stats_builds_age
+TABLE[3]=stats_bugs
 RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,suite from ${TABLE[0]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\"")
 if [ -z $RESULT ] ; then
 	ALL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(name) from sources")
@@ -784,26 +785,84 @@ if [ -z $RESULT ] ; then
 	sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")"
 fi
 
+# query bts
+USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness"
+RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"")
+if [ -z $RESULT ] ; then
+	declare -a DONE
+	declare -a OPEN
+	SQL="INSERT INTO ${TABLE[3]} VALUES (\"$DATE\" "
+	for TAG in $USERTAGS ; do
+		OPEN[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:open status:forwarded 2>/dev/null|wc -l)
+		DONE[$TAG]=$(bts select usertag:$TAG users:reproducible-builds@lists.alioth.debian.org status:done archive:both 2>/dev/null|wc -l)
+		# test if both values are integers
+		if ! ( [[ ${DONE[$TAG]} =~ ^-?[0-9]+$ ]] && [[ ${OPEN[$TAG]} =~ ^-?[0-9]+$ ]] ) ; then
+			echo "Non-integers value detected, exiting."
+			echo "Usertag: $TAG"
+			echo "Open: ${OPEN[$TAG]}"
+			echo "Done: ${DONE[$TAG]}"
+			exit 1
+		fi
+		SQL="$SQL, ${OPEN[$TAG]}, ${DONE[$TAG]}"
+	done
+	SQL="$SQL)"
+	echo $SQL
+	sqlite3 -init ${INIT} ${PACKAGES_DB} "$SQL"
+fi
+
+# used for redo_png (but only needed to define once)
+FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested"
+FIELDS[1]="datum, reproducible, unreproducible, FTBFS, other"
+FIELDS[2]="datum, oldest_reproducible, oldest_unreproducible, oldest_FTBFS"
+FIELDS[3]="datum "
+for TAG in $USERTAGS ; do
+	FIELDS[3]="${FIELDS[3]}, open_$TAG, done_$TAG"
+done
+COLOR[0]=5
+COLOR[1]=4
+COLOR[2]=3
+COLOR[3]=18
+MAINLABEL[0]="Package reproducibility status"
+MAINLABEL[1]="Amout of packages build each day"
+MAINLABEL[2]="Age in days of oldest kind of logfile"
+MAINLABEL[3]="Bugs with usertags for user reproducible-builds@lists.alioth.debian.org"
+YLABEL[0]="Amount (total)"
+YLABEL[1]="Amount (per day)"
+YLABEL[2]="Age in days"
+YLABEL[3]="Amount of bugs"
 redo_png() {
-	FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested"
-	FIELDS[1]="datum, reproducible, unreproducible, FTBFS, other"
-	FIELDS[2]="datum, oldest_reproducible, oldest_unreproducible, oldest_FTBFS"
-	COLOR[0]=5
-	COLOR[1]=4
-	COLOR[2]=3
-	MAINLABEL[0]="Package reproducibility status"
-	MAINLABEL[1]="Amout of packages build each day"
-	MAINLABEL[2]="Age in days of oldest kind of logfile"
-	YLABEL[0]="Amount (total)"
-	YLABEL[1]="Amount (per day)"
-	YLABEL[2]="Age in days"
 	echo "${FIELDS[$i]}" > ${TABLE[$i]}.csv
-	sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$i]} from ${TABLE[$i]} WHERE suite = '$SUITE' ORDER BY datum" >> ${TABLE[$i]}.csv
+	# TABLE[3] doesn't have a suite column...
+	if [ $i -ne 3 ] ; then
+		WHERE_SUITE="WHERE suite = '$SUITE'"
+	else
+		WHERE_SUITE=""
+	fi
+	sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$i]} from ${TABLE[$i]} ${WHERE_SUITE} ORDER BY datum" >> ${TABLE[$i]}.csv
 	/srv/jenkins/bin/make_graph.py ${TABLE[$i]}.csv ${TABLE[$i]}.png ${COLOR[$i]} "${MAINLABEL[$i]}" "${YLABEL[$i]}"
 	rm ${TABLE[$i]}.csv
 	mv ${TABLE[$i]}.png /var/lib/jenkins/userContent/
 }
 
+write_usertag_table() {
+	RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"")
+	if [ -z "$RESULTS" ] ; then
+		COUNT=0
+		for FIELD in $(echo ${FIELDS[3]} | tr -d ,) ; do
+			let "COUNT+=1"
+			VALUE=$(echo $RESULT | cut -d "|" -f$COUNT)
+			if [ $COUNT -eq 1 ] ; then
+				write_page "<table><tr><td colspan=\"4\"><em>Bugs with usertags for reproducible-builds@lists.alioth.debian.org on $VALUE</em></td></tr>"
+			elif [ $((COUNT%2)) -eq 0 ] ; then
+				write_page "<tr><td>&nbsp;</td><td><a href=\"https://bugs.debian.org/cgi-bin/pkgreport.cgi?tag=${FIELD:5};users=reproducible-builds@lists.alioth.debian.org&archive=both\">${FIELD:5}</a></td><td>Open: $VALUE</td>"
+			else
+				write_page "<td>Done: $VALUE</td></tr>"
+			fi
+		done
+		write_page "</table>"
+	fi
+}
+
 VIEW=stats
 PAGE=index_${VIEW}.html
 echo "Starting to write $PAGE page."
@@ -835,13 +894,16 @@ set_icon blacklisted
 write_icon
 write_page "$COUNT_BLACKLISTED blacklisted packages neither.</p>"
 write_page "<p>"
-# FIXME: we don't do stats_builds_age.png yet :/
-for i in 0 1 ; do
+# FIXME: we don't do 2 / stats_builds_age.png yet :/
+for i in 0 1 3 ; do
+	if [ "$i" = "3" ] ; then
+		write_usertag_table
+	fi
+	write_page " <a href=\"$JENKINS_URL/userContent/${TABLE[$i]}.png\"><img src=\"$JENKINS_URL/userContent/${TABLE[$i]}.png\" width=\"50%\" height=\"50%\" alt=\"${MAINLABEL[$i]}\"></a>"
 	# redo pngs once a day
 	if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime -1 -name ${TABLE[$i]}.png) ] ; then
 		redo_png
 	fi
-	write_page " <a href=\"$JENKINS_URL/userContent/${TABLE[$i]}.png\"><img src=\"$JENKINS_URL/userContent/${TABLE[$i]}.png\" width=\"50%\" height=\"50%\" alt=\"${MAINLABEL[$i]}\"></a>"
 done
 write_page "</p>"
 write_page_footer
diff --git a/update_jdn.sh b/update_jdn.sh
index f2bf6f46..eee5c280 100755
--- a/update_jdn.sh
+++ b/update_jdn.sh
@@ -60,7 +60,7 @@ fi
 sudo apt-get install vim screen less etckeeper moreutils curl mtr-tiny dstat devscripts bash-completion shorewall shorewall6 cron-apt apt-listchanges munin munin-plugins-extra calamaris visitors procmail libjson-rpc-perl libfile-touch-perl zutils ip2host pigz \
 	build-essential python-setuptools \
 	debootstrap sudo figlet graphviz apache2 python-yaml python-pip mr subversion subversion-tools vnstat webcheck poxml vncsnapshot imagemagick ffmpeg2theora python-twisted python-imaging gocr guestmount schroot sqlite3\
-	unzip python-hachoir-metadata ghc python-rpy2
+	unzip python-hachoir-metadata ghc python-rpy2 libsoap-lite-perl
 sudo apt-get install -t wheezy-backports qemu
 explain "Packages installed."