reproducible: add new graph, unlinked atm: https://reproducible.debian.net/stats_pkgs_to_fix_amd64.png

3 files changed, 26 insertions, 11 deletions

diff --git a/TODO b/TODO
index 1e6beebd..3d8d612b 100644
--- a/TODO
+++ b/TODO
@@ -141,7 +141,6 @@ properties:
 ** give helmut ssh access to at least pb-node4, if he agrees to all the others too
 ** more graphs:
 *** graph average build duration by day
-*** graph packages in testing+unstable which need to be fixed
 ** notes related:
 *** #786396: classify issue by "toolchain" or "package" fix needed: show bugs which block a bug
 *** new page with annoted packages without categorized issues (and probably without bugs as only note content too, else there are too many)
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index 81d1ad7f..a63be612 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -69,7 +69,7 @@ for i in $SUITES ; do
 	mkdir -p "$BASE/$i"
 done
 
-# tables for stats
+# table names and image names
 TABLE[0]=stats_pkg_state
 TABLE[1]=stats_builds_per_day
 TABLE[2]=stats_builds_age
@@ -78,6 +78,7 @@ TABLE[4]=stats_notes
 TABLE[5]=stats_issues
 TABLE[6]=stats_meta_pkg_state
 TABLE[7]=stats_bugs_state
+TABLE[8]=stats_pkgs_to_fix
 
 # known package sets
 META_PKGSET[1]="essential"
@@ -527,6 +528,12 @@ create_png_from_table() {
 	echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv
 	# prepare query
 	WHERE_EXTRA="WHERE suite = '$SUITE'"
+	if [ "$ARCH" = "armhf" ] ; then
+		# armhf was only build since 2015-08-30
+		WHERE2_EXTRA="WHERE s.datum >= '2015-08-30'"
+	else
+		WHERE2_EXTRA=""
+	fi
 	if [ $1 -eq 3 ] || [ $1 -eq 4 ] || [ $1 -eq 5 ] ; then
 		# TABLE[3+4+5] don't have a suite column:
 		WHERE_EXTRA=""
@@ -546,14 +553,10 @@ create_png_from_table() {
 	if [ $1 -eq 1 ] ; then
 		# not sure if it's worth to generate the following query...
 		WHERE_EXTRA="AND architecture='$ARCH'"
-		if [ "$ARCH" = "armhf" ] ; then
-			# armhf was only build since 2015-08-30
-			WHERE2_EXTRA="WHERE s.datum >= '2015-08-30'"
-		fi
 		sqlite3 -init ${INIT} --nullvalue 0 -csv ${PACKAGES_DB} "SELECT s.datum,
-			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='testing' $WHERE_EXTRA),0) AS 'reproducible_testing',
-			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA),0) AS 'reproducible_unstable', 
-			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA),0) AS 'reproducible_experimental',
+			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='testing' $WHERE_EXTRA),0) AS reproducible_testing,
+			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA),0) AS reproducible_unstable,
+			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA),0) AS reproducible_experimental,
 			 (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing' $WHERE_EXTRA) AS unreproducible_testing,
 			 (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable' $WHERE_EXTRA) AS unreproducible_unstable,
 			 (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental' $WHERE_EXTRA) AS unreproducible_experimental,
@@ -569,6 +572,11 @@ create_png_from_table() {
 		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
 	elif [ $1 -eq 7 ] ; then
 		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, $SUM_DONE, $SUM_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv
+	elif [ $1 -eq 8 ] ; then
+		sqlite3 -init ${INIT} --nullvalue 0 -csv ${PACKAGES_DB} "SELECT s.datum,
+			(SELECT (e.unreproducible+e.ftbfs) FROM ${TABLE[0]} AS e WHERE s.datum=e.datum AND e.suite='unstable' AND e.architecture='$ARCH') AS unfixed_unstable,
+			COALESCE((SELECT (e.unreproducible+e.ftbfs) FROM ${TABLE[0]} AS e WHERE s.datum=e.datum AND e.suite='testing' AND e.architecture='$ARCH'),0) AS unfixed_testing
+			FROM ${TABLE[0]} AS s $WHERE2_EXTRA GROUP BY s.datum" >> ${TABLE[$1]}.csv
 	else
 		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
 	fi
diff --git a/bin/reproducible_html_dashboard.sh b/bin/reproducible_html_dashboard.sh
index aaf889e8..c8bed4ef 100755
--- a/bin/reproducible_html_dashboard.sh
+++ b/bin/reproducible_html_dashboard.sh
@@ -43,6 +43,7 @@ for TAG in $USERTAGS ; do
 done
 SUM_DONE="$SUM_DONE)"
 SUM_OPEN="$SUM_OPEN)"
+FIELDS[8]="datum, unfixed_unstable, unfixed_testing"
 COLOR[0]=5
 COLOR[1]=12
 COLOR[2]=1
@@ -50,11 +51,13 @@ COLOR[3]=32
 COLOR[4]=1
 COLOR[5]=1
 COLOR[7]=2
+COLOR[8]=2
 MAINLABEL[1]="Amount of packages built each day"
 MAINLABEL[3]="Usertags on bugs for user reproducible-builds@lists.alioth.debian.org"
 MAINLABEL[4]="Packages which have notes"
 MAINLABEL[5]="Identified issues"
 MAINLABEL[7]="Open and closed bugs"
+MAINLABEL[8]="Packages which need to be fixed"
 YLABEL[0]="Amount (total)"
 YLABEL[1]="Amount (per day)"
 YLABEL[2]="Age in days"
@@ -62,6 +65,7 @@ YLABEL[3]="Amount of bugs"
 YLABEL[4]="Amount of packages"
 YLABEL[5]="Amount of issues"
 YLABEL[7]="Amount of bugs open / closed"
+YLABEL[8]="Amount (unreproducible+ftbfs)"
 
 #
 # update package + build stats
@@ -98,7 +102,7 @@ update_suite_arch_stats() {
 		sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", \"$ARCH\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)"
 		sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$ARCH\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")"
 		# we do 3 later and 6 is special anyway...
-		for i in 0 1 2 4 5 ; do
+		for i in 0 1 2 4 5 8 ; do
 			PREFIX=""
 			if [ $i -eq 0 ] ; then
 				PREFIX=$SUITE
@@ -428,13 +432,17 @@ create_main_stats_page() {
 	# write build per day graph
 	write_page "<p style=\"clear:both;\">"
 	write_page " <a href=\"/${TABLE[1]}_$ARCH.png\"><img src=\"/${TABLE[1]}_$ARCH.png\" alt=\"${MAINLABEL[$i]}\"></a>"
-	# redo png once a day
+	# redo arch specific pngs once a day
 	for ARCH in ${ARCHS} ; do
 		if [ ! -f $BASE/${TABLE[1]}_$ARCH.png ] || [ ! -z $(find $BASE -maxdepth 1 -mtime +0 -name ${TABLE[1]}_$ARCH.png) ] ; then
 				create_png_from_table 1 ${TABLE[1]}_$ARCH.png
 		fi
 	done
 	ARCH="amd64"
+	if [ ! -f $BASE/${TABLE[8]}_$ARCH.png ] || [ ! -z $(find $BASE -maxdepth 1 -mtime +0 -name ${TABLE[8]}_$ARCH.png) ] ; then
+				# FIXME: this graph needs to be linked from somewhere…
+				create_png_from_table 8 ${TABLE[8]}_$ARCH.png
+	fi
 	# write suite builds age graphs
 	write_page "</p><p style=\"clear:both;\">"
 	for SUITE in $SUITES ; do