reproducible: split _html_graphs.sh into _dashboards.sh and _pkg_sets.sh

5 files changed, 292 insertions, 251 deletions

diff --git a/TODO b/TODO
index 4e10ef18..7cf330d3 100644
--- a/TODO
+++ b/TODO
@@ -180,7 +180,6 @@ properties:
 ** fix https://jenkins.debian.net/munin/debian.net/jenkins.debian.net/jenkins_builds.html which is broken since jessie upgrade
 
 * lesser prio
-** html_graphs.sh rewrite: split out html_pkg_sets.py first, thats the easiest and the biggest speed gain
 ** more graphs:
 *** graph average build duration by day
 *** graph packages in testing+unstable which need to be fixed
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index 7d73afdb..361b231f 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -425,3 +425,83 @@ get_filesize() {
 			SIZE="$BYTESIZE bytes"
 		fi
 }
+
+#
+# create the png (and query the db to populate a csv file...)
+#
+create_png_from_table() {
+	echo "Checking whether to update $2..."
+	# $1 = id of the stats table
+	# $2 = image file name
+	# $3 = meta package set, only sensible if $1=6
+	echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv
+	# prepare query
+	WHERE_EXTRA="WHERE suite = '$SUITE'"
+	if [ $1 -eq 3 ] || [ $1 -eq 4 ] || [ $1 -eq 5 ] ; then
+		# TABLE[3+4+5] don't have a suite column:
+		WHERE_EXTRA=""
+	elif [ $1 -eq 6 ] ; then
+		# 6 is special too:
+		WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$3'"
+	fi
+	# run query
+	if [ $1 -eq 1 ] ; then
+		# not sure if it's worth to generate the following query...
+		sqlite3 -init ${INIT} --nullvalue 0 -csv ${PACKAGES_DB} "SELECT s.datum,
+			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='testing'),0) as 'reproducible_testing',
+			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='unstable'),0) as 'reproducible_unstable', 
+			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='experimental'),0) as 'reproducible_experimental',
+			 (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS unreproducible_testing,
+			 (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS unreproducible_unstable,
+			 (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS unreproducible_experimental,
+			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS FTBFS_testing,
+			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS FTBFS_unstable,
+			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS FTBFS_experimental,
+			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS other_testing,
+			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS other_unstable,
+			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS other_experimental
+			 FROM stats_builds_per_day AS s GROUP BY s.datum" >> ${TABLE[$1]}.csv
+	elif [ $1 -eq 2 ] ; then
+		# just make a graph of the oldest reproducible build (ignore FTBFS and unreproducible)
+		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
+	elif [ $1 -eq 7 ] ; then
+		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, $SUM_DONE, $SUM_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv
+	else
+		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
+	fi
+	# this is a gross hack: normally we take the number of colors a table should have...
+	#  for the builds_age table we only want one color, but different ones, so this hack:
+	COLORS=${COLOR[$1]}
+	if [ $1 -eq 2 ] ; then
+		case "$SUITE" in
+			testing)	COLORS=40 ;;
+			unstable)	COLORS=41 ;;
+			experimental)	COLORS=42 ;;
+		esac
+	fi
+	# only generate graph if the query returned data
+	if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then
+		echo "Updating $2..."
+		DIR=$(dirname $2)
+		mkdir -p $DIR
+		echo "Generating $2."
+		/srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}"
+		mv $2 $BASE/$DIR
+		[ "$DIR" = "." ] || rmdir $(dirname $2)
+	# create empty dummy png if there havent been any results ever
+	elif [ ! -f $BASE/$DIR/$(basename $2) ] ; then
+		DIR=$(dirname $2)
+		mkdir -p $DIR
+		echo "Creating $2 dummy."
+		convert -size 1920x960 xc:#aaaaaa -depth 8 $2
+		if [ "$3" != "" ] ; then
+			local THUMB="${TABLE[1]}_${3}-thumbnail.png"
+			convert $2 -adaptive-resize 160x80 ${THUMB}
+			mv ${THUMB} $BASE/$DIR
+		fi
+		mv $2 $BASE/$DIR
+		[ "$DIR" = "." ] || rmdir $(dirname $2)
+	fi
+	rm ${TABLE[$1]}.csv
+}
+
diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_dashboards.sh
index 2dc80128..31ba1fcc 100755
--- a/bin/reproducible_html_graphs.sh
+++ b/bin/reproducible_html_dashboards.sh
@@ -27,7 +27,7 @@ TABLE[2]=stats_builds_age
 TABLE[3]=stats_bugs
 TABLE[4]=stats_notes
 TABLE[5]=stats_issues
-TABLE[6]=stats_meta_pkg_state
+#TABLE[6]
 TABLE[7]=stats_bugs_state
 FIELDS[0]="datum, reproducible, unreproducible, FTBFS, other, untested"
 FIELDS[1]="datum"
@@ -43,7 +43,7 @@ for TAG in $USERTAGS ; do
 done
 FIELDS[4]="datum, packages_with_notes"
 FIELDS[5]="datum, known_issues"
-FIELDS[6]="datum, reproducible, unreproducible, FTBFS, other"
+#FIELDS[6]
 FIELDS[7]="datum, done_bugs, open_bugs"
 SUM_DONE="(0"
 SUM_OPEN="(0"
@@ -59,7 +59,7 @@ COLOR[2]=1
 COLOR[3]=32
 COLOR[4]=1
 COLOR[5]=1
-COLOR[6]=4
+#COLOR[6]
 COLOR[7]=2
 MAINLABEL[1]="Amount of packages built each day"
 MAINLABEL[3]="Usertags on bugs for user reproducible-builds@lists.alioth.debian.org"
@@ -174,69 +174,6 @@ gather_suite_stats() {
 }
 
 #
-# gather meta pkg stats
-#
-gather_meta_stats() {
-	PKGSET_PATH=/srv/reproducible-results/meta_pkgsets-$SUITE/${META_PKGSET[$1]}.pkgset
-	if [ -f $PKGSET_PATH ] ; then
-		META_LIST=$(cat $PKGSET_PATH)
-		if [ ! -z "$META_LIST" ] ; then
-			META_WHERE=""
-			# gather data about all packages we know about
-			# as a result, unknown packages in the package set
-			# are silently ignored
-			set +x
-			for PKG in $META_LIST ; do
-				if [ -z "$META_WHERE" ] ; then
-					META_WHERE="s.name in ('$PKG'"
-				else
-					META_WHERE="$META_WHERE, '$PKG'"
-				fi
-			done
-			if "$DEBUG" ; then set -x ; fi
-			META_WHERE="$META_WHERE)"
-		else
-			META_WHERE="name = 'meta-name-does-not-exist'"
-		fi
-		COUNT_META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;")
-		COUNT_META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;")
-		COUNT_META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE;")
-		COUNT_META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE;")
-		let META_ALL=COUNT_META_GOOD+COUNT_META_BAD+COUNT_META_UGLY+COUNT_META_REST || META_ALL=1
-		PERCENT_META_GOOD=$(echo "scale=1 ; ($COUNT_META_GOOD*100/$META_ALL)" | bc)
-		PERCENT_META_BAD=$(echo "scale=1 ; ($COUNT_META_BAD*100/$META_ALL)" | bc)
-		PERCENT_META_UGLY=$(echo "scale=1 ; ($COUNT_META_UGLY*100/$META_ALL)" | bc)
-		PERCENT_META_REST=$(echo "scale=1 ; ($COUNT_META_REST*100/$META_ALL)" | bc)
-		# order reproducible packages by name, the rest by build_date
-		META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY s.name;")
-		META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;")
-		META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;")
-		META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name AS NAME FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;")
-	else
-		META_RESULT=false
-	fi
-}
-
-#
-# update meta pkg stats
-#
-update_meta_pkg_stats() {
-	for i in $(seq 1 ${#META_PKGSET[@]}) ; do
-		RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"${META_PKGSET[$i]}\"")
-		if [ -z $RESULT ] ; then
-			META_RESULT=true
-			gather_meta_stats $i
-			if $META_RESULT ; then
-				 sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)"
-				echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE on $DATE."
-			fi
-			echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..."
-			touch -d "$FORCE_DATE 00:00" $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png
-		fi
-	done
-}
-
-#
 # update bug stats
 #
 update_bug_stats() {
@@ -277,85 +214,6 @@ update_bug_stats() {
 }
 
 #
-# create the png (and query the db to populate a csv file...)
-#
-create_png_from_table() {
-	echo "Checking whether to update $2..."
-	# $1 = id of the stats table
-	# $2 = image file name
-	# $3 = meta package set, only sensible if $1=6
-	echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv
-	# prepare query
-	WHERE_EXTRA="WHERE suite = '$SUITE'"
-	if [ $1 -eq 3 ] || [ $1 -eq 4 ] || [ $1 -eq 5 ] ; then
-		# TABLE[3+4+5] don't have a suite column:
-		WHERE_EXTRA=""
-	elif [ $1 -eq 6 ] ; then
-		# 6 is special too:
-		WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$3'"
-	fi
-	# run query
-	if [ $1 -eq 1 ] ; then
-		# not sure if it's worth to generate the following query...
-		sqlite3 -init ${INIT} --nullvalue 0 -csv ${PACKAGES_DB} "SELECT s.datum,
-			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='testing'),0) as 'reproducible_testing',
-			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='unstable'),0) as 'reproducible_unstable', 
-			 COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='experimental'),0) as 'reproducible_experimental',
-			 (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS unreproducible_testing,
-			 (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS unreproducible_unstable,
-			 (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS unreproducible_experimental,
-			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS FTBFS_testing,
-			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS FTBFS_unstable,
-			 (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS FTBFS_experimental,
-			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS other_testing,
-			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS other_unstable,
-			 (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS other_experimental
-			 FROM stats_builds_per_day AS s GROUP BY s.datum" >> ${TABLE[$1]}.csv
-	elif [ $1 -eq 2 ] ; then
-		# just make a graph of the oldest reproducible build (ignore FTBFS and unreproducible)
-		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
-	elif [ $1 -eq 7 ] ; then
-		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, $SUM_DONE, $SUM_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv
-	else
-		sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
-	fi
-	# this is a gross hack: normally we take the number of colors a table should have...
-	#  for the builds_age table we only want one color, but different ones, so this hack:
-	COLORS=${COLOR[$1]}
-	if [ $1 -eq 2 ] ; then
-		case "$SUITE" in
-			testing)	COLORS=40 ;;
-			unstable)	COLORS=41 ;;
-			experimental)	COLORS=42 ;;
-		esac
-	fi
-	# only generate graph if the query returned data
-	if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then
-		echo "Updating $2..."
-		DIR=$(dirname $2)
-		mkdir -p $DIR
-		echo "Generating $2."
-		/srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}"
-		mv $2 $BASE/$DIR
-		[ "$DIR" = "." ] || rmdir $(dirname $2)
-	# create empty dummy png if there havent been any results ever
-	elif [ ! -f $BASE/$DIR/$(basename $2) ] ; then
-		DIR=$(dirname $2)
-		mkdir -p $DIR
-		echo "Creating $2 dummy."
-		convert -size 1920x960 xc:#aaaaaa -depth 8 $2
-		if [ "$3" != "" ] ; then
-			local THUMB="${TABLE[1]}_${3}-thumbnail.png"
-			convert $2 -adaptive-resize 160x80 ${THUMB}
-			mv ${THUMB} $BASE/$DIR
-		fi
-		mv $2 $BASE/$DIR
-		[ "$DIR" = "." ] || rmdir $(dirname $2)
-	fi
-	rm ${TABLE[$1]}.csv
-}
-
-#
 # gather bugs stats and generate html table
 #
 write_usertag_table() {
@@ -436,101 +294,6 @@ create_suite_stats_page() {
 }
 
 #
-# create pkg set navigation
-#
-create_pkg_sets_navigation() {
-	local i
-	write_page "<ul><li>Tracked package sets in $SUITE: </li>"
-	for i in $(seq 1 ${#META_PKGSET[@]}) ; do
-		if [ -f $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png ] ; then
-			THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png"
-			LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'"
-			write_page "<a href=\"/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"><img src=\"/userContent/$SUITE/$ARCH/$THUMB\" class=\"metaoverview\" alt=\"$LABEL\" title=\"${META_PKGSET[$i]}\" name=\"${META_PKGSET[$i]}\"></a>"
-			write_page "<li>"
-			write_page "<a href=\"/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\">${META_PKGSET[$i]}</a>"
-			write_page "</li>"
-		fi
-	done
-	write_page "</ul>"
-}
-
-#
-# create pkg sets pages
-#
-create_pkg_sets_pages() {
-	#
-	# create index page
-	#
-	VIEW=pkg_sets
-	PAGE=index_${VIEW}.html
-	echo "$(date) - starting to write $PAGE page."
-	write_page_header $VIEW "Overview about reproducible builds of specific package sets in $SUITE/$ARCH"
-	create_pkg_sets_navigation
-	write_page_footer
-	publish_page $SUITE/$ARCH
-	#
-	# create individual pages for all the sets
-	#
-	local i
-	for i in $(seq 1 ${#META_PKGSET[@]}) ; do
-		PAGE="pkg_set_${META_PKGSET[$i]}.html"
-		echo "$(date) - starting to write $PAGE page."
-		write_page_header $VIEW "Overview about reproducible builds for the ${META_PKGSET[$i]} package set in $SUITE/$ARCH"
-		create_pkg_sets_navigation
-		write_page "<hr />"
-		META_RESULT=true
-		gather_meta_stats $i
-		if $META_RESULT ; then
-			MAINLABEL[6]="Reproducibility status for packages in $SUITE from '${META_PKGSET[$i]}'"
-			YLABEL[6]="Amount (${META_PKGSET[$i]} packages)"
-			PNG=${TABLE[6]}_${META_PKGSET[$i]}.png
-			THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png"
-			# redo pngs once a day
-			if [ ! -f $BASE/$SUITE/$ARCH/$PNG ] || [ ! -z $(find $BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then
-				create_png_from_table 6 $SUITE/$ARCH/$PNG ${META_PKGSET[$i]}
-				convert $BASE/$SUITE/$ARCH/$PNG -adaptive-resize 160x80 $BASE/$SUITE/$ARCH/$THUMB
-			fi
-			LABEL="package set '${META_PKGSET[$j]}' in $SUITE/$ARCH"
-			write_page "<p><a href=\"/userContent/$SUITE/$ARCH/$PNG\"><img src=\"/userContent/$SUITE/$ARCH/$PNG\" class=\"overview\" alt=\"$LABEL\"></a>"
-			write_page "<br />The package set '${META_PKGSET[$i]}' in $SUITE/$ARCH consists of: <br />&nbsp;<br />"
-			set_icon unreproducible
-			write_icon
-			write_page "$COUNT_META_BAD ($PERCENT_META_BAD%) packages failed to build reproducibly:"
-			link_packages $META_BAD
-			write_page "<br />"
-			if [ $COUNT_META_UGLY -gt 0 ] ; then
-				set_icon FTBFS
-				write_icon
-				write_page "$COUNT_META_UGLY ($PERCENT_META_UGLY%) packages failed to build from source:"
-				link_packages $META_UGLY
-				write_page "<br />"
-			fi
-			if [ $COUNT_META_REST -gt 0 ] ; then
-				set_icon not_for_us
-				write_icon
-				set_icon blacklisted
-				write_icon
-				set_icon 404
-				write_icon
-				write_page "$COUNT_META_REST ($PERCENT_META_REST%) packages are either blacklisted, not for us or cannot be downloaded:"
-				link_packages $META_REST
-				write_page "<br />"
-			fi
-			write_page "&nbsp;<br />"
-			set_icon reproducible
-			write_icon
-			write_page "$COUNT_META_GOOD packages ($PERCENT_META_GOOD%) successfully built reproducibly:"
-			link_packages $META_GOOD
-			write_page "<br />"
-			write_page "</p>"
-			write_page_meta_sign
-		fi
-		write_page_footer
-		publish_page $SUITE/$ARCH
-	done
-}
-
-#
 # create main stats page
 #
 create_main_stats_page() {
@@ -661,12 +424,6 @@ for SUITE in $SUITES ; do
 	update_suite_stats
 	gather_suite_stats
 	create_suite_stats_page
-	if [ "$SUITE" = "experimental" ] ; then
-		# no pkg sets in experimental
-		continue
-	fi
-	update_meta_pkg_stats
-	create_pkg_sets_pages
 done
 SUITE="unstable"
 create_main_stats_page
diff --git a/bin/reproducible_html_pkg_sets.sh b/bin/reproducible_html_pkg_sets.sh
new file mode 100755
index 00000000..8196d3eb
--- /dev/null
+++ b/bin/reproducible_html_pkg_sets.sh
@@ -0,0 +1,196 @@
+#!/bin/bash
+
+# Copyright 2014-2015 Holger Levsen <holger@layer-acht.org>
+#         © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# released under the GPLv=2
+
+DEBUG=false
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code defining db access
+. /srv/jenkins/bin/reproducible_common.sh
+
+#
+# init some variables
+#
+ARCH="amd64"  # we only care about amd64 status here (for now)
+# we only do stats up until yesterday... we also could do today too but not update the db yet...
+DATE=$(date -d "1 day ago" '+%Y-%m-%d')
+FORCE_DATE=$(date -d "2 day ago" '+%Y-%m-%d')
+
+# variables related to the stats we do
+TABLE[6]=stats_meta_pkg_state
+FIELDS[6]="datum, reproducible, unreproducible, FTBFS, other"
+COLOR[6]=4
+
+#
+# gather meta pkg stats
+#
+gather_meta_stats() {
+	PKGSET_PATH=/srv/reproducible-results/meta_pkgsets-$SUITE/${META_PKGSET[$1]}.pkgset
+	if [ -f $PKGSET_PATH ] ; then
+		META_LIST=$(cat $PKGSET_PATH)
+		if [ ! -z "$META_LIST" ] ; then
+			META_WHERE=""
+			# gather data about all packages we know about
+			# as a result, unknown packages in the package set
+			# are silently ignored
+			set +x
+			for PKG in $META_LIST ; do
+				if [ -z "$META_WHERE" ] ; then
+					META_WHERE="s.name in ('$PKG'"
+				else
+					META_WHERE="$META_WHERE, '$PKG'"
+				fi
+			done
+			if "$DEBUG" ; then set -x ; fi
+			META_WHERE="$META_WHERE)"
+		else
+			META_WHERE="name = 'meta-name-does-not-exist'"
+		fi
+		COUNT_META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;")
+		COUNT_META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;")
+		COUNT_META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE;")
+		COUNT_META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE;")
+		let META_ALL=COUNT_META_GOOD+COUNT_META_BAD+COUNT_META_UGLY+COUNT_META_REST || META_ALL=1
+		PERCENT_META_GOOD=$(echo "scale=1 ; ($COUNT_META_GOOD*100/$META_ALL)" | bc)
+		PERCENT_META_BAD=$(echo "scale=1 ; ($COUNT_META_BAD*100/$META_ALL)" | bc)
+		PERCENT_META_UGLY=$(echo "scale=1 ; ($COUNT_META_UGLY*100/$META_ALL)" | bc)
+		PERCENT_META_REST=$(echo "scale=1 ; ($COUNT_META_REST*100/$META_ALL)" | bc)
+		# order reproducible packages by name, the rest by build_date
+		META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY s.name;")
+		META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;")
+		META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;")
+		META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name AS NAME FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;")
+	else
+		META_RESULT=false
+	fi
+}
+
+#
+# update meta pkg stats
+#
+update_meta_pkg_stats() {
+	for i in $(seq 1 ${#META_PKGSET[@]}) ; do
+		RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"${META_PKGSET[$i]}\"")
+		if [ -z $RESULT ] ; then
+			META_RESULT=true
+			gather_meta_stats $i
+			if $META_RESULT ; then
+				 sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)"
+				echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE on $DATE."
+			fi
+			echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..."
+			touch -d "$FORCE_DATE 00:00" $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png
+		fi
+	done
+}
+
+#
+# create pkg set navigation
+#
+create_pkg_sets_navigation() {
+	local i
+	write_page "<ul><li>Tracked package sets in $SUITE: </li>"
+	for i in $(seq 1 ${#META_PKGSET[@]}) ; do
+		if [ -f $BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png ] ; then
+			THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png"
+			LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'"
+			write_page "<a href=\"/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"><img src=\"/userContent/$SUITE/$ARCH/$THUMB\" class=\"metaoverview\" alt=\"$LABEL\" title=\"${META_PKGSET[$i]}\" name=\"${META_PKGSET[$i]}\"></a>"
+			write_page "<li>"
+			write_page "<a href=\"/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\">${META_PKGSET[$i]}</a>"
+			write_page "</li>"
+		fi
+	done
+	write_page "</ul>"
+}
+
+#
+# create pkg sets pages
+#
+create_pkg_sets_pages() {
+	#
+	# create index page
+	#
+	VIEW=pkg_sets
+	PAGE=index_${VIEW}.html
+	echo "$(date) - starting to write $PAGE page."
+	write_page_header $VIEW "Overview about reproducible builds of specific package sets in $SUITE/$ARCH"
+	create_pkg_sets_navigation
+	write_page_footer
+	publish_page $SUITE/$ARCH
+	#
+	# create individual pages for all the sets
+	#
+	local i
+	for i in $(seq 1 ${#META_PKGSET[@]}) ; do
+		PAGE="pkg_set_${META_PKGSET[$i]}.html"
+		echo "$(date) - starting to write $PAGE page."
+		write_page_header $VIEW "Overview about reproducible builds for the ${META_PKGSET[$i]} package set in $SUITE/$ARCH"
+		create_pkg_sets_navigation
+		write_page "<hr />"
+		META_RESULT=true
+		gather_meta_stats $i
+		if $META_RESULT ; then
+			MAINLABEL[6]="Reproducibility status for packages in $SUITE from '${META_PKGSET[$i]}'"
+			YLABEL[6]="Amount (${META_PKGSET[$i]} packages)"
+			PNG=${TABLE[6]}_${META_PKGSET[$i]}.png
+			THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png"
+			# redo pngs once a day
+			if [ ! -f $BASE/$SUITE/$ARCH/$PNG ] || [ ! -z $(find $BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then
+				create_png_from_table 6 $SUITE/$ARCH/$PNG ${META_PKGSET[$i]}
+				convert $BASE/$SUITE/$ARCH/$PNG -adaptive-resize 160x80 $BASE/$SUITE/$ARCH/$THUMB
+			fi
+			LABEL="package set '${META_PKGSET[$j]}' in $SUITE/$ARCH"
+			write_page "<p><a href=\"/userContent/$SUITE/$ARCH/$PNG\"><img src=\"/userContent/$SUITE/$ARCH/$PNG\" class=\"overview\" alt=\"$LABEL\"></a>"
+			write_page "<br />The package set '${META_PKGSET[$i]}' in $SUITE/$ARCH consists of: <br />&nbsp;<br />"
+			set_icon unreproducible
+			write_icon
+			write_page "$COUNT_META_BAD ($PERCENT_META_BAD%) packages failed to build reproducibly:"
+			link_packages $META_BAD
+			write_page "<br />"
+			if [ $COUNT_META_UGLY -gt 0 ] ; then
+				set_icon FTBFS
+				write_icon
+				write_page "$COUNT_META_UGLY ($PERCENT_META_UGLY%) packages failed to build from source:"
+				link_packages $META_UGLY
+				write_page "<br />"
+			fi
+			if [ $COUNT_META_REST -gt 0 ] ; then
+				set_icon not_for_us
+				write_icon
+				set_icon blacklisted
+				write_icon
+				set_icon 404
+				write_icon
+				write_page "$COUNT_META_REST ($PERCENT_META_REST%) packages are either blacklisted, not for us or cannot be downloaded:"
+				link_packages $META_REST
+				write_page "<br />"
+			fi
+			write_page "&nbsp;<br />"
+			set_icon reproducible
+			write_icon
+			write_page "$COUNT_META_GOOD packages ($PERCENT_META_GOOD%) successfully built reproducibly:"
+			link_packages $META_GOOD
+			write_page "<br />"
+			write_page "</p>"
+			write_page_meta_sign
+		fi
+		write_page_footer
+		publish_page $SUITE/$ARCH
+	done
+}
+
+#
+# main
+#
+for SUITE in $SUITES ; do
+	if [ "$SUITE" = "experimental" ] ; then
+		# no pkg sets in experimental
+		continue
+	fi
+	update_meta_pkg_stats
+	create_pkg_sets_pages
+done
+
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index 2f8eafa6..f03f3484 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -203,7 +203,11 @@
 
 - job-template:
     defaults: reproducible
-    name: '{name}_html_graphs'
+    name: '{name}_html_dashboards'
+
+- job-template:
+    defaults: reproducible
+    name: '{name}_html_pkg_sets'
 
 - job-template:
     defaults: reproducible
@@ -338,10 +342,15 @@
             my_timed: '42 * * * *'
             my_shell: '/srv/jenkins/bin/reproducible_scheduler.py'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
-        - '{name}_html_graphs':
-            my_description: 'Generate HTML results (stats with graphs) for reproducible builds.'
+        - '{name}_html_dashboards':
+            my_description: 'Generate HTML results (dashboards with graphs) for reproducible builds.'
+            my_timed: '*/15 * * * *'
+            my_shell: '/srv/jenkins/bin/reproducible_html_dashboards.sh'
+            my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
+        - '{name}_html_pkg_sets':
+            my_description: 'Generate HTML results (package sets with graphs) for reproducible builds.'
             my_timed: '0 * * * *'
-            my_shell: '/srv/jenkins/bin/reproducible_html_graphs.sh'
+            my_shell: '/srv/jenkins/bin/reproducible_html_pkg_sets.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
         - '{name}_html_indexes':
             my_description: 'Generate HTML results (indexes) for reproducible builds.'