reproducible: proof-of-concept for graphing and displaying status of package-sets. currently there are two: required + build-essential. more suggestions welcome.

1 files changed, 60 insertions, 35 deletions

diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_graphs.sh
index 94f6637b..8d005aef 100755
--- a/bin/reproducible_html_graphs.sh
+++ b/bin/reproducible_html_graphs.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2014 Holger Levsen <holger@layer-acht.org>
+# Copyright 2014-2015 Holger Levsen <holger@layer-acht.org>
 # released under the GPLv=2
 
 DEBUG=false
@@ -10,7 +10,6 @@ common_init "$@"
 # common code defining db access
 . /srv/jenkins/bin/reproducible_common.sh
 
-set +x
 init_html
 gather_stats
 
@@ -51,8 +50,8 @@ if [ -z $RESULT ] ; then
 	sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[0]} VALUES (\"$DATE\", \"$SUITE\", $UNTESTED, $GOOD, $BAD, $UGLY, $REST)" 
 	sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)"
 	sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")"
-	# FIXME: we don't do 2 / stats_builds_age.png yet :/ (and do 3 later)
-	for i in 0 1 4 5 6 ; do
+	# FIXME: we don't do 2 / stats_builds_age.png yet :/ (and do 3 later) and 6 is special anyway
+	for i in 0 1 4 5 ; do
 		# force regeneration of the image
 		touch -d "$DATE 00:00" ${TABLE[$i]}.png
 	done
@@ -65,28 +64,34 @@ if [ -z $RESULT ] ; then
 fi
 
 # FIXME: work in progress: meta package state graphs
-META_PKG="required"
-RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"$META_PKG\"")
-if [ -z $RESULT ] ; then
-	META_TOTAL=0
-	META_REQUIRED="acl attr base-files base-passwd bash coreutils dash debconf debianutils diffutils dpkg e2fsprogs eglibc findutils gcc-4.7 grep gzip hostname liblocale-gettext-perl libselinux libsepol libtext-charwidth-perl libtext-iconv-perl libtext-wrapi18n-perl lsb mawk ncurses pam perl sed sensible-utils shadow sysvinit tar tzdata util-linux xz-utils zlib"
-	META_WHERE=""
-	for i in $META_REQUIRED ; do
-		if [ -z "$META_WHERE" ] ; then
-			META_WHERE="name in ('$i'"
-		else
-			META_WHERE="$META_WHERE, '$i'"
-		fi
-		let "META_TOTAL=META_TOTAL+1"
-	done
-	META_WHERE="$META_WHERE)"
-	META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
-	META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
-	META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)<='$DATE' AND $META_WHERE;")
-	META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)<='$DATE' AND $META_WHERE;")
-	let "META_UNTESTED=META_TOTAL-META_GOOD-META_BAD-META_UGLY-META-REST"
-	sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"$META_PKG\", $META_UNTESTED, $META_GOOD, $META_BAD, $META_UGLY, $META_REST)" 
-fi
+META_PKG[1]="required"
+META_PKG[2]="build-essential"
+META_LIST[1]="acl attr base-files base-passwd bash coreutils dash debconf debianutils diffutils dpkg e2fsprogs eglibc findutils gcc-4.7 grep gzip hostname liblocale-gettext-perl libselinux libsepol libtext-charwidth-perl libtext-iconv-perl libtext-wrapi18n-perl lsb mawk ncurses pam perl sed sensible-utils shadow sysvinit tar tzdata util-linux xz-utils zlib"
+META_LIST[2]="binutils build-essential dpkg eglibc gcc-defaults make-dfsg patch perl"
+
+for i in $(seq 1 ${#META_PKG[@]}) ; do
+	RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"${META_PKG[$i]}\"")
+	if [ -z $RESULT ] ; then
+		META_TOTAL=0
+		META_WHERE=""
+		for PKG in ${META_LIST[$i]} ; do
+			if [ -z "$META_WHERE" ] ; then
+				META_WHERE="name in ('$PKG'"
+			else
+				META_WHERE="$META_WHERE, '$PKG'"
+			fi
+			let "META_TOTAL=META_TOTAL+1"
+		done
+		META_WHERE="$META_WHERE)"
+		META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
+		META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
+		META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)<='$DATE' AND $META_WHERE;")
+		META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)<='$DATE' AND $META_WHERE;")
+		let "META_UNTESTED=META_TOTAL-META_GOOD-META_BAD-META_UGLY-META-REST"
+		sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKG[$i]}\", $META_UNTESTED, $META_GOOD, $META_BAD, $META_UGLY, $META_REST)" 
+	fi
+	# FIXME: touch -d to force re-creation is missing...
+done
 
 # query bts
 USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness"
@@ -139,30 +144,28 @@ MAINLABEL[2]="Age in days of oldest kind of logfile"
 MAINLABEL[3]="Bugs with usertags for user reproducible-builds@lists.alioth.debian.org"
 MAINLABEL[4]="Packages which have notes"
 MAINLABEL[5]="Identified issues"
-MAINLABEL[6]="Package reproducibility status for $META_PKG packages"
 YLABEL[0]="Amount (total)"
 YLABEL[1]="Amount (per day)"
 YLABEL[2]="Age in days"
 YLABEL[3]="Amount of bugs"
 YLABEL[4]="Amount of packages"
 YLABEL[5]="Amount of issues"
-YLABEL[6]="Amount ($META_PKG packages)"
 
 redo_png() {
 	echo "${FIELDS[$i]}" > ${TABLE[$i]}.csv
 	# TABLE[3+4+5] don't have a suite column...
 	# 6 is special anyway
 	if [ $i -eq 6 ] ; then
-		WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$META_PKG'"
+		WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$2'"
 	elif [ $i -ne 3 ] && [ $i -ne 4 ] && [ $i -ne 5 ] ; then
 		WHERE_EXTRA="WHERE suite = '$SUITE'"
 	else
 		WHERE_EXTRA=""
 	fi
 	sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$i]} from ${TABLE[$i]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$i]}.csv
-	/srv/jenkins/bin/make_graph.py ${TABLE[$i]}.csv ${TABLE[$i]}.png ${COLOR[$i]} "${MAINLABEL[$i]}" "${YLABEL[$i]}"
+	/srv/jenkins/bin/make_graph.py ${TABLE[$i]}.csv $1 ${COLOR[$i]} "${MAINLABEL[$i]}" "${YLABEL[$i]}"
 	rm ${TABLE[$i]}.csv
-	mv ${TABLE[$i]}.png /var/lib/jenkins/userContent/
+	mv $1 /var/lib/jenkins/userContent/
 }
 
 write_usertag_table() {
@@ -220,10 +223,32 @@ for i in 0 3 4 5 6 1 ; do
 	if [ "$i" = "3" ] ; then
 		write_usertag_table
 	fi
-	write_page " <a href=\"/userContent/${TABLE[$i]}.png\"><img src=\"/userContent/${TABLE[$i]}.png\" class=\"graph\" alt=\"${MAINLABEL[$i]}\"></a>"
-	# redo pngs once a day
-	if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then
-		redo_png
+	# FIXME: split this out in html_meta_graphs... really.
+	if [ "$i" = "6" ] ; then
+		# FIXME: THIS IS A MESS
+		for j in $(seq 1 ${#META_PKG[@]}) ; do
+			MAINLABEL[6]="Package reproducibility status for ${META_PKG[$j]} packages"
+			YLABEL[6]="Amount (${META_PKG[$j]} packages)"
+			PNG=${TABLE[$i]}_${META_PKG[$j]}.png
+			write_page " <div>"
+			write_page " <a href=\"/userContent/$PNG\"><img src=\"/userContent/$PNG\" class=\"graph\" alt=\"${MAINLABEL[$i]}\"></a>"
+			write_page " <br />The package set '${META_PKG[$j]}' consists of: "
+			# FIXME: split into good/bad/ugly too
+			force_package_targets ${META_LIST[$j]}
+			link_packages ${META_LIST[$j]}
+			write_page " </div>"
+			# redo pngs once a day
+			if [ ! -f /var/lib/jenkins/userContent/$PNG ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name $PNG) ] ; then
+				# FIXME: call redo_png differently here.. sux
+				redo_png $PNG ${META_PKG[$j]}
+			fi
+		done
+	else
+		write_page " <a href=\"/userContent/${TABLE[$i]}.png\"><img src=\"/userContent/${TABLE[$i]}.png\" class=\"graph\" alt=\"${MAINLABEL[$i]}\"></a>"
+		# redo pngs once a day
+		if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then
+			redo_png ${TABLE[$i]}.png
+		fi
 	fi
 done
 write_page "</p>"