reproducible: create new page: https://reproducible.debian.net/userContent/index_pkg_sets.html

2 files changed, 92 insertions, 53 deletions

diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index d571c0c2..d505b6f9 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -149,7 +149,7 @@ init_html() {
 	SUITE=sid
 	MAINVIEW="stats"
 	ALLSTATES="reproducible FTBR_with_buildinfo FTBR FTBFS 404 not_for_us blacklisted"
-	ALLVIEWS="issues notes scheduled last_24h last_48h all_abc dd-list stats"
+	ALLVIEWS="issues notes scheduled last_24h last_48h all_abc dd-list stats pkg_sets"
 	SPOKENTARGET["reproducible"]="packages which built reproducibly"
 	SPOKENTARGET["FTBR"]="packages which failed to build reproducibly and do not create a .buildinfo file"
 	SPOKENTARGET["FTBR_with_buildinfo"]="packages which failed to build reproducibly and create a .buildinfo file"
@@ -165,6 +165,7 @@ init_html() {
 	SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)"
 	SPOKENTARGET["dd-list"]="maintainers of unreproducible packages"
 	SPOKENTARGET["stats"]="various statistics about reproducible builds"
+	SPOKENTARGET["pkg_sets"]="statistics about reproducible builds of specific package sets"
 	# query some data we need everywhere
 	AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(name) FROM sources")
 	COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages")
@@ -243,6 +244,8 @@ write_page_header() {
 			SPOKEN_TARGET=$TARGET
 		elif [ "$TARGET" = "scheduled" ] ; then
 			SPOKEN_TARGET="currently scheduled"
+		elif [ "$TARGET" = "pkg_sets" ] ; then
+			SPOKEN_TARGET="package sets stats"
 		else
 			SPOKEN_TARGET=${SPOKENTARGET[$TARGET]}
 		fi
@@ -395,10 +398,6 @@ process_packages() {
 gather_schedule_stats() {
 	SCHEDULED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM sources_scheduled ORDER BY date_scheduled" | xargs echo)
 	COUNT_SCHEDULED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(name) FROM sources_scheduled" | xargs echo)
-	let "COUNT_NOTYET=AMOUNT-COUNT_TOTAL-COUNT_SCHEDULED"
-	if [ $COUNT_NOTYET -le 0 ] ; then
-		COUNT_NOTYET=0
-	fi
 	QUERY="	SELECT count(sources.name) FROM sources,source_packages
 			WHERE sources.name NOT IN
 			(SELECT sources.name FROM sources,sources_scheduled
@@ -434,7 +433,6 @@ update_html_schedule() {
 	if [ ${COUNT_NEW_VERSIONS} -ne 0 ] ; then
 		write_page "<p>For ${COUNT_NEW_VERSIONS} packages newer versions are available which have not been tested yet.</p>"
 	fi
-	write_page "<p>${COUNT_NOTYET} packages have not been tested at all.</p>"
 	write_page "<p>${COUNT_SCHEDULED} packages are currently scheduled for testing: <code>"
 	force_package_targets $SCHEDULED
 	link_packages $SCHEDULED
diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_graphs.sh
index f8d20ea9..a584ce25 100755
--- a/bin/reproducible_html_graphs.sh
+++ b/bin/reproducible_html_graphs.sh
@@ -63,11 +63,9 @@ if [ -z $RESULT ] ; then
 	sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[5]} VALUES (\"$DATE\", \"$ISSUES\")"
 fi
 
-for i in $(seq 1 ${#META_PKGSET[@]}) ; do
-	META_LIST=$(cat /srv/reproducible-results/meta_pkgsets/${META_PKGSET[$i]}.pkgset)
-	RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"${META_PKGSET[$i]}\"")
-	if [ -z $RESULT ] ; then
-		META_TOTAL=0
+gather_meta_stats() {
+	META_LIST=$(cat /srv/reproducible-results/meta_pkgsets/${META_PKGSET[$1]}.pkgset)
+	if [ ! -z "$META_LIST" ] ; then
 		META_WHERE=""
 		for PKG in $META_LIST ; do
 			if [ -z "$META_WHERE" ] ; then
@@ -75,14 +73,27 @@ for i in $(seq 1 ${#META_PKGSET[@]}) ; do
 			else
 				META_WHERE="$META_WHERE, '$PKG'"
 			fi
-			let "META_TOTAL=META_TOTAL+1"
 		done
 		META_WHERE="$META_WHERE)"
-		META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
-		META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
-		META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)<='$DATE' AND $META_WHERE;")
-		META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)<='$DATE' AND $META_WHERE;")
-		sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKGSET[$i]}\", $META_GOOD, $META_BAD, $META_UGLY, $META_REST)"
+	else
+		META_WHERE="name = 'meta-name-does-not-exist'"
+	fi
+	COUNT_META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
+	COUNT_META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
+	COUNT_META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)<='$DATE' AND $META_WHERE;")
+	COUNT_META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)<='$DATE' AND $META_WHERE;")
+	META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT name from source_packages WHERE status = 'reproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
+	META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT name from source_packages WHERE status = 'unreproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
+	META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT name from source_packages WHERE status = 'FTBFS' AND date(build_date)<='$DATE' AND $META_WHERE;")
+	META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT name NAME from source_packages WHERE (status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)<='$DATE' AND $META_WHERE;")
+}
+
+for i in $(seq 1 ${#META_PKGSET[@]}) ; do
+	META_LIST=$(cat /srv/reproducible-results/meta_pkgsets/${META_PKGSET[$i]}.pkgset)
+	RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"${META_PKGSET[$i]}\"")
+	if [ -z $RESULT ] ; then
+		gather_meta_stats $i
+		sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)"
 		touch -d "$DATE 00:00" ${TABLE[6]}_${META_PKGSET[$i]}.png
 	fi
 done
@@ -146,20 +157,20 @@ YLABEL[4]="Amount of packages"
 YLABEL[5]="Amount of issues"
 
 redo_png() {
-	echo "${FIELDS[$i]}" > ${TABLE[$i]}.csv
+	echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv
 	# TABLE[3+4+5] don't have a suite column...
 	# 6 is special anyway
-	if [ $i -eq 6 ] ; then
-		WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$2'"
-	elif [ $i -ne 3 ] && [ $i -ne 4 ] && [ $i -ne 5 ] ; then
+	if [ $1 -eq 6 ] ; then
+		WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$3'"
+	elif [ $1 -ne 3 ] && [ $1 -ne 4 ] && [ $1 -ne 5 ] ; then
 		WHERE_EXTRA="WHERE suite = '$SUITE'"
 	else
 		WHERE_EXTRA=""
 	fi
-	sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$i]} from ${TABLE[$i]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$i]}.csv
-	/srv/jenkins/bin/make_graph.py ${TABLE[$i]}.csv $1 ${COLOR[$i]} "${MAINLABEL[$i]}" "${YLABEL[$i]}"
-	rm ${TABLE[$i]}.csv
-	mv $1 /var/lib/jenkins/userContent/
+	sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv
+	/srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLOR[$1]} "${MAINLABEL[$1]}" "${YLABEL[$1]}"
+	rm ${TABLE[$1]}.csv
+	mv $2 /var/lib/jenkins/userContent/
 }
 
 write_usertag_table() {
@@ -213,40 +224,70 @@ write_icon
 write_page "$COUNT_BLACKLISTED blacklisted packages neither.</p>"
 write_page "<p>"
 # FIXME: we don't do 2 / stats_builds_age.png yet :/ (also see above)
-for i in 0 3 4 5 6 1 ; do
+for i in 0 3 4 5 1 ; do
 	if [ "$i" = "3" ] ; then
 		write_usertag_table
 	fi
-	# FIXME: split this out in html_meta_graphs... really.
-	if [ "$i" = "6" ] ; then
-		# FIXME: THIS IS A MESS
-		for j in $(seq 1 ${#META_PKGSET[@]}) ; do
-			MAINLABEL[6]="Package reproducibility status for ${META_PKGSET[$j]} packages"
-			YLABEL[6]="Amount (${META_PKGSET[$j]} packages)"
-			PNG=${TABLE[$i]}_${META_PKGSET[$j]}.png
-			write_page " <div>"
-			write_page " <a href=\"/userContent/$PNG\"><img src=\"/userContent/$PNG\" class=\"graph\" alt=\"${MAINLABEL[$i]}\"></a>"
-			write_page " <br />The package set '${META_PKGSET[$j]}' consists of: "
-			# FIXME: split into good/bad/ugly too
-			META_LIST=$(cat /srv/reproducible-results/meta_pkgsets/${META_PKGSET[$i]}.pkgset)
-			force_package_targets $META_LIST
-			link_packages $META_LIST
-			write_page " </div>"
-			# redo pngs once a day
-			if [ ! -f /var/lib/jenkins/userContent/$PNG ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name $PNG) ] ; then
-				# FIXME: call redo_png differently here.. sux
-				redo_png $PNG ${META_PKGSET[$j]}
-			fi
-		done
-	else
-		write_page " <a href=\"/userContent/${TABLE[$i]}.png\"><img src=\"/userContent/${TABLE[$i]}.png\" class=\"graph\" alt=\"${MAINLABEL[$i]}\"></a>"
-		# redo pngs once a day
-		if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then
-			redo_png ${TABLE[$i]}.png
-		fi
+	write_page " <a href=\"/userContent/${TABLE[$i]}.png\"><img src=\"/userContent/${TABLE[$i]}.png\" class=\"graph\" alt=\"${MAINLABEL[$i]}\"></a>"
+	# redo pngs once a day
+	if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then
+		redo_png $i ${TABLE[$i]}.png
 	fi
 done
 write_page "</p>"
 write_page_footer
 publish_page
 
+VIEW=pkg_sets
+PAGE=index_${VIEW}.html
+echo "$(date) - starting to write $PAGE page."
+write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}"
+for i in $(seq 1 ${#META_PKGSET[@]}) ; do
+	gather_meta_stats $i
+	MAINLABEL[6]="Package reproducibility status for ${META_PKGSET[$i]} packages"
+	YLABEL[6]="Amount (${META_PKGSET[$i]} packages)"
+	PNG=${TABLE[6]}_${META_PKGSET[$i]}.png
+	# redo pngs once a day
+	if [ ! -f /var/lib/jenkins/userContent/$PNG ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name $PNG) ] ; then
+		# FIXME: call redo_png differently here.. sux
+		redo_png 6 $PNG ${META_PKGSET[$i]}
+	fi
+	write_page "<p><a href=\"/userContent/$PNG\"><img src=\"/userContent/$PNG\" class=\"graph\" alt=\"${MAINLABEL[6]}\"></a>"
+	write_page "<br />The package set '${META_PKGSET[$i]}' consists of: <br />"
+	set_icon reproducible
+	write_icon
+	write_page "$COUNT_META_GOOD packages successfully built reproducibly:"
+	force_package_targets $META_GOOD
+	link_packages $META_GOOD
+	write_page "<br />"
+	set_icon unreproducible with
+	write_icon
+	write_page "$COUNT_META_BAD packages failed to built reproducibly:"
+	force_package_targets $META_BAD
+	link_packages $META_BAD
+	write_page "<br />"
+	if [ $COUNT_META_UGLY -gt 0 ] ; then
+		set_icon FTBFS
+		write_icon
+		write_page "$COUNT_META_UGLY packages failed to build from source:"
+		force_package_targets $META_UGLY
+		link_packages $META_UGLY
+		write_page "<br />"
+	fi
+	if [ $COUNT_META_REST -gt 0 ] ; then
+		set_icon not_for_us
+		write_icon
+		set_icon blacklisted
+		write_icon
+		set_icon 404
+		write_icon
+		write_page "$COUNT_META_REST packages are either blacklisted, not for us or cannot be downloaded:"
+		force_package_targets $META_REST
+		link_packages $META_REST
+		write_page "<br />"
+	fi
+	write_page "</p>"
+done
+write_page_footer
+publish_page
+