reproducible: improve output for debugging, fix image updating

1 files changed, 25 insertions, 10 deletions

diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_graphs.sh
index 369522af..3eca35fc 100755
--- a/bin/reproducible_html_graphs.sh
+++ b/bin/reproducible_html_graphs.sh
@@ -26,6 +26,7 @@ gather_stats
 #
 # we only do stats up until yesterday... we also could do today too but not update the db yet...
 DATE=$(date -d "1 day ago" '+%Y-%m-%d')
+FORCE_DATE=$(date -d "2 day ago" '+%Y-%m-%d')
 TABLE[0]=stats_pkg_state
 TABLE[1]=stats_builds_per_day
 TABLE[2]=stats_builds_age
@@ -39,6 +40,7 @@ TABLE[6]=stats_meta_pkg_state
 #
 RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,suite from ${TABLE[0]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\"")
 if [ -z $RESULT ] ; then
+	echo "Updating packages and builds stats for $SUITE on $DATE."
 	ALL=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(name) FROM sources WHERE suite='${SUITE}'")
 	GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE';")
 	GOOAY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = 'reproducible' AND date(r.build_date)='$DATE';")
@@ -69,11 +71,14 @@ if [ -z $RESULT ] ; then
 	# we do 3 later and 6 is special anyway...
 	for i in 0 1 2 4 5 ; do
 		PREFIX=""
-		# force regeneration of the image if it exists
 		if [ $i -eq 0 ] ; then
 			PREFIX=$SUITE
 		fi
-		[ ! -f /var/lib/jenkins/userContent/$PREFIX/${TABLE[$i]}.png ] || touch -d "$DATE 00:00" /var/lib/jenkins/userContent/$PREFIX/${TABLE[$i]}.png
+		# force regeneration of the image if it exists
+		if [ -f /var/lib/jenkins/userContent/$PREFIX/${TABLE[$i]}.png ] ; then
+			echo "Touching $PREFIX/${TABLE[$i]}.png..."
+			touch -d "$FORCE_DATE 00:00" /var/lib/jenkins/userContent/$PREFIX/${TABLE[$i]}.png
+		fi
 	done
 fi
 
@@ -82,6 +87,7 @@ fi
 #
 RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum from ${TABLE[4]} WHERE datum = \"$DATE\"")
 if [ -z $RESULT ] ; then
+	echo "Updating notes stats for $DATE."
 	NOTES_GIT_PATH="/var/lib/jenkins/jobs/reproducible_html_notes/workspace"
 	if [ ! -d ${NOTES_GIT_PATH} ] ; then
 		echo "Warning: ${NOTES_GIT_PATH} does not exist, has the job been renamed???"
@@ -142,8 +148,12 @@ if [ "$SUITE" != "experimental" ] ; then
 		if [ -z $RESULT ] ; then
 			META_RESULT=true
 			gather_meta_stats $i
-			! $META_RESULT || sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)"
-			touch -d "$DATE 00:00" /var/lib/jenkins/userContent/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png
+			if $META_RESULT ; then
+				 sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)"
+				echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE on $DATE."
+			fi
+			echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..."
+			touch -d "$FORCE_DATE 00:00" /var/lib/jenkins/userContent/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png
 		fi
 	done
 fi
@@ -154,6 +164,7 @@ fi
 USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness buildinfo cpu signatures environment umask"
 RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT * from ${TABLE[3]} WHERE datum = \"$DATE\"")
 if [ -z $RESULT ] ; then
+	echo "Updating bug stats for $DATE."
 	declare -a DONE
 	declare -a OPEN
 	SQL="INSERT INTO ${TABLE[3]} VALUES (\"$DATE\" "
@@ -174,7 +185,8 @@ if [ -z $RESULT ] ; then
 	echo $SQL
 	sqlite3 -init ${INIT} ${PACKAGES_DB} "$SQL"
 	# force regeneration of the image
-	touch -d "$DATE 00:00" /var/lib/jenkins/userContent/${TABLE[3]}.png
+	echo "Touching ${TABLE[3]}.png..."
+	touch -d "$FORCE_DATE 00:00" /var/lib/jenkins/userContent/${TABLE[3]}.png
 fi
 
 # used for redo_png (but only needed to define once)
@@ -214,6 +226,7 @@ YLABEL[4]="Amount of packages"
 YLABEL[5]="Amount of issues"
 
 redo_png() {
+	echo "Checking whether to update $2..."
 	# $1 = id of the stats table
 	# $2 = image file name
 	# $3 = meta package set, only sensible if $1=6
@@ -244,8 +257,10 @@ redo_png() {
 	fi
 	# only generate graph is the query returned data
 	if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then
+		echo "Updating $2..."
 		DIR=$(dirname $2)
 		mkdir -p $DIR
+		echo "Generating $2."
 		/srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLOR[$1]} "${MAINLABEL[$1]}" "${YLABEL[$1]}"
 		mv $2 /var/lib/jenkins/userContent/$DIR
 		[ "$DIR" = "." ] || rmdir $(dirname $2)
@@ -315,9 +330,9 @@ write_page "$COUNT_BLACKLISTED blacklisted packages neither.</p>"
 write_page "<p>"
 write_page " <a href=\"/userContent/$SUITE/${TABLE[0]}.png\"><img src=\"/userContent/$SUITE/${TABLE[0]}.png\" alt=\"${MAINLABEL[0]}\"></a>"
 # redo png once a day
-if [ ! -f /var/lib/jenkins/userContent/$SUITE/${TABLE[0]}.png ] || [ -z $(find /var/lib/jenkins/userContent/$SUITE -maxdepth 1 -mtime +0 -name ${TABLE[0]}.png) ] ; then
-		redo_png 0 $SUITE/${TABLE[0]}.png
-	fi
+if [ ! -f /var/lib/jenkins/userContent/$SUITE/${TABLE[0]}.png ] || [ ! -z $(find /var/lib/jenkins/userContent/$SUITE -maxdepth 1 -mtime +0 -name ${TABLE[0]}.png) ] ; then
+	redo_png 0 $SUITE/${TABLE[0]}.png
+fi
 write_page "</p>"
 write_page_footer
 publish_page $SUITE
@@ -346,7 +361,7 @@ for i in $(seq 1 ${#META_PKGSET[@]}) ; do
 		YLABEL[6]="Amount (${META_PKGSET[$i]} packages)"
 		PNG=${TABLE[6]}_${META_PKGSET[$i]}.png
 		# redo pngs once a day
-		if [ ! -f /var/lib/jenkins/userContent/$SUITE/$ARCH/$PNG ] || [ -z $(find /var/lib/jenkins/userContent/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then
+		if [ ! -f /var/lib/jenkins/userContent/$SUITE/$ARCH/$PNG ] || [ ! -z $(find /var/lib/jenkins/userContent/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then
 			redo_png 6 $SUITE/$ARCH/$PNG ${META_PKGSET[$i]}
 		fi
 		write_page "<p><a href=\"/userContent/$SUITE/$ARCH/$PNG\"><img src=\"/userContent/$SUITE/$ARCH/$PNG\" alt=\"${MAINLABEL[6]}\"></a>"
@@ -416,7 +431,7 @@ for i in 3 4 5 1 ; do
 	fi
 	write_page " <a href=\"/userContent/${TABLE[$i]}.png\"><img src=\"/userContent/${TABLE[$i]}.png\" alt=\"${MAINLABEL[$i]}\"></a>"
 	# redo pngs once a day
-	if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then
+	if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ ! -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then
 		redo_png $i ${TABLE[$i]}.png
 	fi
 done