reproducible archlinux: include repository stats in html page

3 files changed, 44 insertions, 1 deletions

diff --git a/bin/reproducible_build_archlinux_pkg.sh b/bin/reproducible_build_archlinux_pkg.sh
index 80b0897c..8dc5a5d3 100755
--- a/bin/reproducible_build_archlinux_pkg.sh
+++ b/bin/reproducible_build_archlinux_pkg.sh
@@ -218,7 +218,6 @@ TIMEOUT=8	# maximum time in hours for a single build
 DATE=$(date -u +'%Y-%m-%d %H:%M')
 START=$(date +'%s')
 BUILDER="${JOB_NAME#reproducible_builder_}/${BUILD_ID}"
-ARCHLINUX_PKGS=/srv/reproducible-results/.archlinux_pkgs
 DUMMY=$(mktemp -t archlinux-dummy-XXXXXXXX)
 
 #
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index 43349a3b..d4953bc2 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -55,6 +55,7 @@ USERTAGS="toolchain infrastructure timestamps fileordering buildpath username ho
 # common settings for testing Archlinux
 ARCHLINUX_BUILD_NODE=profitbricks-build3-amd64
 ARCHLINUX_REPOS="core extra"
+ARCHLINUX_PKGS=/srv/reproducible-results/.archlinux_pkgs
 
 # common settings for testing rpm based distros
 RPM_BUILD_NODE=profitbricks-build3-amd64
diff --git a/bin/reproducible_html_archlinux.sh b/bin/reproducible_html_archlinux.sh
index a333de32..11220893 100755
--- a/bin/reproducible_html_archlinux.sh
+++ b/bin/reproducible_html_archlinux.sh
@@ -14,6 +14,7 @@ ARCHBASE=$BASE/archlinux
 #
 # analyse results to create the webpage
 #
+echo "$(date -u) - starting to analyse build results."
 HTML_FTBFS=$(mktemp)
 HTML_FTBR=$(mktemp)
 HTML_DEPWAIT=$(mktemp)
@@ -22,13 +23,24 @@ HTML_GOOD=$(mktemp)
 HTML_UNKNOWN=$(mktemp)
 HTML_BUFFER=$(mktemp)
 HTML_TARGET=""
+HTML_REPOSTATS=$(mktemp)
 for REPOSITORY in $ARCHLINUX_REPOS ; do
+	echo "$(date -u) - starting to analyse build results for '$REPOSITORY'."
+	TOTAL=$(cat ${ARCHLINUX_PKGS}_$REPOSITORY | sed -s "s# #\n#g" | wc -l)
+	TESTED=0
+	NR_FTBFS=0
+	NR_FTBR=0
+	NR_DEPWAIT=0
+	NR_404=0
+	NR_GOOD=0
+	NR_UNKNOWN=0
 	for PKG in $(find $ARCHBASE/$REPOSITORY/* -maxdepth 1 -type d -exec basename {} \;) ; do
 		if [ -z "$(cd $ARCHBASE/$REPOSITORY/$PKG/ ; ls)" ] ; then
 			# directory exists but is empty: package is building…
 			echo "$(date -u ) - ignoring $PKG from '$REPOSITORY' which is building right now…"
 			continue
 		fi
+		let TESTED+=1
 		echo "     <tr>" >> $HTML_BUFFER
 		echo "      <td>$REPOSITORY</td>" >> $HTML_BUFFER
 		echo "      <td>$PKG</td>" >> $HTML_BUFFER
@@ -36,32 +48,41 @@ for REPOSITORY in $ARCHLINUX_REPOS ; do
 		if [ -z "$(cd $ARCHBASE/$REPOSITORY/$PKG/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
 			if [ ! -z "$(grep '==> ERROR: Could not resolve all dependencies' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
 				HTML_TARGET=$HTML_DEPWAIT
+				let NR_DEPWAIT+=1
 				echo "       <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> could not resolve dependencies" >> $HTML_BUFFER
 			elif [ ! -z "$(egrep '==> ERROR: .pacman. failed to install missing dependencies.' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
 				HTML_TARGET=$HTML_DEPWAIT
+				let NR_DEPWAIT+=1
 				echo "       <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> failed to install dependencies" >> $HTML_BUFFER
 			elif [ ! -z "$(egrep '==> ERROR: A failure occurred in (build|package)' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
 				HTML_TARGET=$HTML_FTBFS
+				let NR_FTBFS+=1
 				echo "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build from source" >> $HTML_BUFFER
 			elif [ ! -z "$(egrep '==> ERROR: A failure occurred in check' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
 				HTML_TARGET=$HTML_FTBFS
+				let NR_FTBFS+=1
 				echo "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build from source, while running tests" >> $HTML_BUFFER
 			elif [ ! -z "$(egrep '==> ERROR: Failure while downloading' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
 				HTML_TARGET=$HTML_404
+				let NR_404+=1
 				echo "       <img src=\"/userContent/static/weather-severe-alert.png\" alt=\"404 icon\" /> failed to download source" >> $HTML_BUFFER
 			elif [ ! -z "$(egrep '==> ERROR: One or more files did not pass the validity check' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
 				HTML_TARGET=$HTML_FTBFS
+				let NR_FTBFS+=1
 				echo "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to verify source" >> $HTML_BUFFER
 			elif [ ! -z "$(egrep 'makepkg was killed by timeout after' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
 				HTML_TARGET=$HTML_FTBFS
+				let NR_FTBFS+=1
 				echo "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build, killed by timeout" >> $HTML_BUFFER
 			else
 				echo "       probably failed to build from source, please investigate" >> $HTML_BUFFER
 				HTML_TARGET=$HTML_UNKNOWN
+				let NR_UNKNOWN+=1
 				# or is it reproducible???
 			fi
 		else
 			HTML_TARGET=$HTML_FTBR
+			let NR_FTBR+=1
 			for ARTIFACT in $(cd $ARCHBASE/$REPOSITORY/$PKG/ ; ls *.pkg.tar.xz.html) ; do
 				echo "       <img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> <a href=\"/archlinux/$REPOSITORY/$PKG/$ARTIFACT\">${ARTIFACT:0:-5}</a> is unreproducible<br />" >> $HTML_BUFFER
 			done
@@ -79,6 +100,24 @@ for REPOSITORY in $ARCHLINUX_REPOS ; do
 		cat $HTML_BUFFER >> $HTML_TARGET
 		rm $HTML_BUFFER > /dev/null
 	done
+	# prepare stats per repository
+	PERCENT_TOTAL=$(echo "scale=1 ; ($TESTED*100/$TOTAL)" | bc)
+	if [ $(echo $PERCENT_TOTAL/1|bc) -lt 98 ] ; then
+		NR_TESTED="$TESTED <span style=\"font-size:0.8em;\">($PERCENT_TOTAL% of $TOTAL tested)</span>"
+	else
+		NR_TESTED=$TESTED
+	fi
+	echo "     <tr>" >> $HTML_REPOSTATS
+	echo "      <td>$REPOSITORY</td><td>$NR_TESTED</td>" >> $HTML_REPOSTATS
+	for i in $NR_GOOD $NR_FTBR $NR_FTBFS $NR_DEPWAIT $NR_404 $NR_UNKNOWN ; do
+		PERCENT_i=$(echo "scale=1 ; ($i*100/$TOTAL)" | bc)
+		if [ "$PERCENT_i" != "0" ] ; then
+			echo "      <td>$i ($PERCENT_i%)</td>" >> $HTML_REPOSTATS
+		else
+			echo "      <td>$i</td>" >> $HTML_REPOSTATS
+		fi
+	done
+	echo "     </tr>" >> $HTML_REPOSTATS
 done
 #
 # write out the actual webpage
@@ -106,6 +145,10 @@ cat > $PAGE <<- EOF
 EOF
 write_page_intro 'Arch Linux'
 write_explaination_table 'Arch Linux'
+write_page "    <table><tr><th>repository</th><th>all sources packages</th><th>reproducible packages</th><th>unreproducible packages</th><th>packages failing to build</th><th>packages in depwait state</th><th>packages 404</th><th>unknown state</th></tr>"
+cat $HTML_REPOSTATS >> $PAGE
+rm $HTML_REPOSTATS > /dev/null
+write_page "    </table>"
 write_page "    <table><tr><th>repository</th><th>source package</th><th>test result</th><th>test date</th><th>1st build log</th><th>2nd build log</th></tr>"
 for i in $HTML_UNKNOWN $HTML_FTBFS $HTML_DEPWAIT $HTML_404 $HTML_FTBR $HTML_GOOD ; do
 	cat $i >> $PAGE