reproducible fedora rpms: add job to build overview webpage for Fedora 23

3 files changed, 158 insertions, 2 deletions

diff --git a/bin/reproducible_build_rpm.sh b/bin/reproducible_build_rpm.sh
index 3229cba6..a9db601a 100755
--- a/bin/reproducible_build_rpm.sh
+++ b/bin/reproducible_build_rpm.sh
@@ -207,7 +207,7 @@ MODE="master"
 # main - only used in master-mode
 #
 delay_start # randomize start times
-# first, we need to choose a packagey…
+# first, we need to choose a package…
 RELEASE="$1"
 ARCH="$2"
 SRCPACKAGE=""	# package name
diff --git a/bin/reproducible_html_rpms.sh b/bin/reproducible_html_rpms.sh
new file mode 100755
index 00000000..3d9d7706
--- /dev/null
+++ b/bin/reproducible_html_rpms.sh
@@ -0,0 +1,146 @@
+#!/bin/bash
+
+# Copyright 2014-2015 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv=2
+
+DEBUG=false
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code
+. /srv/jenkins/bin/reproducible_common.sh
+
+RPMBASE=$BASE/rpms
+RELEASE="$1"
+ARCH="$2"
+#
+# analyse results to create the webpage
+#
+echo "$(date -u) - starting to analyse build results."
+HTML_FTBFS=$(mktemp)
+HTML_FTBR=$(mktemp)
+HTML_DEPWAIT=$(mktemp)
+HTML_404=$(mktemp)
+HTML_GOOD=$(mktemp)
+HTML_UNKNOWN=$(mktemp)
+HTML_BUFFER=$(mktemp)
+HTML_TARGET=""
+HTML_RPM_STATS=$(mktemp)
+
+echo "$(date -u) - starting to analyse build results for '$RELEASE' ($ARCH)."
+TOTAL=187 # FIXME: calculate this like $(cat ${ARCHLINUX_PKGS}_$RELEASE | sed -s "s# #\n#g" | wc -l)
+TESTED=0
+NR_FTBFS=0
+NR_FTBR=0
+NR_DEPWAIT=0
+NR_404=0
+NR_GOOD=0
+NR_UNKNOWN=0
+for PKG in $(find $RPMBASE/$RELEASE/$ARCH/* -maxdepth 1 -type d -exec basename {} \;) ; do
+	if [ -z "$(cd $RPMBASE/$RELEASE/$ARCH/$PKG/ ; ls)" ] ; then
+		# directory exists but is empty: package is building…
+		echo "$(date -u ) - ignoring $PKG from '$RELEASE' ($ARCH) which is building right now…"
+		continue
+	fi
+	let TESTED+=1
+	echo "     <tr>" >> $HTML_BUFFER
+	echo "      <td>$RELEASE ($ARCH)</td>" >> $HTML_BUFFER
+	echo "      <td>$PKG</td>" >> $HTML_BUFFER
+	echo "      <td>" >> $HTML_BUFFER
+	if [ -z "$(cd $RPMBASE/$RELEASE/$ARCH/$PKG/ ; ls *.rpm.html 2>/dev/null)" ] ; then
+		if [ ! -z "$(grep '==> ERROR: Could not resolve all dependencies' $RPMBASE/$RELEASE/$ARCH/$PKG/build1.log)" ] ; then
+			HTML_TARGET=$HTML_DEPWAIT
+			let NR_DEPWAIT+=1
+			echo "       <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> could not resolve dependencies" >> $HTML_BUFFER
+		elif [ ! -z "$(egrep '==> ERROR: Failure while downloading' $RPMBASE/$RELEASE/$ARCH/$PKG/build1.log)" ] ; then
+			HTML_TARGET=$HTML_404
+			let NR_404+=1
+			echo "       <img src=\"/userContent/static/weather-severe-alert.png\" alt=\"404 icon\" /> failed to download source" >> $HTML_BUFFER
+		elif [ ! -z "$(egrep '==> ERROR: One or more files did not pass the validity check' $RPMBASE/$RELEASE/$ARCH/$PKG/build1.log)" ] ; then
+			HTML_TARGET=$HTML_FTBFS
+			let NR_FTBFS+=1
+			echo "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to verify source" >> $HTML_BUFFER
+		else
+			echo "       probably failed to build from source, please investigate" >> $HTML_BUFFER
+			HTML_TARGET=$HTML_UNKNOWN
+			let NR_UNKNOWN+=1
+			# or is it reproducible???
+		fi
+	else
+		HTML_TARGET=$HTML_FTBR
+		let NR_FTBR+=1
+		for ARTIFACT in $(cd $RPMBASE/$RELEASE/$ARCH/$PKG/ ; ls *.rpm.html) ; do
+			echo "       <img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> <a href=\"/rpms/$RELEASE/$PKG/$ARTIFACT\">${ARTIFACT:0:-5}</a> is unreproducible<br />" >> $HTML_BUFFER
+		done
+	fi
+	echo "      </td>" >> $HTML_BUFFER
+	echo "      <td>$(LANG=C TZ=UTC ls --full-time $RPMBASE/$RELEASE/$ARCH/$PKG/build1.log | cut -d ' ' -f6 )</td>" >> $HTML_BUFFER
+	for LOG in build1.log build2.log ; do
+		if [ -f $RPMBASE/$RELEASE/$ARCH/$PKG/$LOG ] ; then
+			echo "      <td><a href=\"/rpms/$RELEASE/$PKG/$LOG\">$LOG</a></td>" >> $HTML_BUFFER
+		else
+			echo "      <td>&nbsp;</td>" >> $HTML_BUFFER
+		fi
+	done
+	echo "     </tr>" >> $HTML_BUFFER
+	cat $HTML_BUFFER >> $HTML_TARGET
+	rm $HTML_BUFFER > /dev/null
+done
+# prepare stats per repository
+PERCENT_TOTAL=$(echo "scale=1 ; ($TESTED*100/$TOTAL)" | bc)
+if [ $(echo $PERCENT_TOTAL/1|bc) -lt 98 ] ; then
+	NR_TESTED="$TESTED <span style=\"font-size:0.8em;\">($PERCENT_TOTAL% of $TOTAL tested)</span>"
+else
+	NR_TESTED=$TESTED
+fi
+echo "     <tr>" >> $HTML_RPM_STATS
+echo "      <td>$RELEASE ($ARCH)</td><td>$NR_TESTED</td>" >> $HTML_RPM_STATS
+for i in $NR_GOOD $NR_FTBR $NR_FTBFS $NR_DEPWAIT $NR_404 $NR_UNKNOWN ; do
+	PERCENT_i=$(echo "scale=1 ; ($i*100/$TOTAL)" | bc)
+	if [ "$PERCENT_i" != "0" ] ; then
+		echo "      <td>$i ($PERCENT_i%)</td>" >> $HTML_RPM_STATS
+	else
+		echo "      <td>$i</td>" >> $HTML_RPM_STATS
+	fi
+done
+echo "     </tr>" >> $HTML_RPM_STATS
+
+#
+# write out the actual webpage
+#
+DATE=$(date -u +'%Y-%m-%d')
+cd $RPMBASE
+PAGE=rpms/$RELEASE.html
+echo "$(date -u) - starting to build $PAGE"
+cat > $PAGE <<- EOF
+<!DOCTYPE html>
+<html lang="en-US">
+  <head>
+    <meta charset="UTF-8">
+    <title>Repoducible $RELEASE ($ARCH) ?!</title>
+    <link rel='stylesheet' href='global.css' type='text/css' media='all' />
+  </head>
+  <body>
+    <div id="archnavbar">
+	    <div id="logo"></div>
+    </div>
+    <div class="content">
+      <h1>Reproducible $RELEASE ($ARCH) ?!</h1>
+      <div class="page-content">
+
+EOF
+write_page_intro '$RELEASE'
+write_explaination_table '$RELEASE'
+write_page "    <table><tr><th>release (architecture)</th><th>all sources packages</th><th>reproducible packages</th><th>unreproducible packages</th><th>packages failing to build</th><th>packages in depwait state</th><th>packages 404</th><th>unknown state</th></tr>"
+cat $HTML_RPM_STATS >> $PAGE
+rm $HTML_RPM_STATS > /dev/null
+write_page "    </table>"
+write_page "    <table><tr><th>release (architecture)</th><th>source package</th><th>test result</th><th>test date</th><th>1st build log</th><th>2nd build log</th></tr>"
+for i in $HTML_UNKNOWN $HTML_FTBFS $HTML_DEPWAIT $HTML_404 $HTML_FTBR $HTML_GOOD ; do
+	cat $i >> $PAGE
+	rm $i > /dev/null
+done
+write_page "    </table>"
+write_page "</div></div>"
+write_page_footer '$RELEASE'
+echo "$(date -u) - enjoy $REPRODUCIBLE_URL/rpms/$PAGE"
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index 2d518821..66d8b83f 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -781,6 +781,10 @@
     defaults: reproducible
     name: '{name}_html_archlinux'
 
+- job-template:
+    defaults: reproducible
+    name: '{name}_html_fedora-23_x86_64'
+
 - project:
     name: reproducible
     jobs:
@@ -1562,7 +1566,7 @@
         - '{name}_builder_fedora-23_x86_64_1':
             my_description: 'Try to reproducibly build a Fedora 23 package.'
             my_timed: '* * * * *'
-            my_trigger: ''
+            my_trigger: 'reproducible_html_fedora-23_x86_64'
             my_shell: '/srv/jenkins/bin/reproducible_build_rpm.sh fedora-23 x86_64'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: ''
@@ -1572,4 +1576,10 @@
             my_shell: '/srv/jenkins/bin/reproducible_html_archlinux.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: ''
+        - '{name}_html_fedora-23_x86_64':
+            my_description: 'Build a simple webpage for Fedora 23 reproducibility tests'
+            my_timed: ''
+            my_shell: '/srv/jenkins/bin/reproducible_html_rpms.sh fedora-23 x86_64'
+            my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
+            my_node: ''