reproducible: refactor, move common functions out, new script+job to just create the dd-list page daily

author: Holger Levsen <holger@layer-acht.org> 2014-10-17 16:02:50 +0200
committer: Holger Levsen <holger@layer-acht.org> 2014-10-17 16:08:44 +0200
commit: 1aa7e0422d9f0a4561aa952ce6a89a1ba9bedd31 (patch)
tree: 4d1349b8270187c0ff4c5c192b034b3eb24da0e0 /bin
parent: 3eedec3a7cdbe2e364cd2783f4f8ef7873041cd4 (diff)
download: jenkins.debian.net-1aa7e0422d9f0a4561aa952ce6a89a1ba9bedd31.tar.xz
3 files changed, 176 insertions, 128 deletions
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index c828759e..954aa384 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -100,3 +100,134 @@ fi
 # shop trailing slash
 JENKINS_URL=${JENKINS_URL:0:-1}
 
+init_html() {
+	SUITE=sid
+	ALLSTATES="reproducible FTBR_with_buildinfo FTBR FTBFS 404 not_for_us blacklisted"
+	MAINVIEW="stats"
+	ALLVIEWS="last_24h last_48h all_abc"
+	declare -A SPOKENTARGET
+	SPOKENTARGET["last_24h"]="packages tested in the last 24h"
+	SPOKENTARGET["last_48h"]="packages tested in the last 48h"
+	SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)"
+	SPOKENTARGET["dd-list"]="maintainers of unreproducible packages"
+	SPOKENTARGET["stats"]="various statistics about reproducible builds"
+	SPOKENTARGET["notes"]="packages with notes"
+	SPOKENTARGET["issues"]="known issues related to reproducible builds"
+	SPOKENTARGET["reproducible"]="packages which built reproducibly"
+	SPOKENTARGET["FTBR"]="packages which failed to build reproducibly and don't create a .buildinfo file"
+	SPOKENTARGET["FTBR_with_buildinfo"]="packages which failed to build reproducibly and create a .buildinfo file"
+	SPOKENTARGET["FTBFS"]="packages which failed to build from source"
+	SPOKENTARGET["404"]="packages where the sources failed to downloaded"
+	SPOKENTARGET["not_for_us"]="packages which should not be build on 'amd64'"
+	SPOKENTARGET["blacklisted"]="packages which have been blacklisted"
+	NOTES_PATH=/var/lib/jenkins/userContent/notes
+	ISSUES_PATH=/var/lib/jenkins/userContent/issues
+	mkdir -p $NOTES_PATH $ISSUES_PATH
+	# FIXME RB_PATH would also be a good idea
+	mkdir -p /var/lib/jenkins/userContent/rb-pkg/
+	# query some data we need everywhere
+	AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(name) FROM sources")
+	COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages")
+	COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"reproducible\"")
+	PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc)
+	PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc)
+	GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc)
+}
+
+write_page() {
+	echo "$1" >> $PAGE
+}
+
+set_icon() {
+	# icons taken from tango-icon-theme (0.8.90-5)
+	# licenced under http://creativecommons.org/licenses/publicdomain/
+	STATE_TARGET_NAME="$1"
+	case "$1" in
+		reproducible)		ICON=weather-clear.png
+					;;
+		unreproducible|FTBR*)	if [ "$2" != "" ] ; then
+						ICON=weather-showers-scattered.png
+						STATE_TARGET_NAME=FTBR_with_buildinfo
+					else
+						ICON=weather-showers.png
+						STATE_TARGET_NAME=FTBR
+					fi
+					;;
+		FTBFS)			ICON=weather-storm.png
+					;;
+		404)			ICON=weather-severe-alert.png
+					;;
+		not_for_us|"not for us")	ICON=weather-few-clouds-night.png
+					STATE_TARGET_NAME="not_for_us"
+					;;
+		blacklisted)		ICON=error.png
+					;;
+		*)			ICON=""
+	esac
+}
+
+write_icon() {
+	# ICON and STATE_TARGET_NAME are set by set_icon()
+	write_page "<a href=\"$JENKINS_URL/userContent/index_${STATE_TARGET_NAME}.html\" target=\"_parent\"><img src=\"$JENKINS_URL/userContent/static/$ICON\" alt=\"${STATE_TARGET_NAME} icon\" /></a>"
+}
+
+write_page_header() {
+	rm -f $PAGE
+	write_page "<!DOCTYPE html><html><head>"
+	write_page "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />"
+	write_page "<link href=\"$JENKINS_URL/userContent/static/style.css\" type=\"text/css\" rel=\"stylesheet\" />"
+	write_page "<title>$2</title></head>"
+	write_page "<body><header><h2>$2</h2>"
+	if [ "$1" = "$MAINVIEW" ] ; then
+		write_page "<p>These pages are updated every six hours. Results are obtained from <a href=\"$JENKINS_URL/view/reproducible\">several jobs running on jenkins.debian.net</a>. Thanks to <a href=\"https://www.profitbricks.com\">Profitbricks</a> for donating the virtual machine it's running on!</p>"
+	fi
+	write_page "<p>$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD <a href=\"https://wiki.debian.org/ReproducibleBuilds\">packages should be reproducibly buildable!</a>"
+	if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] || [ "$1" = "stats" ] ; then
+		write_page " Join <code>#debian-reproducible</code> on OFTC to get support for making sure your packages build reproducibly too!"
+	fi
+	write_page "</p>"
+	write_page "<ul><li>Have a look at:</li>"
+	for MY_STATE in $ALLSTATES ; do
+		WITH=""
+		if [ "$MY_STATE" = "FTBR_with_buildinfo" ] ; then
+			WITH="YES"
+		fi
+		set_icon $MY_STATE $WITH
+		write_page "<li>"
+		write_icon
+		write_page "</li>"
+	done
+	for TARGET in issues notes $ALLVIEWS dd-list stats ; do
+		if [ "$TARGET" = "issues" ] || [ "$TARGET" = "stats" ]; then
+			SPOKEN_TARGET=$TARGET
+		else
+			SPOKEN_TARGET=${SPOKENTARGET[$TARGET]}
+		fi
+		write_page "<li><a href=\"$JENKINS_URL/userContent/index_${TARGET}.html\">${SPOKEN_TARGET}</a></li>"
+	done
+	write_page "</ul>"
+	write_page "</header>"
+}
+
+write_page_footer() {
+	write_page "<hr/><p style=\"font-size:0.9em;\">There is more information <a href=\"$JENKINS_URL/userContent/about.html\">about jenkins.debian.net</a> and about <a href=\"https://wiki.debian.org/ReproducibleBuilds\"> reproducible builds of Debian</a> available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014 <a href=\"mailto:holger@layer-acht.org\">Holger Levsen</a>, GPL-2 licensed. The weather icons are public domain and have been taken from the <a href="http://tango.freedesktop.org/Tango_Icon_Library" target="_blank">Tango Icon Library</a>.</p>"
+	write_page "</body></html>"
+}
+
+write_page_meta_sign() {
+	write_page "<p style=\"font-size:0.9em;\">An underlined package is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue."
+	if $BETA_SIGN ; then
+		write_page "A &beta; sign after a package which is unreproducible indicates that a .buildinfo file was generated."
+		write_page "And that means the <a href=\"https://wiki.debian.org/ReproducibleBuilds#The_basics_for_making_packages_build_reproducible\">basics for building packages reproducibly are covered</a>."
+	fi
+	write_page "</p>"
+}
+
+publish_page() {
+	cp $PAGE /var/lib/jenkins/userContent/
+	if [ "$VIEW" = "$MAINVIEW" ] ; then
+		cp $PAGE /var/lib/jenkins/userContent/reproducible.html
+	fi
+	rm $PAGE
+}
+
diff --git a/bin/reproducible_html_dd_list.sh b/bin/reproducible_html_dd_list.sh
new file mode 100755
index 00000000..07c6ba5a
--- /dev/null
+++ b/bin/reproducible_html_dd_list.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Copyright 2014 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv=2
+
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code defining db access
+. /srv/jenkins/bin/reproducible_common.sh
+
+set +x
+init_html
+
+VIEW=dd-list
+PAGE=index_${VIEW}.html
+echo "$(date) - starting to write $PAGE page."
+write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}"
+TMPFILE=$(mktemp)
+BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo)
+echo "${BAD}" | dd-list -i > $TMPFILE || true
+write_page "<p>The following maintainers and uploaders are listed for packages which have built unreproducibly:</p><p><pre>"
+while IFS= read -r LINE ; do
+	if [ "${LINE:0:3}" = "   " ] ; then
+		PACKAGE=$(echo "${LINE:3}" | cut -d " " -f1)
+		UPLOADERS=$(echo "${LINE:3}" | cut -d " " -f2-)
+		if [ "$UPLOADERS" = "$PACKAGE" ] ; then
+			UPLOADERS=""
+		fi
+		write_page "   <a href=\"$JENKINS_URL/userContent/rb-pkg/$PACKAGE.html\">$PACKAGE</a> $UPLOADERS"
+	else
+		LINE="$(echo $LINE | sed 's#&#\&amp;#g ; s#<#\&lt;#g ; s#>#\&gt;#g')"
+		write_page "$LINE"
+	fi
+done < $TMPFILE
+write_page "</pre></p>"
+rm $TMPFILE
+write_page_footer
+publish_page
+
+echo "Enjoy $JENKINS_URL/userContent/$PAGE"
diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh
index 560b678a..08e1d730 100755
--- a/bin/reproducible_stats.sh
+++ b/bin/reproducible_stats.sh
@@ -14,6 +14,8 @@ common_init "$@"
 cp $PACKAGES_DB /var/lib/jenkins/userContent/reproducible.db
 
 set +x
+init_html
+
 declare -A GOOD
 declare -A BAD
 declare -A UGLY
@@ -21,19 +23,12 @@ declare -A SOURCELESS
 declare -A NOTFORUS
 declare -A STAR
 declare -A LINKTARGET
-declare -A SPOKENTARGET
 LAST24="AND build_date > datetime('now', '-24 hours') "
 LAST48="AND build_date > datetime('now', '-48 hours') "
-SUITE=sid
-AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(name) FROM sources")
-ALLSTATES="reproducible FTBR_with_buildinfo FTBR FTBFS 404 not_for_us blacklisted"
-MAINVIEW="stats"
-ALLVIEWS="last_24h last_48h all_abc"
 GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY build_date DESC" | xargs echo)
 GOOD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo)
 GOOD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo)
 GOOD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY name" | xargs echo)
-COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"reproducible\"")
 BAD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo)
 BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo)
 BAD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo)
@@ -52,28 +47,10 @@ NOTFORUS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_
 COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"not for us\"" | xargs echo)
 BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"blacklisted\" ORDER BY name" | xargs echo)
 COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"blacklisted\"" | xargs echo)
-COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages")
-PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc)
-PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc)
 PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc)
 PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc)
 PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc)
 PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc)
-GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc)
-SPOKENTARGET["last_24h"]="packages tested in the last 24h"
-SPOKENTARGET["last_48h"]="packages tested in the last 48h"
-SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)"
-SPOKENTARGET["dd-list"]="maintainers of unreproducible packages"
-SPOKENTARGET["stats"]="various statistics about reproducible builds"
-SPOKENTARGET["notes"]="packages with notes"
-SPOKENTARGET["issues"]="known issues related to reproducible builds"
-SPOKENTARGET["reproducible"]="packages which built reproducibly"
-SPOKENTARGET["FTBR"]="packages which failed to build reproducibly and don't create a .buildinfo file"
-SPOKENTARGET["FTBR_with_buildinfo"]="packages which failed to build reproducibly and create a .buildinfo file"
-SPOKENTARGET["FTBFS"]="packages which failed to build from source"
-SPOKENTARGET["404"]="packages where the sources failed to downloaded"
-SPOKENTARGET["not_for_us"]="packages which should not be build on 'amd64'"
-SPOKENTARGET["blacklisted"]="packages which have been blacklisted"
 
 #
 # gather notes
@@ -87,11 +64,9 @@ else
 	git clone git://git.debian.org/git/reproducible/notes.git notes.git
 fi
 cd $WORKSPACE
+
 PACKAGES_YML=/var/lib/jenkins/notes.git/packages.yml
 ISSUES_YML=/var/lib/jenkins/notes.git/issues.yml
-NOTES_PATH=/var/lib/jenkins/userContent/notes
-ISSUES_PATH=/var/lib/jenkins/userContent/issues
-mkdir -p $NOTES_PATH $ISSUES_PATH
 
 declare -A NOTES_VERSION
 declare -A NOTES_ISSUES
@@ -173,7 +148,7 @@ create_pkg_note() {
 	write_page "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />"
 	write_page "<link href=\"$JENKINS_URL/userContent/static/style.css\" type=\"text/css\" rel=\"stylesheet\" />"
 	write_page "<title>Notes for $1</title></head>"
-	write_page "<body><header>"
+	write_page "<body></header>"
 	write_page "<table class=\"body\">"
 
 	write_page "<tr><td>Version annotated:</td><td colspan=\"2\">${NOTES_VERSION[$1]}</td></tr>"
@@ -353,45 +328,6 @@ validate_yaml() {
 # end note parsing functions...
 #
 
-mkdir -p /var/lib/jenkins/userContent/rb-pkg/
-
-write_page() {
-	echo "$1" >> $PAGE
-}
-
-set_icon() {
-	# icons taken from tango-icon-theme (0.8.90-5)
-	# licenced under http://creativecommons.org/licenses/publicdomain/
-	STATE_TARGET_NAME="$1"
-	case "$1" in
-		reproducible)		ICON=weather-clear.png
-					;;
-		unreproducible|FTBR*)	if [ "$2" != "" ] ; then
-						ICON=weather-showers-scattered.png
-						STATE_TARGET_NAME=FTBR_with_buildinfo
-					else
-						ICON=weather-showers.png
-						STATE_TARGET_NAME=FTBR
-					fi
-					;;
-		FTBFS)			ICON=weather-storm.png
-					;;
-		404)			ICON=weather-severe-alert.png
-					;;
-		not_for_us|"not for us")	ICON=weather-few-clouds-night.png
-					STATE_TARGET_NAME="not_for_us"
-					;;
-		blacklisted)		ICON=error.png
-					;;
-		*)			ICON=""
-	esac
-}
-
-write_icon() {
-	# ICON and STATE_TARGET_NAME are set by set_icon()
-	write_page "<a href=\"$JENKINS_URL/userContent/index_${STATE_TARGET_NAME}.html\" target=\"_parent\"><img src=\"$JENKINS_URL/userContent/static/$ICON\" alt=\"${STATE_TARGET_NAME} icon\" /></a>"
-}
-
 init_pkg_page() {
 	echo "<!DOCTYPE html><html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />" > ${PKG_FILE}
 	echo "<link href=\"../static/style.css\" type=\"text/css\" rel=\"stylesheet\" />" >> ${PKG_FILE}
@@ -496,66 +432,6 @@ link_packages() {
 	done
 }
 
-write_page_header() {
-	rm -f $PAGE
-	write_page "<!DOCTYPE html><html><head>"
-	write_page "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />"
-	write_page "<link href=\"$JENKINS_URL/userContent/static/style.css\" type=\"text/css\" rel=\"stylesheet\" />"
-	write_page "<title>$2</title></head>"
-	write_page "<body><header><h2>$2</h2>"
-	if [ "$1" = "$MAINVIEW" ] || ; then
-		write_page "<p>These pages are updated every six hours. Results are obtained from <a href=\"$JENKINS_URL/view/reproducible\">several jobs running on jenkins.debian.net</a>. Thanks to <a href=\"https://www.profitbricks.com\">Profitbricks</a> for donating the virtual machine it's running on!</p>"
-	fi
-	write_page "<p>$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD <a href=\"https://wiki.debian.org/ReproducibleBuilds\">packages should be reproducibly buildable!</a>"
-	if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] || [ "$1" = "stats" ] ; then
-		write_page " Join <code>#debian-reproducible</code> on OFTC to get support for making sure your packages build reproducibly too!"
-	fi
-	write_page "</p>"
-	write_page "<ul><li>Have a look at:</li>"
-	for MY_STATE in $ALLSTATES ; do
-		WITH=""
-		if [ "$MY_STATE" = "FTBR_with_buildinfo" ] ; then
-			WITH="YES"
-		fi
-		set_icon $MY_STATE $WITH
-		write_page "<li>"
-		write_icon
-		write_page "</li>"
-	done
-	for TARGET in issues notes $ALLVIEWS dd-list stats ; do
-		if [ "$TARGET" = "issues" ] || [ "$TARGET" = "stats" ]; then
-			SPOKEN_TARGET=$TARGET
-		else
-			SPOKEN_TARGET=${SPOKENTARGET[$TARGET]}
-		fi
-		write_page "<li><a href=\"$JENKINS_URL/userContent/index_${TARGET}.html\">${SPOKEN_TARGET}</a></li>"
-	done
-	write_page "</ul>"
-	write_page "</header>"
-}
-
-write_page_footer() {
-	write_page "<hr/><p style=\"font-size:0.9em;\">There is more information <a href=\"$JENKINS_URL/userContent/about.html\">about jenkins.debian.net</a> and about <a href=\"https://wiki.debian.org/ReproducibleBuilds\"> reproducible builds of Debian</a> available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014 <a href=\"mailto:holger@layer-acht.org\">Holger Levsen</a>, GPL-2 licensed. The weather icons are public domain and have been taken from the <a href="http://tango.freedesktop.org/Tango_Icon_Library" target="_blank">Tango Icon Library</a>.</p>"
-	write_page "</body></html>"
-}
-
-write_page_meta_sign() {
-	write_page "<p style=\"font-size:0.9em;\">An underlined package is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue."
-	if $BETA_SIGN ; then
-		write_page "A &beta; sign after a package which is unreproducible indicates that a .buildinfo file was generated."
-		write_page "And that means the <a href=\"https://wiki.debian.org/ReproducibleBuilds#The_basics_for_making_packages_build_reproducible\">basics for building packages reproducibly are covered</a>."
-	fi
-	write_page "</p>"
-}
-
-publish_page() {
-	cp $PAGE /var/lib/jenkins/userContent/
-	if [ "$VIEW" = "$MAINVIEW" ] ; then
-		cp $PAGE /var/lib/jenkins/userContent/reproducible.html
-	fi
-	rm $PAGE
-}
-
 #
 # actually parse the notes
 #
author	Holger Levsen <holger@layer-acht.org>	2014-10-17 16:02:50 +0200
committer	Holger Levsen <holger@layer-acht.org>	2014-10-17 16:08:44 +0200
commit	1aa7e0422d9f0a4561aa952ce6a89a1ba9bedd31 (patch)
tree	4d1349b8270187c0ff4c5c192b034b3eb24da0e0 /bin
parent	3eedec3a7cdbe2e364cd2783f4f8ef7873041cd4 (diff)
download	jenkins.debian.net-1aa7e0422d9f0a4561aa952ce6a89a1ba9bedd31.tar.xz