reproducible: use html framesets to provide easy+fast access to files

2 files changed, 154 insertions, 27 deletions

diff --git a/bin/reproducible_scheduler.sh b/bin/reproducible_scheduler.sh
new file mode 100755
index 00000000..b9fc0c62
--- /dev/null
+++ b/bin/reproducible_scheduler.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+# Copyright 2014 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv=2
+
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# define db
+PACKAGES_DB=/var/lib/jenkins/reproducible.db
+INIT=/var/lib/jenkins/reproducible.init
+if [ ! -f $PACKAGES_DB ] ; then
+	echo "$PACKAGES_DB doesn't exist, no builds possible."
+	exit 1
+fi
+
+# this needs sid entries in sources.list:
+grep deb-src /etc/apt/sources.list | grep sid
+# try apt-get update twice, else fail gracefully, aka not.
+sudo apt-get update || ( sleep $(( $RANDOM % 70 + 30 )) ; sudo apt-get update || true )
+
+# update sources table in db
+update_sources_table() {
+	TMPFILE=$(mktemp)
+	curl $MIRROR/dists/sid/main/source/Sources.xz > $TMPFILE
+	CSVFILE=$(mktemp)
+	(xzcat $TMPFILE | egrep "(^Package:|^Version:)" | sed -s "s#^Version: ##g; s#Package: ##g; s#\n# #g"| while read PKG ; do read VERSION ; echo "$PKG,$VERSION" ; done) > $CSVFILE
+	sqlite3 -csv -init $INIT ${PACKAGES_DB} "DELETE from sources"
+	echo ".import $CSVFILE sources" | sqlite3 -csv -init $INIT ${PACKAGES_DB}
+	# update amount of available packages (for doing statistics later)
+	P_IN_SOURCES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | sort -u | wc -l)
+	sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_stats VALUES (\"sid\", \"${P_IN_SOURCES}\")"
+	rm $CSVFILE # $TMPFILE is still being used
+}
+
+set +x
+if [ $1 = "unknown" ] ; then
+	update_sources_table
+	AMOUNT=$2
+	REAL_AMOUNT=0
+	GUESSES=$(echo "${AMOUNT}*3" | bc)
+	PACKAGES=""
+	CANDIDATES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" |  cut -d " " -f2 | egrep -v "^(linux|cups|zurl)$" | sort -R | head -$GUESSES | xargs echo)
+	for PKG in $CANDIDATES ; do
+		if [ $REAL_AMOUNT -eq $AMOUNT ] ; then
+			continue
+		fi
+		RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"")
+		if [ "$RESULT" = "" ] ; then
+			PACKAGES="${PACKAGES} $PKG"
+		fi
+	done
+elif [ $1 = "known" ] ; then
+	update_sources_table
+	AMOUNT=$2
+	PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT DISTINCT source_packages.name FROM source_packages,sources WHERE sources.version IN (SELECT version FROM sources WHERE name=source_packages.name ORDER by sources.version DESC LIMIT 1) AND (( source_packages.status = 'unreproducible' OR source_packages.status = 'FTBFS') AND source_packages.name = sources.name AND source_packages.version < sources.version) ORDER BY source_packages.build_date LIMIT $AMOUNT" | xargs -r echo)
+else
+	# CANDIDATES is defined in that file
+	. /srv/jenkins/bin/reproducible_candidates.sh
+	PACKAGES=""
+	AMOUNT=$2
+	REAL_AMOUNT=0
+	for i in $(seq 0 ${#CANDIDATES[@]}) ; do
+		if [ $REAL_AMOUNT -eq $AMOUNT ] ; then
+			continue
+		fi
+		PKG=${CANDIDATES[$i]}
+		RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"")
+		if [ "$RESULT" = "" ] ; then
+			PACKAGES="${PACKAGES} $PKG"
+			let "REAL_AMOUNT=REAL_AMOUNT+1"
+		fi
+	done
+fi
+AMOUNT=0
+for PKG in $PACKAGES ; do
+	let "AMOUNT=AMOUNT+1"
+done
+echo "============================================================================="
+echo "The following $AMOUNT source packages will be scheduled: ${PACKAGES}"
+echo "============================================================================="
+echo
+rm -f $TMPFILE
+
diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh
index 996eaefd..4efb0a3e 100755
--- a/bin/reproducible_stats.sh
+++ b/bin/reproducible_stats.sh
@@ -38,65 +38,108 @@ PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc)
 PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc)
 GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc)
 
-htmlecho() {
+write_index() {
 	echo "$1" >> index.html
 }
 
+mkdir -p /var/lib/jenkins/userContent/rb-pkg/
+write_pkg_frameset() {
+	FRAMESET="/var/lib/jenkins/userContent/rb-pkg/$1.html"
+	cat > $FRAMESET <<-EOF
+<!DOCTYPE html>
+<html>
+    <head>
+    </head>
+    <frameset framespacing="0" rows="20,*" frameborder="0" noresize>
+        <frame name="top" src="$1_navigation.html" target="top">
+        <frame name="main" src="$2" target="main">
+    </frameset>
+</html>
+EOF
+}
+
+init_navi_frame() {
+	NAVI="/var/lib/jenkins/userContent/rb-pkg/$1_navigation.html"
+	echo "<!DOCTYPE html><html><body><p>" > $NAVI
+	echo "<font size=+1>$1</font>: " >> $NAVI
+}
+
+append2navi_frame() {
+	echo "$2" >> $NAVI
+}
+
+finish_navi_frame() {
+	echo "</p></body></html>" >> $NAVI
+}
+
 link_packages() {
 	for PKG in $@ ; do
 		VERSION=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT version FROM source_packages WHERE name = \"$PKG\"")
 		# remove epoch
 		EVERSION=$(echo $VERSION | cut -d ":" -f2)
-		htmlecho $PKG
 		# FIXME: remove unused code once all diffp.log and pbuilder.log files are gone
 		if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] || [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.diffp.log" ] || [ -f "/var/lib/jenkins/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log" ] || [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] || [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ]; then
-			htmlecho "<font size='-2'> ("
+			write_index " <a href=\"$JENKINS_URL/userContent/rb-pkg/$PKG.html\">$PKG</a> "
+			MAINLINK=""
+			init_navi_frame "$PKG"
 			if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
-				htmlecho " <a href=\"$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo\">buildinfo</a> "
+				append2navi_frame " <a href=\"$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo\">buildinfo</a> "
+				MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo"
 			fi
 			if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then
-				htmlecho " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html\">debbindiff</a> "
+				append2navi_frame " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html\">debbindiff</a> "
+				MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html"
 			elif [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.diffp.log" ] ; then
-				htmlecho " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.diffp.log\">diffp</a> "
+				append2navi_frame " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.diffp.log\">diffp</a> "
+				MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.diffp.log"
 			fi
 			if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then
-				htmlecho " <a href=\"$JENKINS_URL/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log\">rbuild</a> "
+				append2navi_frame " <a href=\"$JENKINS_URL/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log\">rbuild</a> "
+				if [ "$MAINLINK" = "" ] ; then
+					MAINLINK="$JENKINS_URL/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log"
+				fi
 			fi
 			if [ -f "/var/lib/jenkins/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log" ] ; then
-				htmlecho " <a href=\"$JENKINS_URL/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log\">pbuilder</a> "
+				append2navi_frame " <a href=\"$JENKINS_URL/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log\">pbuilder</a> "
+				if [ "$MAINLINK" = "" ] ; then
+					MAINLINK="$JENKINS_URL/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log"
+				fi
 			fi
-			htmlecho ")</font> "
+			finish_navi_frame
+			write_pkg_frameset "$PKG" "$MAINLINK"
+		else
+			write_index " $PKG "
 		fi
 	done
 }
 
-htmlecho "<html><body>" > index.html
-htmlecho "<h2>Statistics for reproducible builds</h2>"
-htmlecho "<p>Results were obtained by <a href=\"$JENKINS_URL/view/reproducible\">several jobs running on jenkins.debian.net</a>. This page is updated after each job run.</p>"
-htmlecho "<p>$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!</p>"
-htmlecho "<p>$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly: <code>"
+write_index "<!DOCTYPE html><html><body>" > index.html
+write_index "<h2>Statistics for reproducible builds</h2>"
+write_index "<p>Results were obtained by <a href=\"$JENKINS_URL/view/reproducible\">several jobs running on jenkins.debian.net</a>. This page is updated after each job run.</p>"
+write_index "<p>$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!</p>"
+write_index "<p>$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly: <code>"
 link_packages $BAD
-htmlecho "</code></p>"
-htmlecho
-htmlecho "<p>$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source: <code>"
+write_index "</code></p>"
+write_index
+write_index "<p>$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source: <code>"
 link_packages $UGLY
-htmlecho "</code></p>"
+write_index "</code></p>"
 if [ $COUNT_SOURCELESS -gt 0 ] ; then
-	htmlecho "<p>$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages where the source could not be downloaded. <code>$SOURCELESS</code></p>"
+	write_index "<p>$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages where the source could not be downloaded. <code>$SOURCELESS</code></p>"
 fi
 if [ $COUNT_NOTFORUS -gt 0 ] ; then
-	htmlecho "<p>$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': <code>$NOTFORUS</code></p>"
+	write_index "<p>$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': <code>$NOTFORUS</code></p>"
 fi
 if [ $COUNT_BLACKLISTED -gt 0 ] ; then
-	htmlecho "<p>$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: <code>$BLACKLISTED</code></p>"
+	write_index "<p>$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: <code>$BLACKLISTED</code></p>"
 fi
-htmlecho "<p>$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly: <code>"
+write_index "<p>$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly: <code>"
 link_packages $GOOD
-htmlecho "</code></p>"
-htmlecho "<hr><p>Packages which failed to build reproducibly, sorted by Maintainers: and Uploaders: fields."
-htmlecho "<pre>$(echo $BAD | dd-list -i) </pre></p>"
-htmlecho "<hr><p><font size='-1'><a href=\"$JENKINS_URL/userContent/reproducible.html\">Static URL for this page.</a> Last modified: $(date)</font>"
-htmlecho "</p></body></html>"
+write_index "</code></p>"
+write_index "<hr><p>Packages which failed to build reproducibly, sorted by Maintainers: and Uploaders: fields."
+write_index "<pre>$(echo $BAD | dd-list -i) </pre></p>"
+write_index "<hr><p><font size='-1'><a href=\"$JENKINS_URL/userContent/reproducible.html\">Static URL for this page.</a> Last modified: $(date)</font>"
+write_index "</p></body></html>"
 
 # job output
 html2text index.html