summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHolger Levsen <holger@layer-acht.org>2014-10-05 20:54:36 +0200
committerHolger Levsen <holger@layer-acht.org>2014-10-05 20:54:36 +0200
commit3e4905c545650a1e80a8e22b67c52deafa2667e0 (patch)
tree59d31bb72e351403a8949d471e9e1f47c6764bc5
parent50d1b5e106f60aeca27cab58095c015546e6b56e (diff)
downloadjenkins.debian.net-3e4905c545650a1e80a8e22b67c52deafa2667e0.tar.xz
reproducible: use html framesets to provide easy+fast access to files
-rwxr-xr-xbin/reproducible_scheduler.sh84
-rwxr-xr-xbin/reproducible_stats.sh97
2 files changed, 154 insertions, 27 deletions
diff --git a/bin/reproducible_scheduler.sh b/bin/reproducible_scheduler.sh
new file mode 100755
index 00000000..b9fc0c62
--- /dev/null
+++ b/bin/reproducible_scheduler.sh
@@ -0,0 +1,84 @@
+#!/bin/bash
+
+# Copyright 2014 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv=2
+
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# define db
+PACKAGES_DB=/var/lib/jenkins/reproducible.db
+INIT=/var/lib/jenkins/reproducible.init
+if [ ! -f $PACKAGES_DB ] ; then
+ echo "$PACKAGES_DB doesn't exist, no builds possible."
+ exit 1
+fi
+
+# this needs sid entries in sources.list:
+grep deb-src /etc/apt/sources.list | grep sid
+# try apt-get update twice, else fail gracefully, aka not.
+sudo apt-get update || ( sleep $(( $RANDOM % 70 + 30 )) ; sudo apt-get update || true )
+
+# update sources table in db
+update_sources_table() {
+ TMPFILE=$(mktemp)
+ curl $MIRROR/dists/sid/main/source/Sources.xz > $TMPFILE
+ CSVFILE=$(mktemp)
+ (xzcat $TMPFILE | egrep "(^Package:|^Version:)" | sed -s "s#^Version: ##g; s#Package: ##g; s#\n# #g"| while read PKG ; do read VERSION ; echo "$PKG,$VERSION" ; done) > $CSVFILE
+ sqlite3 -csv -init $INIT ${PACKAGES_DB} "DELETE from sources"
+ echo ".import $CSVFILE sources" | sqlite3 -csv -init $INIT ${PACKAGES_DB}
+ # update amount of available packages (for doing statistics later)
+ P_IN_SOURCES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | sort -u | wc -l)
+ sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_stats VALUES (\"sid\", \"${P_IN_SOURCES}\")"
+ rm $CSVFILE # $TMPFILE is still being used
+}
+
+set +x
+if [ $1 = "unknown" ] ; then
+ update_sources_table
+ AMOUNT=$2
+ REAL_AMOUNT=0
+ GUESSES=$(echo "${AMOUNT}*3" | bc)
+ PACKAGES=""
+ CANDIDATES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | egrep -v "^(linux|cups|zurl)$" | sort -R | head -$GUESSES | xargs echo)
+ for PKG in $CANDIDATES ; do
+ if [ $REAL_AMOUNT -eq $AMOUNT ] ; then
+ continue
+ fi
+ RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"")
+ if [ "$RESULT" = "" ] ; then
+ PACKAGES="${PACKAGES} $PKG"
+ fi
+ done
+elif [ $1 = "known" ] ; then
+ update_sources_table
+ AMOUNT=$2
+ PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT DISTINCT source_packages.name FROM source_packages,sources WHERE sources.version IN (SELECT version FROM sources WHERE name=source_packages.name ORDER by sources.version DESC LIMIT 1) AND (( source_packages.status = 'unreproducible' OR source_packages.status = 'FTBFS') AND source_packages.name = sources.name AND source_packages.version < sources.version) ORDER BY source_packages.build_date LIMIT $AMOUNT" | xargs -r echo)
+else
+ # CANDIDATES is defined in that file
+ . /srv/jenkins/bin/reproducible_candidates.sh
+ PACKAGES=""
+ AMOUNT=$2
+ REAL_AMOUNT=0
+ for i in $(seq 0 ${#CANDIDATES[@]}) ; do
+ if [ $REAL_AMOUNT -eq $AMOUNT ] ; then
+ continue
+ fi
+ PKG=${CANDIDATES[$i]}
+ RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"")
+ if [ "$RESULT" = "" ] ; then
+ PACKAGES="${PACKAGES} $PKG"
+ let "REAL_AMOUNT=REAL_AMOUNT+1"
+ fi
+ done
+fi
+AMOUNT=0
+for PKG in $PACKAGES ; do
+ let "AMOUNT=AMOUNT+1"
+done
+echo "============================================================================="
+echo "The following $AMOUNT source packages will be scheduled: ${PACKAGES}"
+echo "============================================================================="
+echo
+rm -f $TMPFILE
+
diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh
index 996eaefd..4efb0a3e 100755
--- a/bin/reproducible_stats.sh
+++ b/bin/reproducible_stats.sh
@@ -38,65 +38,108 @@ PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc)
PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc)
GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc)
-htmlecho() {
+write_index() {
echo "$1" >> index.html
}
+mkdir -p /var/lib/jenkins/userContent/rb-pkg/
+write_pkg_frameset() {
+ FRAMESET="/var/lib/jenkins/userContent/rb-pkg/$1.html"
+ cat > $FRAMESET <<-EOF
+<!DOCTYPE html>
+<html>
+ <head>
+ </head>
+ <frameset framespacing="0" rows="20,*" frameborder="0" noresize>
+ <frame name="top" src="$1_navigation.html" target="top">
+ <frame name="main" src="$2" target="main">
+ </frameset>
+</html>
+EOF
+}
+
+init_navi_frame() {
+ NAVI="/var/lib/jenkins/userContent/rb-pkg/$1_navigation.html"
+ echo "<!DOCTYPE html><html><body><p>" > $NAVI
+ echo "<font size=+1>$1</font>: " >> $NAVI
+}
+
+append2navi_frame() {
+ echo "$2" >> $NAVI
+}
+
+finish_navi_frame() {
+ echo "</p></body></html>" >> $NAVI
+}
+
link_packages() {
for PKG in $@ ; do
VERSION=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT version FROM source_packages WHERE name = \"$PKG\"")
# remove epoch
EVERSION=$(echo $VERSION | cut -d ":" -f2)
- htmlecho $PKG
# FIXME: remove unused code once all diffp.log and pbuilder.log files are gone
if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] || [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.diffp.log" ] || [ -f "/var/lib/jenkins/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log" ] || [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] || [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ]; then
- htmlecho "<font size='-2'> ("
+ write_index " <a href=\"$JENKINS_URL/userContent/rb-pkg/$PKG.html\">$PKG</a> "
+ MAINLINK=""
+ init_navi_frame "$PKG"
if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
- htmlecho " <a href=\"$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo\">buildinfo</a> "
+ append2navi_frame " <a href=\"$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo\">buildinfo</a> "
+ MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo"
fi
if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then
- htmlecho " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html\">debbindiff</a> "
+ append2navi_frame " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html\">debbindiff</a> "
+ MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html"
elif [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.diffp.log" ] ; then
- htmlecho " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.diffp.log\">diffp</a> "
+ append2navi_frame " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.diffp.log\">diffp</a> "
+ MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.diffp.log"
fi
if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then
- htmlecho " <a href=\"$JENKINS_URL/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log\">rbuild</a> "
+ append2navi_frame " <a href=\"$JENKINS_URL/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log\">rbuild</a> "
+ if [ "$MAINLINK" = "" ] ; then
+ MAINLINK="$JENKINS_URL/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log"
+ fi
fi
if [ -f "/var/lib/jenkins/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log" ] ; then
- htmlecho " <a href=\"$JENKINS_URL/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log\">pbuilder</a> "
+ append2navi_frame " <a href=\"$JENKINS_URL/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log\">pbuilder</a> "
+ if [ "$MAINLINK" = "" ] ; then
+ MAINLINK="$JENKINS_URL/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log"
+ fi
fi
- htmlecho ")</font> "
+ finish_navi_frame
+ write_pkg_frameset "$PKG" "$MAINLINK"
+ else
+ write_index " $PKG "
fi
done
}
-htmlecho "<html><body>" > index.html
-htmlecho "<h2>Statistics for reproducible builds</h2>"
-htmlecho "<p>Results were obtained by <a href=\"$JENKINS_URL/view/reproducible\">several jobs running on jenkins.debian.net</a>. This page is updated after each job run.</p>"
-htmlecho "<p>$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!</p>"
-htmlecho "<p>$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly: <code>"
+write_index "<!DOCTYPE html><html><body>" > index.html
+write_index "<h2>Statistics for reproducible builds</h2>"
+write_index "<p>Results were obtained by <a href=\"$JENKINS_URL/view/reproducible\">several jobs running on jenkins.debian.net</a>. This page is updated after each job run.</p>"
+write_index "<p>$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!</p>"
+write_index "<p>$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly: <code>"
link_packages $BAD
-htmlecho "</code></p>"
-htmlecho
-htmlecho "<p>$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source: <code>"
+write_index "</code></p>"
+write_index
+write_index "<p>$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source: <code>"
link_packages $UGLY
-htmlecho "</code></p>"
+write_index "</code></p>"
if [ $COUNT_SOURCELESS -gt 0 ] ; then
- htmlecho "<p>$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages where the source could not be downloaded. <code>$SOURCELESS</code></p>"
+ write_index "<p>$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages where the source could not be downloaded. <code>$SOURCELESS</code></p>"
fi
if [ $COUNT_NOTFORUS -gt 0 ] ; then
- htmlecho "<p>$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': <code>$NOTFORUS</code></p>"
+ write_index "<p>$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': <code>$NOTFORUS</code></p>"
fi
if [ $COUNT_BLACKLISTED -gt 0 ] ; then
- htmlecho "<p>$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: <code>$BLACKLISTED</code></p>"
+ write_index "<p>$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: <code>$BLACKLISTED</code></p>"
fi
-htmlecho "<p>$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly: <code>"
+write_index "<p>$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly: <code>"
link_packages $GOOD
-htmlecho "</code></p>"
-htmlecho "<hr><p>Packages which failed to build reproducibly, sorted by Maintainers: and Uploaders: fields."
-htmlecho "<pre>$(echo $BAD | dd-list -i) </pre></p>"
-htmlecho "<hr><p><font size='-1'><a href=\"$JENKINS_URL/userContent/reproducible.html\">Static URL for this page.</a> Last modified: $(date)</font>"
-htmlecho "</p></body></html>"
+write_index "</code></p>"
+write_index "<hr><p>Packages which failed to build reproducibly, sorted by Maintainers: and Uploaders: fields."
+write_index "<pre>$(echo $BAD | dd-list -i) </pre></p>"
+write_index "<hr><p><font size='-1'><a href=\"$JENKINS_URL/userContent/reproducible.html\">Static URL for this page.</a> Last modified: $(date)</font>"
+write_index "</p></body></html>"
# job output
html2text index.html