diff options
Diffstat (limited to 'bin')
| -rwxr-xr-x | bin/reproducible_scheduler.sh | 84 | ||||
| -rwxr-xr-x | bin/reproducible_stats.sh | 97 |
2 files changed, 154 insertions, 27 deletions
diff --git a/bin/reproducible_scheduler.sh b/bin/reproducible_scheduler.sh new file mode 100755 index 00000000..b9fc0c62 --- /dev/null +++ b/bin/reproducible_scheduler.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# Copyright 2014 Holger Levsen <holger@layer-acht.org> +# released under the GPLv=2 + +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# define db +PACKAGES_DB=/var/lib/jenkins/reproducible.db +INIT=/var/lib/jenkins/reproducible.init +if [ ! -f $PACKAGES_DB ] ; then + echo "$PACKAGES_DB doesn't exist, no builds possible." + exit 1 +fi + +# this needs sid entries in sources.list: +grep deb-src /etc/apt/sources.list | grep sid +# try apt-get update twice, else fail gracefully, aka not. +sudo apt-get update || ( sleep $(( $RANDOM % 70 + 30 )) ; sudo apt-get update || true ) + +# update sources table in db +update_sources_table() { + TMPFILE=$(mktemp) + curl $MIRROR/dists/sid/main/source/Sources.xz > $TMPFILE + CSVFILE=$(mktemp) + (xzcat $TMPFILE | egrep "(^Package:|^Version:)" | sed -s "s#^Version: ##g; s#Package: ##g; s#\n# #g"| while read PKG ; do read VERSION ; echo "$PKG,$VERSION" ; done) > $CSVFILE + sqlite3 -csv -init $INIT ${PACKAGES_DB} "DELETE from sources" + echo ".import $CSVFILE sources" | sqlite3 -csv -init $INIT ${PACKAGES_DB} + # update amount of available packages (for doing statistics later) + P_IN_SOURCES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | sort -u | wc -l) + sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_stats VALUES (\"sid\", \"${P_IN_SOURCES}\")" + rm $CSVFILE # $TMPFILE is still being used +} + +set +x +if [ $1 = "unknown" ] ; then + update_sources_table + AMOUNT=$2 + REAL_AMOUNT=0 + GUESSES=$(echo "${AMOUNT}*3" | bc) + PACKAGES="" + CANDIDATES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | egrep -v "^(linux|cups|zurl)$" | sort -R | head -$GUESSES | xargs echo) + for PKG in $CANDIDATES ; do + if [ $REAL_AMOUNT -eq $AMOUNT ] ; then + continue + fi + RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"") + if [ "$RESULT" = "" ] ; then + PACKAGES="${PACKAGES} $PKG" + fi + done +elif [ $1 = "known" ] ; then + update_sources_table + AMOUNT=$2 + PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT DISTINCT source_packages.name FROM source_packages,sources WHERE sources.version IN (SELECT version FROM sources WHERE name=source_packages.name ORDER by sources.version DESC LIMIT 1) AND (( source_packages.status = 'unreproducible' OR source_packages.status = 'FTBFS') AND source_packages.name = sources.name AND source_packages.version < sources.version) ORDER BY source_packages.build_date LIMIT $AMOUNT" | xargs -r echo) +else + # CANDIDATES is defined in that file + . /srv/jenkins/bin/reproducible_candidates.sh + PACKAGES="" + AMOUNT=$2 + REAL_AMOUNT=0 + for i in $(seq 0 ${#CANDIDATES[@]}) ; do + if [ $REAL_AMOUNT -eq $AMOUNT ] ; then + continue + fi + PKG=${CANDIDATES[$i]} + RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"") + if [ "$RESULT" = "" ] ; then + PACKAGES="${PACKAGES} $PKG" + let "REAL_AMOUNT=REAL_AMOUNT+1" + fi + done +fi +AMOUNT=0 +for PKG in $PACKAGES ; do + let "AMOUNT=AMOUNT+1" +done +echo "=============================================================================" +echo "The following $AMOUNT source packages will be scheduled: ${PACKAGES}" +echo "=============================================================================" +echo +rm -f $TMPFILE + diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh index 996eaefd..4efb0a3e 100755 --- a/bin/reproducible_stats.sh +++ b/bin/reproducible_stats.sh @@ -38,65 +38,108 @@ PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc) -htmlecho() { +write_index() { echo "$1" >> index.html } +mkdir -p /var/lib/jenkins/userContent/rb-pkg/ +write_pkg_frameset() { + FRAMESET="/var/lib/jenkins/userContent/rb-pkg/$1.html" + cat > $FRAMESET <<-EOF +<!DOCTYPE html> +<html> + <head> + </head> + <frameset framespacing="0" rows="20,*" frameborder="0" noresize> + <frame name="top" src="$1_navigation.html" target="top"> + <frame name="main" src="$2" target="main"> + </frameset> +</html> +EOF +} + +init_navi_frame() { + NAVI="/var/lib/jenkins/userContent/rb-pkg/$1_navigation.html" + echo "<!DOCTYPE html><html><body><p>" > $NAVI + echo "<font size=+1>$1</font>: " >> $NAVI +} + +append2navi_frame() { + echo "$2" >> $NAVI +} + +finish_navi_frame() { + echo "</p></body></html>" >> $NAVI +} + link_packages() { for PKG in $@ ; do VERSION=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT version FROM source_packages WHERE name = \"$PKG\"") # remove epoch EVERSION=$(echo $VERSION | cut -d ":" -f2) - htmlecho $PKG # FIXME: remove unused code once all diffp.log and pbuilder.log files are gone if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] || [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.diffp.log" ] || [ -f "/var/lib/jenkins/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log" ] || [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] || [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ]; then - htmlecho "<font size='-2'> (" + write_index " <a href=\"$JENKINS_URL/userContent/rb-pkg/$PKG.html\">$PKG</a> " + MAINLINK="" + init_navi_frame "$PKG" if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then - htmlecho " <a href=\"$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo\">buildinfo</a> " + append2navi_frame " <a href=\"$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo\">buildinfo</a> " + MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" fi if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then - htmlecho " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html\">debbindiff</a> " + append2navi_frame " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html\">debbindiff</a> " + MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" elif [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.diffp.log" ] ; then - htmlecho " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.diffp.log\">diffp</a> " + append2navi_frame " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.diffp.log\">diffp</a> " + MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.diffp.log" fi if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then - htmlecho " <a href=\"$JENKINS_URL/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log\">rbuild</a> " + append2navi_frame " <a href=\"$JENKINS_URL/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log\">rbuild</a> " + if [ "$MAINLINK" = "" ] ; then + MAINLINK="$JENKINS_URL/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" + fi fi if [ -f "/var/lib/jenkins/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log" ] ; then - htmlecho " <a href=\"$JENKINS_URL/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log\">pbuilder</a> " + append2navi_frame " <a href=\"$JENKINS_URL/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log\">pbuilder</a> " + if [ "$MAINLINK" = "" ] ; then + MAINLINK="$JENKINS_URL/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log" + fi fi - htmlecho ")</font> " + finish_navi_frame + write_pkg_frameset "$PKG" "$MAINLINK" + else + write_index " $PKG " fi done } -htmlecho "<html><body>" > index.html -htmlecho "<h2>Statistics for reproducible builds</h2>" -htmlecho "<p>Results were obtained by <a href=\"$JENKINS_URL/view/reproducible\">several jobs running on jenkins.debian.net</a>. This page is updated after each job run.</p>" -htmlecho "<p>$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!</p>" -htmlecho "<p>$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly: <code>" +write_index "<!DOCTYPE html><html><body>" > index.html +write_index "<h2>Statistics for reproducible builds</h2>" +write_index "<p>Results were obtained by <a href=\"$JENKINS_URL/view/reproducible\">several jobs running on jenkins.debian.net</a>. This page is updated after each job run.</p>" +write_index "<p>$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!</p>" +write_index "<p>$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly: <code>" link_packages $BAD -htmlecho "</code></p>" -htmlecho -htmlecho "<p>$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source: <code>" +write_index "</code></p>" +write_index +write_index "<p>$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source: <code>" link_packages $UGLY -htmlecho "</code></p>" +write_index "</code></p>" if [ $COUNT_SOURCELESS -gt 0 ] ; then - htmlecho "<p>$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages where the source could not be downloaded. <code>$SOURCELESS</code></p>" + write_index "<p>$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages where the source could not be downloaded. <code>$SOURCELESS</code></p>" fi if [ $COUNT_NOTFORUS -gt 0 ] ; then - htmlecho "<p>$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': <code>$NOTFORUS</code></p>" + write_index "<p>$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': <code>$NOTFORUS</code></p>" fi if [ $COUNT_BLACKLISTED -gt 0 ] ; then - htmlecho "<p>$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: <code>$BLACKLISTED</code></p>" + write_index "<p>$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: <code>$BLACKLISTED</code></p>" fi -htmlecho "<p>$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly: <code>" +write_index "<p>$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly: <code>" link_packages $GOOD -htmlecho "</code></p>" -htmlecho "<hr><p>Packages which failed to build reproducibly, sorted by Maintainers: and Uploaders: fields." -htmlecho "<pre>$(echo $BAD | dd-list -i) </pre></p>" -htmlecho "<hr><p><font size='-1'><a href=\"$JENKINS_URL/userContent/reproducible.html\">Static URL for this page.</a> Last modified: $(date)</font>" -htmlecho "</p></body></html>" +write_index "</code></p>" +write_index "<hr><p>Packages which failed to build reproducibly, sorted by Maintainers: and Uploaders: fields." +write_index "<pre>$(echo $BAD | dd-list -i) </pre></p>" +write_index "<hr><p><font size='-1'><a href=\"$JENKINS_URL/userContent/reproducible.html\">Static URL for this page.</a> Last modified: $(date)</font>" +write_index "</p></body></html>" # job output html2text index.html |