From 3e4905c545650a1e80a8e22b67c52deafa2667e0 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Sun, 5 Oct 2014 20:54:36 +0200 Subject: reproducible: use html framesets to provide easy+fast access to files --- bin/reproducible_scheduler.sh | 84 +++++++++++++++++++++++++++++++++++++ bin/reproducible_stats.sh | 97 +++++++++++++++++++++++++++++++------------ 2 files changed, 154 insertions(+), 27 deletions(-) create mode 100755 bin/reproducible_scheduler.sh (limited to 'bin') diff --git a/bin/reproducible_scheduler.sh b/bin/reproducible_scheduler.sh new file mode 100755 index 00000000..b9fc0c62 --- /dev/null +++ b/bin/reproducible_scheduler.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# Copyright 2014 Holger Levsen +# released under the GPLv=2 + +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# define db +PACKAGES_DB=/var/lib/jenkins/reproducible.db +INIT=/var/lib/jenkins/reproducible.init +if [ ! -f $PACKAGES_DB ] ; then + echo "$PACKAGES_DB doesn't exist, no builds possible." + exit 1 +fi + +# this needs sid entries in sources.list: +grep deb-src /etc/apt/sources.list | grep sid +# try apt-get update twice, else fail gracefully, aka not. +sudo apt-get update || ( sleep $(( $RANDOM % 70 + 30 )) ; sudo apt-get update || true ) + +# update sources table in db +update_sources_table() { + TMPFILE=$(mktemp) + curl $MIRROR/dists/sid/main/source/Sources.xz > $TMPFILE + CSVFILE=$(mktemp) + (xzcat $TMPFILE | egrep "(^Package:|^Version:)" | sed -s "s#^Version: ##g; s#Package: ##g; s#\n# #g"| while read PKG ; do read VERSION ; echo "$PKG,$VERSION" ; done) > $CSVFILE + sqlite3 -csv -init $INIT ${PACKAGES_DB} "DELETE from sources" + echo ".import $CSVFILE sources" | sqlite3 -csv -init $INIT ${PACKAGES_DB} + # update amount of available packages (for doing statistics later) + P_IN_SOURCES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | sort -u | wc -l) + sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_stats VALUES (\"sid\", \"${P_IN_SOURCES}\")" + rm $CSVFILE # $TMPFILE is still being used +} + +set +x +if [ $1 = "unknown" ] ; then + update_sources_table + AMOUNT=$2 + REAL_AMOUNT=0 + GUESSES=$(echo "${AMOUNT}*3" | bc) + PACKAGES="" + CANDIDATES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | egrep -v "^(linux|cups|zurl)$" | sort -R | head -$GUESSES | xargs echo) + for PKG in $CANDIDATES ; do + if [ $REAL_AMOUNT -eq $AMOUNT ] ; then + continue + fi + RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"") + if [ "$RESULT" = "" ] ; then + PACKAGES="${PACKAGES} $PKG" + fi + done +elif [ $1 = "known" ] ; then + update_sources_table + AMOUNT=$2 + PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT DISTINCT source_packages.name FROM source_packages,sources WHERE sources.version IN (SELECT version FROM sources WHERE name=source_packages.name ORDER by sources.version DESC LIMIT 1) AND (( source_packages.status = 'unreproducible' OR source_packages.status = 'FTBFS') AND source_packages.name = sources.name AND source_packages.version < sources.version) ORDER BY source_packages.build_date LIMIT $AMOUNT" | xargs -r echo) +else + # CANDIDATES is defined in that file + . /srv/jenkins/bin/reproducible_candidates.sh + PACKAGES="" + AMOUNT=$2 + REAL_AMOUNT=0 + for i in $(seq 0 ${#CANDIDATES[@]}) ; do + if [ $REAL_AMOUNT -eq $AMOUNT ] ; then + continue + fi + PKG=${CANDIDATES[$i]} + RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"") + if [ "$RESULT" = "" ] ; then + PACKAGES="${PACKAGES} $PKG" + let "REAL_AMOUNT=REAL_AMOUNT+1" + fi + done +fi +AMOUNT=0 +for PKG in $PACKAGES ; do + let "AMOUNT=AMOUNT+1" +done +echo "=============================================================================" +echo "The following $AMOUNT source packages will be scheduled: ${PACKAGES}" +echo "=============================================================================" +echo +rm -f $TMPFILE + diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh index 996eaefd..4efb0a3e 100755 --- a/bin/reproducible_stats.sh +++ b/bin/reproducible_stats.sh @@ -38,65 +38,108 @@ PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc) -htmlecho() { +write_index() { echo "$1" >> index.html } +mkdir -p /var/lib/jenkins/userContent/rb-pkg/ +write_pkg_frameset() { + FRAMESET="/var/lib/jenkins/userContent/rb-pkg/$1.html" + cat > $FRAMESET <<-EOF + + + + + + + + + +EOF +} + +init_navi_frame() { + NAVI="/var/lib/jenkins/userContent/rb-pkg/$1_navigation.html" + echo "

" > $NAVI + echo "$1: " >> $NAVI +} + +append2navi_frame() { + echo "$2" >> $NAVI +} + +finish_navi_frame() { + echo "

" >> $NAVI +} + link_packages() { for PKG in $@ ; do VERSION=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT version FROM source_packages WHERE name = \"$PKG\"") # remove epoch EVERSION=$(echo $VERSION | cut -d ":" -f2) - htmlecho $PKG # FIXME: remove unused code once all diffp.log and pbuilder.log files are gone if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] || [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.diffp.log" ] || [ -f "/var/lib/jenkins/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log" ] || [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] || [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ]; then - htmlecho " (" + write_index " $PKG " + MAINLINK="" + init_navi_frame "$PKG" if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then - htmlecho " buildinfo " + append2navi_frame " buildinfo " + MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" fi if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then - htmlecho " debbindiff " + append2navi_frame " debbindiff " + MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" elif [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.diffp.log" ] ; then - htmlecho " diffp " + append2navi_frame " diffp " + MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.diffp.log" fi if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then - htmlecho " rbuild " + append2navi_frame " rbuild " + if [ "$MAINLINK" = "" ] ; then + MAINLINK="$JENKINS_URL/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" + fi fi if [ -f "/var/lib/jenkins/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log" ] ; then - htmlecho " pbuilder " + append2navi_frame " pbuilder " + if [ "$MAINLINK" = "" ] ; then + MAINLINK="$JENKINS_URL/userContent/pbuilder/${PKG}_${EVERSION}.pbuilder.log" + fi fi - htmlecho ") " + finish_navi_frame + write_pkg_frameset "$PKG" "$MAINLINK" + else + write_index " $PKG " fi done } -htmlecho "" > index.html -htmlecho "

Statistics for reproducible builds

" -htmlecho "

Results were obtained by several jobs running on jenkins.debian.net. This page is updated after each job run.

" -htmlecho "

$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!

" -htmlecho "

$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly: " +write_index "" > index.html +write_index "

Statistics for reproducible builds

" +write_index "

Results were obtained by several jobs running on jenkins.debian.net. This page is updated after each job run.

" +write_index "

$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!

" +write_index "

$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly: " link_packages $BAD -htmlecho "

" -htmlecho -htmlecho "

$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source: " +write_index "

" +write_index +write_index "

$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source: " link_packages $UGLY -htmlecho "

" +write_index "

" if [ $COUNT_SOURCELESS -gt 0 ] ; then - htmlecho "

$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages where the source could not be downloaded. $SOURCELESS

" + write_index "

$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages where the source could not be downloaded. $SOURCELESS

" fi if [ $COUNT_NOTFORUS -gt 0 ] ; then - htmlecho "

$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': $NOTFORUS

" + write_index "

$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': $NOTFORUS

" fi if [ $COUNT_BLACKLISTED -gt 0 ] ; then - htmlecho "

$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: $BLACKLISTED

" + write_index "

$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: $BLACKLISTED

" fi -htmlecho "

$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly: " +write_index "

$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly: " link_packages $GOOD -htmlecho "

" -htmlecho "

Packages which failed to build reproducibly, sorted by Maintainers: and Uploaders: fields." -htmlecho "

$(echo $BAD | dd-list -i)

" -htmlecho "

Static URL for this page. Last modified: $(date)" -htmlecho "

" +write_index "

" +write_index "

Packages which failed to build reproducibly, sorted by Maintainers: and Uploaders: fields." +write_index "

$(echo $BAD | dd-list -i)

" +write_index "

Static URL for this page. Last modified: $(date)" +write_index "

" # job output html2text index.html -- cgit v1.2.3-54-g00ecf