#!/bin/bash # Copyright 2014 Holger Levsen # released under the GPLv=2 . /srv/jenkins/bin/common-functions.sh common_init "$@" set +x # define db PACKAGES_DB=/var/lib/jenkins/reproducible.db INIT=/var/lib/jenkins/reproducible.init if [ ! -f $PACKAGES_DB ] ; then echo "$PACKAGES_DB doesn't exist, no stats possible." exit 1 fi declare -A GOOD declare -A BAD declare -A UGLY declare -A SOURCELESS declare -A NOTFORUS LAST24="AND build_date > datetime('now', '-24 hours') " LAST48="AND build_date > datetime('now', '-48 hours') " SUITE=sid AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT amount FROM source_stats WHERE suite = \"$SUITE\"" | xargs echo) GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY name" | xargs echo) GOOD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST24 ORDER BY name" | xargs echo) GOOD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST48 ORDER BY name" | xargs echo) COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"reproducible\"") BAD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo) BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"unreproducible\"") UGLY["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY build_date DESC" | xargs echo) UGLY["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST24 ORDER BY build_date DESC" | xargs echo) UGLY["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST48 ORDER BY build_date DESC" | xargs echo) COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"FTBFS\"") SOURCELESS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY build_date DESC" | xargs echo) SOURCELESS["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" $LAST24 ORDER BY build_date DESC" | xargs echo) SOURCELESS["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" $LAST48 ORDER BY build_date DESC" | xargs echo) COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"404\"" | xargs echo) NOTFORUS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY build_date DESC" | xargs echo) NOTFORUS["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" $LAST24 ORDER BY build_date DESC" | xargs echo) NOTFORUS["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" $LAST48 ORDER BY build_date DESC" | xargs echo) COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"not for us\"" | xargs echo) BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"blacklisted\" ORDER BY name" | xargs echo) COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"blacklisted\"" | xargs echo) COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages") PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc) PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc) PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc) declare -A STAR declare -A LINKTARGET write_summary() { echo "$1" >> $SUMMARY echo "$1" | html2text } mkdir -p /var/lib/jenkins/userContent/rb-pkg/ write_pkg_frameset() { FRAMESET="/var/lib/jenkins/userContent/rb-pkg/$1.html" cat > $FRAMESET <<-EOF EOF } init_navi_frame() { echo "" > $NAVI echo "" >> $NAVI echo "
$1 " >> $NAVI echo "($2) " >> $NAVI echo "at $3: " >> $NAVI } append2navi_frame() { echo "$1" >> $NAVI } finish_navi_frame() { echo "stats for reproducible builds
" >> $NAVI } process_packages() { for PKG in $@ ; do RESULT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT build_date,version FROM source_packages WHERE name = \"$PKG\"") BUILD_DATE=$(echo $RESULT|cut -d "|" -f1) VERSION=$(echo $RESULT|cut -d "|" -f2) # remove epoch EVERSION=$(echo $VERSION | cut -d ":" -f2) if $EXTRA_STAR && [ ! -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then STAR[$PKG]="β" # used to be a star... fi # only build $PKG pages if they don't exist or are older than $BUILD_DATE NAVI="/var/lib/jenkins/userContent/rb-pkg/${PKG}_navigation.html" FILE=$(find $(dirname $NAVI) -name $(basename $NAVI) ! -newermt "$BUILD_DATE" 2>/dev/null || true) if [ ! -f $NAVI ] || [ "$FILE" != "" ] ; then MAINLINK="" init_navi_frame "$PKG" "$VERSION" "$BUILD_DATE" if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then append2navi_frame " buildinfo " MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" fi if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then append2navi_frame " debbindiff " MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" fi RBUILD_LOG="rbuild/${PKG}_${EVERSION}.rbuild.log" if [ -f "/var/lib/jenkins/userContent/${RBUILD_LOG}" ] ; then SIZE=$(du -sh "/var/lib/jenkins/userContent/${RBUILD_LOG}" |cut -f1) append2navi_frame " rbuild ($SIZE) " if [ "$MAINLINK" = "" ] ; then MAINLINK="$JENKINS_URL/userContent/${RBUILD_LOG}" fi fi append2navi_frame " PTS " append2navi_frame " BTS " append2navi_frame " sources " append2navi_frame " debian/rules " finish_navi_frame write_pkg_frameset "$PKG" "$MAINLINK" fi if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then LINKTARGET[$PKG]="$PKG${STAR[$PKG]}" else LINKTARGET[$PKG]="$PKG" fi done } link_packages() { for PKG in $@ ; do write_summary " ${LINKTARGET[$PKG]} " done } echo "Processing packages... this will take a while." EXTRA_STAR=true process_packages ${BAD["all"]} EXTRA_STAR=false process_packages ${UGLY["all"]} ${GOOD["all"]} MAINVIEW="last_24h" ALLVIEWS="all last_24h last_48h" for VIEW in $ALLVIEWS ; do echo "Starting to write statistics index_$VIEW page." echo SUMMARY=index_${VIEW}.html rm -f $SUMMARY write_summary "" write_summary "" write_summary "" write_summary "

Statistics for reproducible builds

" write_summary "

This page is updated every three hours. Results are obtained from several build jobs running on jenkins.debian.net. Thanks to Profitbricks for donating the virtual machine it's running on!

" write_summary "

$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable! Join #debian-reproducible on OFTC to get support for making sure your packages build reproducibly too!

" write_summary "

$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly: " link_packages ${BAD[$VIEW]} write_summary "

" write_summary "

A β sign after a package name indicates that no .buildinfo file was generated.

" write_summary write_summary "

$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source: " link_packages ${UGLY[$VIEW]} write_summary "

" if [ $COUNT_SOURCELESS -gt 0 ] ; then write_summary "

$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages where the source could not be downloaded. ${SOURCELESS[$VIEW]}

" fi if [ $COUNT_NOTFORUS -gt 0 ] ; then write_summary "

$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': ${NOTFORUS[$VIEW]}

" fi if [ $COUNT_BLACKLISTED -gt 0 ] ; then write_summary "

$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: $BLACKLISTED

" fi write_summary "

$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly: " link_packages ${GOOD[$VIEW]} write_summary "

" write_summary "

Packages which failed to build reproducibly, sorted by Maintainers: and Uploaders: fields

" write_summary "

$(echo ${BAD[$VIEW]} | dd-list -i)

" write_summary "

Static URL for this page. Last modified: $(date). Copyright 2014 Holger Levsen, GPL-2 licensed. About jenkins.debian.net" write_summary "

" done echo # job output cp $SUMMARY /var/lib/jenkins/userContent/ if [ "$VIEW" = "$MAINVIEW" ] ; then cp $SUMMARY reproducible.html fi rm $SUMMARY