#!/bin/bash
# Copyright 2014 Holger Levsen
# released under the GPLv=2
. /srv/jenkins/bin/common-functions.sh
common_init "$@"
set +x
# define db
PACKAGES_DB=/var/lib/jenkins/reproducible.db
INIT=/var/lib/jenkins/reproducible.init
if [ ! -f $PACKAGES_DB ] ; then
echo "$PACKAGES_DB doesn't exist, no stats possible."
exit 1
fi
declare -A GOOD
declare -A BAD
declare -A UGLY
declare -A SOURCELESS
declare -A NOTFORUS
LAST24="AND build_date > datetime('now', '-24 hours') "
LAST48="AND build_date > datetime('now', '-48 hours') "
SUITE=sid
AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT amount FROM source_stats WHERE suite = \"$SUITE\"" | xargs echo)
GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY name" | xargs echo)
GOOD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST24 ORDER BY name" | xargs echo)
GOOD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST48 ORDER BY name" | xargs echo)
COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"reproducible\"")
BAD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo)
BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo)
BAD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo)
COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"unreproducible\"")
UGLY["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY build_date DESC" | xargs echo)
UGLY["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST24 ORDER BY build_date DESC" | xargs echo)
UGLY["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST48 ORDER BY build_date DESC" | xargs echo)
COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"FTBFS\"")
SOURCELESS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY build_date DESC" | xargs echo)
SOURCELESS["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" $LAST24 ORDER BY build_date DESC" | xargs echo)
SOURCELESS["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" $LAST48 ORDER BY build_date DESC" | xargs echo)
COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"404\"" | xargs echo)
NOTFORUS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY build_date DESC" | xargs echo)
NOTFORUS["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" $LAST24 ORDER BY build_date DESC" | xargs echo)
NOTFORUS["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" $LAST48 ORDER BY build_date DESC" | xargs echo)
COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"not for us\"" | xargs echo)
BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"blacklisted\" ORDER BY name" | xargs echo)
COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"blacklisted\"" | xargs echo)
COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages")
PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc)
PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc)
PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc)
PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc)
PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc)
PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc)
GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc)
declare -A STAR
declare -A LINKTARGET
write_summary() {
echo "$1" >> $SUMMARY
}
mkdir -p /var/lib/jenkins/userContent/rb-pkg/
write_pkg_frameset() {
FRAMESET="/var/lib/jenkins/userContent/rb-pkg/$1.html"
cat > $FRAMESET <<-EOF
EOF
}
init_navi_frame() {
echo "" > $NAVI
echo "" >> $NAVI
echo "
" >> $NAVI
}
process_packages() {
for PKG in $@ ; do
RESULT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT build_date,version FROM source_packages WHERE name = \"$PKG\"")
BUILD_DATE=$(echo $RESULT|cut -d "|" -f1)
VERSION=$(echo $RESULT|cut -d "|" -f2)
# remove epoch
EVERSION=$(echo $VERSION | cut -d ":" -f2)
if $EXTRA_STAR && [ ! -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
STAR[$PKG]="β" # used to be a star...
fi
# only build $PKG pages if they don't exist or are older than $BUILD_DATE
NAVI="/var/lib/jenkins/userContent/rb-pkg/${PKG}_navigation.html"
FILE=$(find $(dirname $NAVI) -name $(basename $NAVI) ! -newermt "$BUILD_DATE" 2>/dev/null || true)
if [ ! -f $NAVI ] || [ "$FILE" != "" ] ; then
MAINLINK=""
init_navi_frame "$PKG" "$VERSION" "$BUILD_DATE"
if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
append2navi_frame " buildinfo "
MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo"
fi
if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then
append2navi_frame " debbindiff "
MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html"
fi
RBUILD_LOG="rbuild/${PKG}_${EVERSION}.rbuild.log"
if [ -f "/var/lib/jenkins/userContent/${RBUILD_LOG}" ] ; then
SIZE=$(du -sh "/var/lib/jenkins/userContent/${RBUILD_LOG}" |cut -f1)
append2navi_frame " rbuild ($SIZE) "
if [ "$MAINLINK" = "" ] ; then
MAINLINK="$JENKINS_URL/userContent/${RBUILD_LOG}"
fi
fi
append2navi_frame " PTS "
append2navi_frame " BTS "
append2navi_frame " sources "
append2navi_frame " debian/rules "
finish_navi_frame
write_pkg_frameset "$PKG" "$MAINLINK"
fi
if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then
LINKTARGET[$PKG]="$PKG${STAR[$PKG]}"
else
LINKTARGET[$PKG]="$PKG"
fi
done
}
link_packages() {
for PKG in $@ ; do
write_summary " ${LINKTARGET[$PKG]} "
done
}
echo "Processing packages... this will take a while."
EXTRA_STAR=true
process_packages ${BAD["all"]}
EXTRA_STAR=false
process_packages ${UGLY["all"]} ${GOOD["all"]}
SPOKENTARGET["all"]="all tested packages"
SPOKENTARGET["last_24h"]="packages tested in the last 24h"
SPOKENTARGET["last_48"]="packages tested in the last 48h"
MAINVIEW="last_24h"
ALLVIEWS="all last_24h last_48h"
for VIEW in $ALLVIEWS ; do
SUMMARY=index_${VIEW}.html
echo "Starting to write $SUMMARY page."
rm -f $SUMMARY
write_summary ""
write_summary ""
write_summary ""
write_summary "
Statistics for reproducible builds
"
if [ "$VIEW" = "$MAINVIEW" ] ; then
write_summary "
$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable! Join #debian-reproducible on OFTC to get support for making sure your packages build reproducibly too!
"
fi
write_summary "
Other views for the build results:
"
for TARGET in $ALLVIEWS ; do
if [ "$TARGET" = "$VIEW" ] ; then
continue
fi
write_summary "