#!/bin/bash # Copyright 2014-2015 Holger Levsen # © 2015 Mattia Rizzolo # released under the GPLv=2 # # included by all reproducible_*.sh scripts # # define db PACKAGES_DB=/var/lib/jenkins/reproducible.db INIT=/var/lib/jenkins/reproducible.init if [ -f $PACKAGES_DB ] && [ -f $INIT ] ; then if [ -f ${PACKAGES_DB}.lock ] ; then for i in $(seq 0 100) ; do sleep 15 echo "sleeping 15s, $PACKAGES_DB is locked." if [ ! -f ${PACKAGES_DB}.lock ] ; then break fi done if [ -f ${PACKAGES_DB}.lock ] ; then echo "${PACKAGES_DB}.lock still exist, exiting." exit 1 fi fi elif [ ! -f ${PACKAGES_DB} ] ; then echo "Warning: $PACKAGES_DB doesn't exist, creating it now." /srv/jenkins/bin/reproducible_db_maintenance.py # 60 seconds timeout when trying to get a lock cat > $INIT <<-EOF .timeout 60000 EOF fi # common variables REPRODUCIBLE_URL=https://reproducible.debian.net DBDCHROOT_READLOCK=/var/lib/jenkins/reproducible-dbdchroot.readlock DBDCHROOT_WRITELOCK=/var/lib/jenkins/reproducible-dbdchroot.writelock # shop trailing slash JENKINS_URL=${JENKINS_URL:0:-1} # tested suites SUITES="sid experimental" # tested arches ARCHES="amd64" # we only need them for html creation but we cannot declare them in a function declare -A SPOKENTARGET declare -A LINKTARGET NOTES_PATH=/var/lib/jenkins/userContent/notes ISSUES_PATH=/var/lib/jenkins/userContent/issues RB_PATH=/var/lib/jenkins/userContent/rb-pkg/ mkdir -p $NOTES_PATH $ISSUES_PATH $RB_PATH # create subdirs for suites for i in $SUITES ; do mkdir -p /var/lib/jenkins/userContent/$i done # known package sets META_PKGSET[1]="essential" META_PKGSET[2]="required" META_PKGSET[3]="build-essential" META_PKGSET[4]="popcon_top1337-installed-sources" META_PKGSET[5]="installed_on_debian.org" META_PKGSET[6]="had_a_DSA" META_PKGSET[7]="gnome" META_PKGSET[8]="gnome_build-depends" META_PKGSET[9]="tails" META_PKGSET[10]="tails_build-depends" META_PKGSET[11]="grml" META_PKGSET[12]="grml_build-depends" META_PKGSET[13]="maint_pkg-perl-maintainers" init_html() { MAINVIEW="stats" ALLSTATES="reproducible FTBR FTBFS 404 not_for_us blacklisted" ALLVIEWS="issues notes no_notes scheduled last_24h last_48h all_abc dd-list pkg_sets suite_stats repo_stats stats" GLOBALVIEWS="issues notes no_notes repo_stats stats" SUITEVIEWS="dd-list suite_stats" SPOKENTARGET["issues"]="issues" SPOKENTARGET["notes"]="packages with notes" SPOKENTARGET["no_notes"]="packages without notes" SPOKENTARGET["scheduled"]="currently scheduled" SPOKENTARGET["last_24h"]="packages tested in the last 24h" SPOKENTARGET["last_48h"]="packages tested in the last 48h" SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" SPOKENTARGET["pkg_sets"]="package sets stats" SPOKENTARGET["suite_stats"]="$SUITE stats" SPOKENTARGET["repo_stats"]="repositories overview" SPOKENTARGET["stats"]="reproducible stats" # query some data we need everywhere AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(*) FROM sources WHERE suite=\"${SUITE}\"") COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite=\"${SUITE}\"") COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite=\"${SUITE}\" AND r.status=\"reproducible\"") PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc) if [ -z "${PERCENT_TOTAL}" ]; then PERCENT_TOTAL=0; fi PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc) if [ -z "${PERCENT_GOOD}" ]; then PERCENT_GOOD=0; fi } write_page() { echo "$1" >> $PAGE } set_icon() { # icons taken from tango-icon-theme (0.8.90-5) # licenced under http://creativecommons.org/licenses/publicdomain/ STATE_TARGET_NAME="$1" case "$1" in reproducible) ICON=weather-clear.png ;; unreproducible|FTBR) ICON=weather-showers-scattered.png ;; FTBFS) ICON=weather-storm.png ;; 404) ICON=weather-severe-alert.png ;; not_for_us|"not for us") ICON=weather-few-clouds-night.png STATE_TARGET_NAME="not_for_us" ;; blacklisted) ICON=error.png ;; *) ICON="" esac } write_icon() { # ICON and STATE_TARGET_NAME are set by set_icon() write_page "\"${STATE_TARGET_NAME}" } write_page_header() { rm -f $PAGE write_page "" write_page "" write_page "" write_page "$2" write_page "

$2

" if [ "$1" = "$MAINVIEW" ] || [ "$1" = "suite_stats" ] ; then write_page "

These pages contain results obtained from several jobs running on jenkins.debian.net. Thanks to Profitbricks for donating the virtual machine this is running on!

" fi if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] || [ "$1" = "stats" ] || [ "$1" = "suite_stats" ] ; then write_page "

$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $COUNT_GOOD packages ($PERCENT_GOOD%) could be built reproducible!" write_page " Join #debian-reproducible on OFTC to get support for making sure your packages build reproducibly too!" write_page "

" fi write_page "
  • Have a look at:
  • " for MY_STATE in $ALLSTATES ; do set_icon $MY_STATE write_page "
  • " write_icon write_page "
  • " done for TARGET in $ALLVIEWS ; do if [ "$TARGET" = "pkg_sets" ] && [ "$SUITE" = "experimental" ] ; then # no pkg_sets are tested in experimental continue fi SPOKEN_TARGET=${SPOKENTARGET[$TARGET]} BASEURL="/$SUITE/$ARCH" for i in $GLOBALVIEWS ; do if [ "$TARGET" = "$i" ] ; then BASEURL="" fi done for i in ${SUITEVIEWS} ; do if [ "$TARGET" = "$i" ] ; then BASEURL="/$SUITE" fi done write_page "
  • ${SPOKEN_TARGET}
  • " if [ "$TARGET" = "suite_stats" ] ; then for i in $SUITES ; do if [ "$i" != "$SUITE" ] ; then write_page "
  • suite: $i
  • " fi done fi done write_page "
" write_page "
" } write_page_footer() { write_page "

There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014-2015 Holger Levsen and others, GPL2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library.

" write_page "" } write_page_meta_sign() { write_page "

A package name displayed with a bold font is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue.

" } publish_page() { if [ "$1" = "" ] ; then if [ "$VIEW" = "$MAINVIEW" ] ; then cp $PAGE /var/lib/jenkins/userContent/reproducible.html fi TARGET=$PAGE else TARGET=$1/$PAGE fi cp $PAGE /var/lib/jenkins/userContent/$TARGET rm $PAGE echo "Enjoy $REPRODUCIBLE_URL/$TARGET" } set_package_class() { if [ -f ${NOTES_PATH}/${PKG}_note.html ] ; then CLASS="class=\"noted\"" else CLASS="class=\"package\"" fi } set_linktarget() { for PKG in $@ ; do if [ -f $RB_PATH/$SUITE/$ARCH/$PKG.html ] ; then set_package_class LINKTARGET[$PKG]="$PKG" else LINKTARGET[$PKG]="$PKG" fi done } link_packages() { for PKG in $@ ; do write_page " ${LINKTARGET[$PKG]}" done } gen_packages_html() { local suite="$1" shift CWD=$(pwd) cd /srv/jenkins/bin for (( i=1; i<$#+1; i=i+100 )) ; do string='[' delimiter='' for (( j=0; j<100; j++)) ; do item=$(( $j+$i )) if (( $item < $#+1 )) ; then string+="${delimiter}\"${!item}\"" delimiter=',' fi done string+=']' python3 -c "from reproducible_html_packages import gen_packages_html; gen_packages_html(${string}, suite=\"${suite}\", no_clean=True)" done python3 -c "from reproducible_html_packages import purge_old_pages; purge_old_pages()" cd "$CWD" } gather_schedule_stats() { SCHEDULED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM sources_scheduled ORDER BY date_scheduled" | xargs echo) COUNT_SCHEDULED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(name) FROM sources_scheduled" | xargs echo) QUERY=" SELECT count(sources.name) FROM sources,source_packages WHERE sources.name NOT IN (SELECT sources.name FROM sources,sources_scheduled WHERE sources.name=sources_scheduled.name) AND sources.name IN (SELECT sources.name FROM sources,source_packages WHERE sources.name=source_packages.name AND sources.version!=source_packages.version AND source_packages.status!='blacklisted') AND sources.name=source_packages.name" COUNT_NEW_VERSIONS=$(sqlite3 -init $INIT $PACKAGES_DB "$QUERY") } # we only care about sid here gather_stats() { COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='sid' AND r.status = \"unreproducible\"") COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='sid' AND r.status = \"FTBFS\"") COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='sid' AND r.status = \"404\"") COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='sid' AND r.status = \"not for us\"") COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='sid' AND r.status = \"blacklisted\"") PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) }