#!/bin/bash # Copyright 2014-2015 Holger Levsen # © 2015 Mattia Rizzolo # released under the GPLv=2 # # included by all reproducible_*.sh scripts # # define db PACKAGES_DB=/var/lib/jenkins/reproducible.db INIT=/var/lib/jenkins/reproducible.init if [ -f $PACKAGES_DB ] && [ -f $INIT ] ; then if [ -f ${PACKAGES_DB}.lock ] ; then for i in $(seq 0 100) ; do sleep 15 echo "sleeping 15s, $PACKAGES_DB is locked." if [ ! -f ${PACKAGES_DB}.lock ] ; then break fi done if [ -f ${PACKAGES_DB}.lock ] ; then echo "${PACKAGES_DB}.lock still exist, exiting." exit 1 fi fi elif [ ! -f ${PACKAGES_DB} ] ; then echo "Warning: $PACKAGES_DB doesn't exist, creating it now." /srv/jenkins/bin/reproducible_db_maintenance.py # 60 seconds timeout when trying to get a lock cat > $INIT <<-EOF .timeout 60000 EOF fi # common variables REPRODUCIBLE_URL=https://reproducible.debian.net DBDCHROOT_READLOCK=/var/lib/jenkins/reproducible-dbdchroot.readlock DBDCHROOT_WRITELOCK=/var/lib/jenkins/reproducible-dbdchroot.writelock # shop trailing slash JENKINS_URL=${JENKINS_URL:0:-1} # tested suites SUITES="sid experimental" # tested arches ARCHES="amd64" # we only need them for html creation but we cannot declare them in a function declare -A SPOKENTARGET declare -A LINKTARGET NOTES_PATH=/var/lib/jenkins/userContent/notes ISSUES_PATH=/var/lib/jenkins/userContent/issues RB_PATH=/var/lib/jenkins/userContent/rb-pkg/ mkdir -p $NOTES_PATH $ISSUES_PATH $RB_PATH # known package sets META_PKGSET[1]="essential" META_PKGSET[2]="required" META_PKGSET[3]="build-essential" META_PKGSET[4]="popcon_top1337-installed-sources" META_PKGSET[5]="installed_on_debian.org" META_PKGSET[6]="had_a_DSA" META_PKGSET[7]="gnome" META_PKGSET[8]="gnome_build-depends" META_PKGSET[9]="tails" META_PKGSET[10]="tails_build-depends" META_PKGSET[11]="grml" META_PKGSET[12]="grml_build-depends" META_PKGSET[13]="maint_pkg-perl-maintainers" init_html() { SUITE=sid MAINVIEW="stats" ALLSTATES="reproducible FTBR FTBFS 404 not_for_us blacklisted" ALLVIEWS="issues notes no_notes scheduled last_24h last_48h all_abc dd-list repo_stats pkg_sets stats" SPOKENTARGET["reproducible"]="packages which built reproducibly" SPOKENTARGET["FTBR"]="packages which failed to build reproducibly" SPOKENTARGET["FTBFS"]="packages which failed to build from source" SPOKENTARGET["404"]="packages where the sources failed to download" SPOKENTARGET["not_for_us"]="packages which should not be build on 'amd64'" SPOKENTARGET["blacklisted"]="packages which have been blacklisted" SPOKENTARGET["issues"]="known issues related to reproducible builds" SPOKENTARGET["notes"]="packages with notes" SPOKENTARGET["no_notes"]="packages without notes" SPOKENTARGET["scheduled"]="packages currently scheduled for testing for build reproducibility" SPOKENTARGET["last_24h"]="packages tested in the last 24h" SPOKENTARGET["last_48h"]="packages tested in the last 48h" SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" SPOKENTARGET["repo_stats"]="statistics about the reproducible builds apt repository" SPOKENTARGET["pkg_sets"]="statistics about reproducible builds of specific package sets" SPOKENTARGET["stats"]="various statistics about reproducible builds" # query some data we need everywhere (relative to sid, we care only about sid here) AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(*) FROM sources WHERE suite=\"${SUITE}\"") COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite=\"${SUITE}\"") COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(*) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite=\"${SUITE}\" AND r.status=\"reproducible\"") PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc) if [ -z "${PERCENT_TOTAL}" ]; then PERCENT_TOTAL=0; fi PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc) if [ -z "${PERCENT_GOOD}" ]; then PERCENT_GOOD=0; fi } write_page() { echo "$1" >> $PAGE } set_icon() { # icons taken from tango-icon-theme (0.8.90-5) # licenced under http://creativecommons.org/licenses/publicdomain/ STATE_TARGET_NAME="$1" case "$1" in reproducible) ICON=weather-clear.png ;; unreproducible|FTBR) ICON=weather-showers-scattered.png ;; FTBFS) ICON=weather-storm.png ;; 404) ICON=weather-severe-alert.png ;; not_for_us|"not for us") ICON=weather-few-clouds-night.png STATE_TARGET_NAME="not_for_us" ;; blacklisted) ICON=error.png ;; *) ICON="" esac } write_icon() { # ICON and STATE_TARGET_NAME are set by set_icon() write_page "\"${STATE_TARGET_NAME}" } write_page_header() { rm -f $PAGE write_page "" write_page "" write_page "" write_page "$2" write_page "

$2

" if [ "$1" = "$MAINVIEW" ] ; then write_page "

These pages contain results obtained from several jobs running on jenkins.debian.net. Thanks to Profitbricks for donating the virtual machine this is running on!

" fi write_page "

$COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $COUNT_GOOD packages ($PERCENT_GOOD%) could be built reproducible!" if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] || [ "$1" = "stats" ] ; then write_page " Join #debian-reproducible on OFTC to get support for making sure your packages build reproducibly too!" fi write_page "

" write_page "
  • Have a look at:
  • " for MY_STATE in $ALLSTATES ; do set_icon $MY_STATE write_page "
  • " write_icon write_page "
  • " done for TARGET in $ALLVIEWS ; do if [ "$TARGET" = "issues" ] || [ "$TARGET" = "stats" ] ; then SPOKEN_TARGET=$TARGET elif [ "$TARGET" = "scheduled" ] ; then SPOKEN_TARGET="currently scheduled" elif [ "$TARGET" = "pkg_sets" ] ; then SPOKEN_TARGET="package sets stats" elif [ "$TARGET" = "repo_stats" ] ; then SPOKEN_TARGET="apt repository stats" else SPOKEN_TARGET=${SPOKENTARGET[$TARGET]} fi write_page "
  • ${SPOKEN_TARGET}
  • " done write_page "
" write_page "
" } write_page_footer() { write_page "

There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014-2015 Holger Levsen and others, GPL2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library.

" write_page "" } write_page_meta_sign() { write_page "

A package name displayed with a bold font is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue.

" } publish_page() { cp $PAGE /var/lib/jenkins/userContent/ if [ "$VIEW" = "$MAINVIEW" ] ; then cp $PAGE /var/lib/jenkins/userContent/reproducible.html fi rm $PAGE echo "Enjoy $REPRODUCIBLE_URL/$PAGE" } set_package_class() { if [ -f ${NOTES_PATH}/${PKG}_note.html ] ; then CLASS="class=\"noted\"" else CLASS="class=\"package\"" fi } set_linktarget() { local SUITE="$1" local ARCH="$2" shift shift for PKG in $@ ; do if [ -f $RB_PATH/$SUITE/$PKG.html ] ; then set_package_class LINKTARGET[$PKG]="$PKG" else LINKTARGET[$PKG]="$PKG" fi done } link_packages() { for PKG in $@ ; do write_page " ${LINKTARGET[$PKG]}" done } gen_packages_html() { CWD=$(pwd) cd /srv/jenkins/bin for (( i=1; i<$#+1; i=i+100 )) ; do string='[' delimiter='' for (( j=0; j<100; j++)) ; do item=$(( $j+$i )) if (( $item < $#+1 )) ; then string+="${delimiter}\"${!item}\"" delimiter=',' fi done string+=']' python3 -c "from reproducible_html_packages import gen_packages_html; gen_packages_html(${string}, no_clean=True)" done python3 -c "from reproducible_html_packages import purge_old_pages; purge_old_pages()" cd "$CWD" } gather_schedule_stats() { SCHEDULED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM sources_scheduled ORDER BY date_scheduled" | xargs echo) COUNT_SCHEDULED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(name) FROM sources_scheduled" | xargs echo) QUERY=" SELECT count(sources.name) FROM sources,source_packages WHERE sources.name NOT IN (SELECT sources.name FROM sources,sources_scheduled WHERE sources.name=sources_scheduled.name) AND sources.name IN (SELECT sources.name FROM sources,source_packages WHERE sources.name=source_packages.name AND sources.version!=source_packages.version AND source_packages.status!='blacklisted') AND sources.name=source_packages.name" COUNT_NEW_VERSIONS=$(sqlite3 -init $INIT $PACKAGES_DB "$QUERY") } # we only care about sid here gather_stats() { COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='sid' AND r.status = \"unreproducible\"") COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='sid' AND r.status = \"FTBFS\"") COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='sid' AND r.status = \"404\"") COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='sid' AND r.status = \"not for us\"") COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='sid' AND r.status = \"blacklisted\"") PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) }