$2

#!/bin/bash # Copyright 2014-2015 Holger Levsen # © 2015 Mattia Rizzolo # released under the GPLv=2 # # included by all reproducible_*.sh scripts # # define db PACKAGES_DB=/var/lib/jenkins/reproducible.db INIT=/var/lib/jenkins/reproducible.init if [ -f $PACKAGES_DB ] && [ -f $INIT ] ; then if [ -f ${PACKAGES_DB}.lock ] ; then for i in $(seq 0 100) ; do sleep 15 echo "sleeping 15s, $PACKAGES_DB is locked." if [ ! -f ${PACKAGES_DB}.lock ] ; then break fi done if [ -f ${PACKAGES_DB}.lock ] ; then echo "${PACKAGES_DB}.lock still exist, exiting." exit 1 fi fi elif [ ! -f ${PACKAGES_DB} ] ; then echo "Warning: $PACKAGES_DB doesn't exist, creating it now." /srv/jenkins/bin/reproducible_db_maintenance.py # 60 seconds timeout when trying to get a lock cat > $INIT <<-EOF .timeout 60000 EOF fi # common variables REPRODUCIBLE_URL=https://reproducible.debian.net DBDCHROOT_READLOCK=/var/lib/jenkins/reproducible-dbdchroot.readlock DBDCHROOT_WRITELOCK=/var/lib/jenkins/reproducible-dbdchroot.writelock # shop trailing slash JENKINS_URL=${JENKINS_URL:0:-1} # tested suites SUITES="sid experimental" # tested arches ARCHES="amd64" # we only need them for html creation but we cannot declare them in a function declare -A SPOKENTARGET declare -A LINKTARGET NOTES_PATH=/var/lib/jenkins/userContent/notes ISSUES_PATH=/var/lib/jenkins/userContent/issues RB_PATH=/var/lib/jenkins/userContent/rb-pkg/ mkdir -p $NOTES_PATH $ISSUES_PATH $RB_PATH # create subdirs for suites for i in $SUITES ; do mkdir -p /var/lib/jenkins/userContent/$i done # known package sets META_PKGSET[1]="essential" META_PKGSET[2]="required" META_PKGSET[3]="build-essential" META_PKGSET[4]="popcon_top1337-installed-sources" META_PKGSET[5]="installed_on_debian.org" META_PKGSET[6]="had_a_DSA" META_PKGSET[7]="gnome" META_PKGSET[8]="gnome_build-depends" META_PKGSET[9]="tails" META_PKGSET[10]="tails_build-depends" META_PKGSET[11]="grml" META_PKGSET[12]="grml_build-depends" META_PKGSET[13]="maint_pkg-perl-maintainers" init_html() { MAINVIEW="stats" ALLSTATES="reproducible FTBR FTBFS 404 not_for_us blacklisted" ALLVIEWS="issues notes no_notes scheduled last_24h last_48h all_abc dd-list pkg_sets suite_stats repo_stats stats" GLOBALVIEWS="issues notes no_notes repo_stats stats" SUITEVIEWS="dd-list suite_stats" SPOKENTARGET["issues"]="issues" SPOKENTARGET["notes"]="packages with notes" SPOKENTARGET["no_notes"]="packages without notes" SPOKENTARGET["scheduled"]="currently scheduled" SPOKENTARGET["last_24h"]="packages tested in the last 24h" SPOKENTARGET["last_48h"]="packages tested in the last 48h" SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" SPOKENTARGET["pkg_sets"]="package sets stats" SPOKENTARGET["suite_stats"]="suite: $SUITE" SPOKENTARGET["repo_stats"]="repositories overview" SPOKENTARGET["stats"]="reproducible stats" } write_page() { echo "$1" >> $PAGE } set_icon() { # icons taken from tango-icon-theme (0.8.90-5) # licenced under http://creativecommons.org/licenses/publicdomain/ STATE_TARGET_NAME="$1" case "$1" in reproducible) ICON=weather-clear.png ;; unreproducible|FTBR) ICON=weather-showers-scattered.png ;; FTBFS) ICON=weather-storm.png ;; 404) ICON=weather-severe-alert.png ;; not_for_us|"not for us") ICON=weather-few-clouds-night.png STATE_TARGET_NAME="not_for_us" ;; blacklisted) ICON=error.png ;; *) ICON="" esac } write_icon() { # ICON and STATE_TARGET_NAME are set by set_icon() write_page " $\"${STATE_TARGET_NAME}$ " } write_page_header() { rm -f $PAGE write_page "" write_page "" write_page "" write_page "$2" write_page "

$2

" if [ "$1" = "$MAINVIEW" ] ; then write_page "

These pages are showing the prospects of reproducible builds of Debian packages. The results shown were obtained from several jobs running on jenkins.debian.net. Thanks to Profitbricks for donating the virtual machine this is running on!

" fi if [ "$1" = "dd-list" ] || [ "$1" = "stats" ] ; then write_page " Join #debian-reproducible on OFTC or send us an email to get support for making sure your packages build reproducibly too!" write_page "

" fi write_page "

Have a look at:
" write_icon write_page "
${SPOKEN_TARGET}
suite: $i
wiki

" write_page "

" } write_page_footer() { write_page "

There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014-2015 Holger Levsen and others, GPL2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library.

" write_page "" } write_page_meta_sign() { write_page "

A package name displayed with a bold font is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue.

" } publish_page() { if [ "$1" = "" ] ; then if [ "$VIEW" = "$MAINVIEW" ] ; then cp $PAGE /var/lib/jenkins/userContent/reproducible.html fi TARGET=$PAGE else TARGET=$1/$PAGE fi cp $PAGE /var/lib/jenkins/userContent/$TARGET rm $PAGE echo "Enjoy $REPRODUCIBLE_URL/$TARGET" } set_package_class() { if [ -f ${NOTES_PATH}/${PKG}_note.html ] ; then CLASS="class=\"noted\"" else CLASS="class=\"package\"" fi } set_linktarget() { for PKG in $@ ; do if [ -f $RB_PATH/$SUITE/$ARCH/$PKG.html ] ; then set_package_class LINKTARGET[$PKG]="$PKG" else LINKTARGET[$PKG]="$PKG" fi done } link_packages() { for PKG in $@ ; do write_page " ${LINKTARGET[$PKG]}" done } gen_packages_html() { local suite="$1" shift CWD=$(pwd) cd /srv/jenkins/bin for (( i=1; i<$#+1; i=i+100 )) ; do string='[' delimiter='' for (( j=0; j<100; j++)) ; do item=$(( $j+$i )) if (( $item < $#+1 )) ; then string+="${delimiter}\"${!item}\"" delimiter=',' fi done string+=']' python3 -c "from reproducible_html_packages import gen_packages_html; gen_packages_html(${string}, suite=\"${suite}\", no_clean=True)" done python3 -c "from reproducible_html_packages import purge_old_pages; purge_old_pages()" cd "$CWD" } gather_schedule_stats() { SCHEDULED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM sources_scheduled ORDER BY date_scheduled" | xargs echo) COUNT_SCHEDULED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT count(name) FROM sources_scheduled" | xargs echo) QUERY=" SELECT count(sources.name) FROM sources,source_packages WHERE sources.name NOT IN (SELECT sources.name FROM sources,sources_scheduled WHERE sources.name=sources_scheduled.name) AND sources.name IN (SELECT sources.name FROM sources,source_packages WHERE sources.name=source_packages.name AND sources.version!=source_packages.version AND source_packages.status!='blacklisted') AND sources.name=source_packages.name" COUNT_NEW_VERSIONS=$(sqlite3 -init $INIT $PACKAGES_DB "$QUERY") } gather_stats() { COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"unreproducible\"") COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"FTBFS\"") COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"404\"") COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"not for us\"") COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(s.name) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND r.status = \"blacklisted\"") PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) }