#!/bin/bash # Copyright 2014-2015 Holger Levsen # © 2015 Mattia Rizzolo # released under the GPLv=2 # # included by all reproducible_*.sh scripts # # define db PACKAGES_DB=/var/lib/jenkins/reproducible.db INIT=/var/lib/jenkins/reproducible.init if [ -f $PACKAGES_DB ] && [ -f $INIT ] ; then if [ -f ${PACKAGES_DB}.lock ] ; then for i in $(seq 0 200) ; do sleep 15 echo "sleeping 15s, $PACKAGES_DB is locked." if [ ! -f ${PACKAGES_DB}.lock ] ; then break fi done if [ -f ${PACKAGES_DB}.lock ] ; then echo "${PACKAGES_DB}.lock still exist, exiting." exit 1 fi fi elif [ ! -f ${PACKAGES_DB} ] && [ "$HOSTNAME" = "jenkins" ] ; then echo "Warning: $PACKAGES_DB doesn't exist, creating it now." /srv/jenkins/bin/reproducible_db_maintenance.py # 60 seconds timeout when trying to get a lock cat > $INIT <<-EOF .timeout 60000 EOF fi # common variables REPRODUCIBLE_URL=https://reproducible.debian.net # shop trailing slash JENKINS_URL=${JENKINS_URL:0:-1} # suites being tested SUITES="testing unstable experimental" # architectures being tested ARCHS="amd64" # number of cores to be used NUM_CPU=$(grep -c '^processor' /proc/cpuinfo) # existing usertags USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness buildinfo cpu signatures environment umask ftbfs locale" # we only need them for html creation but we cannot declare them in a function declare -A SPOKENTARGET BASE="/var/lib/jenkins/userContent/reproducible" mkdir -p "$BASE" # to hold reproducible temporary files/directories without polluting /tmp TEMPDIR="/tmp/reproducible" mkdir -p "$TEMPDIR" # create subdirs for suites for i in $SUITES ; do mkdir -p "$BASE/$i" done # tables for stats TABLE[0]=stats_pkg_state TABLE[1]=stats_builds_per_day TABLE[2]=stats_builds_age TABLE[3]=stats_bugs TABLE[4]=stats_notes TABLE[5]=stats_issues TABLE[6]=stats_meta_pkg_state TABLE[7]=stats_bugs_state # known package sets META_PKGSET[1]="essential" META_PKGSET[2]="required" META_PKGSET[3]="build-essential" META_PKGSET[4]="build-essential-depends" META_PKGSET[5]="popcon_top1337-installed-sources" META_PKGSET[6]="key_packages" META_PKGSET[7]="installed_on_debian.org" META_PKGSET[8]="had_a_DSA" META_PKGSET[9]="cii-census" META_PKGSET[10]="gnome" META_PKGSET[11]="gnome_build-depends" META_PKGSET[12]="kde" META_PKGSET[13]="kde_build-depends" META_PKGSET[14]="xfce" META_PKGSET[15]="xfce_build-depends" META_PKGSET[16]="tails" META_PKGSET[17]="tails_build-depends" META_PKGSET[18]="grml" META_PKGSET[19]="grml_build-depends" META_PKGSET[20]="maint_pkg-perl-maintainers" META_PKGSET[21]="maint_pkg-java-maintainers" META_PKGSET[22]="maint_pkg-haskell-maintainers" META_PKGSET[23]="maint_pkg-ruby-extras-maintainers" META_PKGSET[24]="maint_pkg-golang-maintainers" META_PKGSET[25]="maint_pkg-php-pear" META_PKGSET[26]="maint_pkg-javascript-devel" META_PKGSET[27]="maint_debian-boot" META_PKGSET[28]="maint_debian-ocaml" META_PKGSET[29]="maint_debian-x" META_PKGSET[30]="maint_lua" schedule_packages() { LC_USER="$REQUESTER" \ LOCAL_CALL="true" \ /srv/jenkins/bin/reproducible_remote_scheduler.py \ --message "$REASON" \ --no-notify \ --suite "$SUITE" \ $@ } write_page() { echo "$1" >> $PAGE } set_icon() { # icons taken from tango-icon-theme (0.8.90-5) # licenced under http://creativecommons.org/licenses/publicdomain/ STATE_TARGET_NAME="$1" case "$1" in reproducible) ICON=weather-clear.png ;; unreproducible|FTBR) ICON=weather-showers-scattered.png ;; FTBFS) ICON=weather-storm.png ;; depwait) ICON=weather-snow.png ;; 404) ICON=weather-severe-alert.png ;; not_for_us|"not for us") ICON=weather-few-clouds-night.png STATE_TARGET_NAME="not_for_us" ;; blacklisted) ICON=error.png ;; *) ICON="" esac } write_icon() { # ICON and STATE_TARGET_NAME are set by set_icon() write_page "\"${STATE_TARGET_NAME}" } write_page_header() { rm -f $PAGE MAINVIEW="stats" ALLSTATES="reproducible FTBR FTBFS depwait not_for_us 404 blacklisted" ALLVIEWS="issues notes no_notes scheduled last_24h last_48h all_abc notify dd-list pkg_sets suite_stats repositories stats" GLOBALVIEWS="issues scheduled notify repositories stats" SUITEVIEWS="dd-list suite_stats" SPOKENTARGET["issues"]="issues" SPOKENTARGET["notes"]="packages with notes" SPOKENTARGET["no_notes"]="packages without notes" SPOKENTARGET["scheduled"]="currently scheduled" SPOKENTARGET["last_24h"]="packages tested in the last 24h" SPOKENTARGET["last_48h"]="packages tested in the last 48h" SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" SPOKENTARGET["notify"]="⚑" SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" SPOKENTARGET["pkg_sets"]="package sets" SPOKENTARGET["suite_stats"]="suite: $SUITE" SPOKENTARGET["repositories"]="repositories overview" SPOKENTARGET["stats"]="reproducible stats" write_page "" write_page "" write_page "" write_page "$2" if [ "$1" != "$MAINVIEW" ] ; then write_page "

$2

" else write_page "

$2

" write_page "

These pages are showing the prospects of reproducible builds of Debian packages." write_page " The results shown were obtained from several jobs running on" write_page " jenkins.debian.net." write_page " Thanks to Profitbricks for donating the virtual machines this is running on!

" fi if [ "$1" = "dd-list" ] || [ "$1" = "stats" ] ; then write_page "

Join #debian-reproducible on OFTC" write_page " or send us an email" write_page " to get support for making sure your packages build reproducibly too. Also, we care about free software in general, so if you are an upstream developer or working on another distribution, we'd love to hear from you! Just now we've started to programatically test coreboot, OpenWrt and NetBSD - and there are plans to test Fedora and FreeBSD soon too." write_page "

" fi write_page "
  • Have a look at:
    • " for MY_STATE in $ALLSTATES ; do set_icon $MY_STATE write_page "
  • " write_icon write_page "
    • " done for TARGET in $ALLVIEWS ; do if [ "$TARGET" = "pkg_sets" ] && [ "$SUITE" = "experimental" ] ; then # no pkg_sets are tested in experimental continue fi SPOKEN_TARGET=${SPOKENTARGET[$TARGET]} BASEURL="/$SUITE/$ARCH" local i for i in $GLOBALVIEWS ; do if [ "$TARGET" = "$i" ] ; then BASEURL="" fi done for i in ${SUITEVIEWS} ; do if [ "$TARGET" = "$i" ] ; then BASEURL="/$SUITE" fi done if [ "$TARGET" = "suite_stats" ] ; then for i in $SUITES ; do write_page "
  • suite: $i
    • " done elif [ "$TARGET" = "notify" ] ; then write_page "
  • ${SPOKEN_TARGET}
    • " else write_page "
  • ${SPOKEN_TARGET}
    • " fi done write_page "
  • wiki
    • " write_page "
" if [ "$1" = "$MAINVIEW" ] ; then LATEST=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id = s.id WHERE r.status IN ('unreproducible') AND s.suite = 'unstable' AND s.id NOT IN (SELECT package_id FROM notes) ORDER BY build_date DESC LIMIT 23"|sort -R|head -1) write_page "
" write_page "https://reproducible.debian.net/" write_page "" fi write_page "
" } write_page_intro() { write_page "

Reproducible builds enable anyone to reproduce bit by bit identical binary packages from a given source, so that anyone can verify that a given binary derived from the source it was said to be derived. There is a lot more information about reproducible builds on the Debian wiki and on https://reproducible.debian.net. The wiki explains in more depth why this is useful, what common issues exist and which workarounds and solutions are known.
" if [ "$1" = "coreboot" ] ; then write_page " Reproducible Coreboot is an effort to apply this to coreboot. Thus each coreboot.rom is build twice (without payloads), with a few varitations added and then those two ROMs are compared using debbindiff. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.

" local PROJECTNAME="$1" local PROJECTURL="https://review.coreboot.org/p/coreboot.git" elif [ "$1" = "OpenWrt" ] ; then write_page " Reproducible OpenWrt is an effort to apply this to OpenWrt. Thus each OpenWrt target is build twice, with a few varitations added and then the resulting images and packages from the two builds are compared using debbindiff, which currently cannot detect .bin files as squashfs filesystems. Thus the resulting debbindiff output is not nearly as clear as it could be - hopefully this limitation will be overcome soon. Also please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.

" local PROJECTNAME="openwrt" local PROJECTURL="git://git.openwrt.org/openwrt.git" elif [ "$1" = "NetBSD" ] ; then write_page " Reproducible NetBSD is an effort to apply this to NetBSD. Thus each NetBSD target is build twice, with a few varitations added and then the resulting files from the two builds are compared using debbindiff. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.

" local PROJECTNAME="netbsd" local PROJECTURL="https://github.com/jsonn/src" fi write_page "

There is a monthly run jenkins job to test the master branch of $PROJECTNAME.git. Currently this job is triggered more often though, because this is still under development and brand new. The jenkins job is simply running reproducible_$PROJECTNAME.sh in a Debian environment and this script is solely responsible for creating this page. Feel invited to join #debian-reproducible (on irc.oftc.net) to request job runs whenever sensible. Patches and other feedback are also very much appreciated!

" } write_page_footer() { write_page "

There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014-2015 Holger Levsen and others, GPL2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library." if [ "$1" = "coreboot" ] ; then write_page "The Coreboot logo is Copyright © 2008 by Konsult Stuge and coresystems GmbH and can be freely used to refer to the Coreboot project." elif [ "$1" = "NetBSD" ] ; then write_page "NetBSD® is a registered trademark of The NetBSD Foundation, Inc." fi write_page "

" } write_page_meta_sign() { write_page "

A package name displayed with a bold font is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue.
" write_page "A # sign after the name of a package indicates that a bug is filed against it. Likewise, a + sign indicates there is a patch available. # indicates a closed bug. In cases of several bugs, the symbol is repeated.

" } write_explaination_table() { write_page "

" write_page "" if [ "$1" = "debian" ] ; then write_page "" write_page "" else write_page "" write_page "" fi export CAPTURE_ENVIRONMENT="I capture the environment" write_page "" write_page "" write_page "" write_page "" write_page "" if [ "$1" = "debian" ] ; then write_page "" write_page "" write_page "" write_page "" write_page "" write_page "" write_page "" else write_page "" write_page "" write_page "" write_page "" fi write_page "" write_page "" write_page "" write_page "" if [ "$1" = "debian" ] ; then write_page "" write_page "" else write_page "" write_page "" fi write_page "
variationfirst buildsecond build
hostname$(hostname)i-capture-the-hostname
domainname$(hostname -d)i-capture-the-domainname
hostname is not yet varied between rebuilds of $1.
domainname is not yet varied between rebuilds of $1.
env CAPTURE_ENVIRONMENTnot setCAPTURE_ENVIRONMENT=\"I capture the environment\"
env TZTZ=\"/usr/share/zoneinfo/Etc/GMT+12\"TZ=\"/usr/share/zoneinfo/Etc/GMT-14\"
env LANGLANG=\"en_GB.UTF-8\"LANG=\"fr_CH.UTF-8\"
env LC_ALLnot setLC_ALL=\"fr_CH.UTF-8\"
env PATHPATH=\"/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:\"PATH=\"/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path\"
env BUILDUSERIDBUILDUSERID=\"1111\"BUILDUSERID=\"2222\"
env BUILDUSERNAMEBUILDUSERNAME=\"pbuilder1\"BUILDUSERNAME=\"pbuilder2\"
env USERUSER=\"pbuilder1\"USER=\"pbuilder2\"
uiduid=1111uid=2222
gidgid=1111gid=2222
env DEB_BUILD_OPTIONSDEB_BUILD_OPTIONS=\"parallel=$NUM_CPU\"DEB_BUILD_OPTIONS=\"parallel=$(echo $NUM_CPU-1|bc)\"
(using a different number of cores is on the agenda)
UTS namespaceshared with the hostmodified using /usr/bin/unshare --uts
env USER is not yet varied between rebuilds of $1.
uid is not yet varied between rebuilds of $1.
gid is not yet varied between rebuilds of $1.
UTS namespace is not yet varied between rebuilds of $1.
kernel version, modified using /usr/bin/linux64 --uname-2.6$(uname -sr)$(/usr/bin/linux64 --uname-2.6 uname -sr)
umask00220002
CPU type$(cat /proc/cpuinfo|grep 'model name'|head -1|cut -d ":" -f2-)same for both builds (currently, work in progress)
year, month, datetoday ($DATE)same for both builds (currently, work in progress)
hour, minutehour is usually the same...usually, the minute differs... (currently, work in progress)
everything else...is likely the same. So far, this is just about the prospects of reproducible builds of Debian - there will be more variations in the wild.
hour, minutehour is usually the same...the minute differs... (currently, work in progress)
everything else...is likely the same. There will be more variations in the wild.

" } publish_page() { if [ "$1" = "" ] ; then TARGET=$PAGE else TARGET=$1/$PAGE fi cp $PAGE $BASE/$TARGET rm $PAGE echo "Enjoy $REPRODUCIBLE_URL/$TARGET" } link_packages() { set +x local i for (( i=1; i<$#+1; i=i+400 )) ; do local string='[' local delimiter='' local j for (( j=0; j<400; j++)) ; do local item=$(( $j+$i )) if (( $item < $#+1 )) ; then string+="${delimiter}\"${!item}\"" delimiter=',' fi done string+=']' cd /srv/jenkins/bin DATA=" $(python3 -c "from reproducible_common import link_packages; \ print(link_packages(${string}, '$SUITE', '$ARCH'))" 2> /dev/null)" cd - > /dev/null write_page "$DATA" done if "$DEBUG" ; then set -x ; fi } gen_packages_html() { local suite="$1" shift CWD=$(pwd) cd /srv/jenkins/bin local i for (( i=1; i<$#+1; i=i+100 )) ; do local string='[' local delimiter='' local j for (( j=0; j<100; j++)) ; do local item=$(( $j+$i )) if (( $item < $#+1 )) ; then string+="${delimiter}\"${!item}\"" delimiter=',' fi done string+=']' python3 -c "from reproducible_html_packages import gen_packages_html; gen_packages_html(${string}, suite=\"${suite}\", no_clean=True)" || echo "Warning: cannot update html pages for ${string} in ${suite}" done cd "$CWD" } calculate_build_duration() { END=$(date +'%s') DURATION=$(( $END - $START )) } print_out_duration() { local HOUR=$(echo "$DURATION/3600"|bc) local MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc) local SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc) echo "$(date) - total duration: ${HOUR}h ${MIN}m ${SEC}s." | tee -a ${RBUILDLOG} } irc_message() { local MESSAGE="$@" kgb-client --conf /srv/jenkins/kgb/debian-reproducible.conf --relay-msg "$MESSAGE" || true # don't fail the whole job } call_debbindiff() { mkdir -p $TMPDIR/$1/$(dirname $2) local TMPLOG=(mktemp --tmpdir=$TMPDIR) local msg="" set +e ( timeout $TIMEOUT schroot \ --directory $TMPDIR \ -c source:jenkins-reproducible-${DBDSUITE}-debbindiff \ debbindiff -- \ --html $TMPDIR/$1/$2.html \ $TMPDIR/b1/$1/$2 \ $TMPDIR/b2/$1/$2 2>&1 \ ) 2>&1 >> $TMPLOG RESULT=$? if ! "$DEBUG" ; then set +x ; fi set -e cat $TMPLOG # print dbd output rm -f $TMPLOG case $RESULT in 0) echo "$(date -u) - $1/$2 is reproducible, yay!" ;; 1) echo "$(date -u) - $DBDVERSION found issues, please investigate $1/$2" ;; 2) msg="$(date -u) - $DBDVERSION had trouble comparing the two builds. Please investigate $1/$2" ;; 124) if [ ! -s $TMPDIR/$1.html ] ; then msg="$(date -u) - $DBDVERSION produced no output for $1/$2 and was killed after running into timeout after ${TIMEOUT}..." else msg="$DBDVERSION was killed after running into timeout after $TIMEOUT, but there is still $TMPDIR/$1/$2.html" fi ;; *) msg="$(date -u) - Something weird happened when running $DBDVERSION on $1/$2 (which exited with $RESULT) and I don't know how to handle it." ;; esac if [ ! -z "$msg" ] ; then echo $msg | tee -a $TMPDIR/$1/$2.html fi } get_filesize() { local BYTESIZE="$(du -h -b $1 | cut -f1)" # numbers below 16384K are understood and more meaningful than 16M... if [ $BYTESIZE -gt 16777216 ] ; then SIZE="$(echo $BYTESIZE/1048576|bc)M" elif [ $BYTESIZE -gt 1024 ] ; then SIZE="$(echo $BYTESIZE/1024|bc)K" else SIZE="$BYTESIZE bytes" fi } cleanup_pkg_files() { rm -vf $BASE/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_*.rbuild.log{,.gz} rm -vf $BASE/logs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.build?.log{,.gz} rm -vf $BASE/dbd/${SUITE}/${ARCH}/${SRCPACKAGE}_*.debbindiff.html rm -vf $BASE/dbdtxt/${SUITE}/${ARCH}/${SRCPACKAGE}_*.debbindiff.txt{,.gz} rm -vf $BASE/buildinfo/${SUITE}/${ARCH}/${SRCPACKAGE}_*.buildinfo rm -vf $BASE/logdiffs/${SUITE}/${ARCH}/${SRCPACKAGE}_*.diff{,.gz} } # # create the png (and query the db to populate a csv file...) # create_png_from_table() { echo "Checking whether to update $2..." # $1 = id of the stats table # $2 = image file name # $3 = meta package set, only sensible if $1=6 echo "${FIELDS[$1]}" > ${TABLE[$1]}.csv # prepare query WHERE_EXTRA="WHERE suite = '$SUITE'" if [ $1 -eq 3 ] || [ $1 -eq 4 ] || [ $1 -eq 5 ] ; then # TABLE[3+4+5] don't have a suite column: WHERE_EXTRA="" elif [ $1 -eq 6 ] ; then # 6 is special too: WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$3'" fi # run query if [ $1 -eq 1 ] ; then # not sure if it's worth to generate the following query... sqlite3 -init ${INIT} --nullvalue 0 -csv ${PACKAGES_DB} "SELECT s.datum, COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='testing'),0) as 'reproducible_testing', COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='unstable'),0) as 'reproducible_unstable', COALESCE((SELECT e.reproducible FROM stats_builds_per_day AS e where s.datum=e.datum and suite='experimental'),0) as 'reproducible_experimental', (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS unreproducible_testing, (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS unreproducible_unstable, (SELECT e.unreproducible FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS unreproducible_experimental, (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS FTBFS_testing, (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS FTBFS_unstable, (SELECT e.FTBFS FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS FTBFS_experimental, (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='testing') AS other_testing, (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='unstable') AS other_unstable, (SELECT e.other FROM stats_builds_per_day e WHERE s.datum=e.datum AND suite='experimental') AS other_experimental FROM stats_builds_per_day AS s GROUP BY s.datum" >> ${TABLE[$1]}.csv elif [ $1 -eq 2 ] ; then # just make a graph of the oldest reproducible build (ignore FTBFS and unreproducible) sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, oldest_reproducible FROM ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv elif [ $1 -eq 7 ] ; then sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT datum, $SUM_DONE, $SUM_OPEN from ${TABLE[3]} ORDER BY datum" >> ${TABLE[$1]}.csv else sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$1]} from ${TABLE[$1]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$1]}.csv fi # this is a gross hack: normally we take the number of colors a table should have... # for the builds_age table we only want one color, but different ones, so this hack: COLORS=${COLOR[$1]} if [ $1 -eq 2 ] ; then case "$SUITE" in testing) COLORS=40 ;; unstable) COLORS=41 ;; experimental) COLORS=42 ;; esac fi # only generate graph if the query returned data if [ $(cat ${TABLE[$1]}.csv | wc -l) -gt 1 ] ; then echo "Updating $2..." DIR=$(dirname $2) mkdir -p $DIR echo "Generating $2." /srv/jenkins/bin/make_graph.py ${TABLE[$1]}.csv $2 ${COLORS} "${MAINLABEL[$1]}" "${YLABEL[$1]}" mv $2 $BASE/$DIR [ "$DIR" = "." ] || rmdir $(dirname $2) # create empty dummy png if there havent been any results ever elif [ ! -f $BASE/$DIR/$(basename $2) ] ; then DIR=$(dirname $2) mkdir -p $DIR echo "Creating $2 dummy." convert -size 1920x960 xc:#aaaaaa -depth 8 $2 if [ "$3" != "" ] ; then local THUMB="${TABLE[1]}_${3}-thumbnail.png" convert $2 -adaptive-resize 160x80 ${THUMB} mv ${THUMB} $BASE/$DIR fi mv $2 $BASE/$DIR [ "$DIR" = "." ] || rmdir $(dirname $2) fi rm ${TABLE[$1]}.csv }