#!/bin/bash # Copyright 2014-2015 Holger Levsen # © 2015 Mattia Rizzolo # released under the GPLv=2 # # included by all reproducible_*.sh scripts # # define db PACKAGES_DB=/var/lib/jenkins/reproducible.db INIT=/var/lib/jenkins/reproducible.init if [ -f $PACKAGES_DB ] && [ -f $INIT ] ; then if [ -f ${PACKAGES_DB}.lock ] ; then for i in $(seq 0 200) ; do sleep 15 echo "sleeping 15s, $PACKAGES_DB is locked." if [ ! -f ${PACKAGES_DB}.lock ] ; then break fi done if [ -f ${PACKAGES_DB}.lock ] ; then echo "${PACKAGES_DB}.lock still exist, exiting." exit 1 fi fi elif [ ! -f ${PACKAGES_DB} ] ; then echo "Warning: $PACKAGES_DB doesn't exist, creating it now." /srv/jenkins/bin/reproducible_db_maintenance.py # 60 seconds timeout when trying to get a lock cat > $INIT <<-EOF .timeout 60000 EOF fi # common variables REPRODUCIBLE_URL=https://reproducible.debian.net # shop trailing slash JENKINS_URL=${JENKINS_URL:0:-1} # suites being tested SUITES="testing unstable experimental" # arches being tested ARCHES="amd64" # number of cores to be used NUM_CPU=$(grep -c '^processor' /proc/cpuinfo) # existing usertags USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness buildinfo cpu signatures environment umask" # we only need them for html creation but we cannot declare them in a function declare -A SPOKENTARGET BASE="/var/lib/jenkins/userContent/reproducible" mkdir -p "$BASE" # create subdirs for suites for i in $SUITES ; do mkdir -p "$BASE/$i" done # known package sets META_PKGSET[1]="essential" META_PKGSET[2]="required" META_PKGSET[3]="build-essential" META_PKGSET[4]="build-essential-depends" META_PKGSET[5]="popcon_top1337-installed-sources" META_PKGSET[6]="key_packages" META_PKGSET[7]="installed_on_debian.org" META_PKGSET[8]="had_a_DSA" META_PKGSET[9]="gnome" META_PKGSET[10]="gnome_build-depends" META_PKGSET[11]="kde" META_PKGSET[12]="kde_build-depends" META_PKGSET[13]="xfce" META_PKGSET[14]="xfce_build-depends" META_PKGSET[15]="tails" META_PKGSET[16]="tails_build-depends" META_PKGSET[17]="grml" META_PKGSET[18]="grml_build-depends" META_PKGSET[19]="maint_pkg-perl-maintainers" META_PKGSET[20]="maint_pkg-java-maintainers" META_PKGSET[21]="maint_pkg-haskell-maintainers" META_PKGSET[22]="maint_pkg-ruby-extras-maintainers" META_PKGSET[23]="maint_pkg-golang-maintainers" META_PKGSET[24]="maint_pkg-php-pear" META_PKGSET[25]="maint_pkg-javascript-devel" META_PKGSET[26]="maint_debian-boot" META_PKGSET[27]="maint_debian-ocaml" schedule_packages() { # these packages are manually scheduled, so should have high priority, # so schedule them in the past, so they are picked earlier :) # the current date is subtracted twice, so that packages scheduled later get higher will be picked sooner DAYS=$(echo "$(date +'%j')*2"|bc) HOURS=$(echo "$(date +'%H')*2"|bc) MINS=$(date +'%M') # schedule on the full hour so we can recognize them easily DATE=$(date +'%Y-%m-%d %H:%M' -d "$DAYS day ago - $HOURS hours - $MINS minutes") TMPFILE=$(mktemp) for PKG_ID in $@ ; do echo "REPLACE INTO schedule (package_id, date_scheduled, date_build_started, save_artifacts, notify) VALUES ('$PKG_ID', '$DATE', '', '$ARTIFACTS', '$NOTIFY');" >> $TMPFILE done cat $TMPFILE | sqlite3 -init $INIT ${PACKAGES_DB} rm $TMPFILE cd /srv/jenkins/bin python3 -c "from reproducible_html_indexes import generate_schedule; generate_schedule()" } check_candidates() { PACKAGE_IDS="" PACKAGES_NAMES="" TOTAL=0 for PKG in $CANDIDATES ; do RESULT=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT id, name from sources WHERE name='$PKG' AND suite='$SUITE';") if [ ! -z "$RESULT" ] ; then PACKAGE_IDS="$PACKAGE_IDS $(echo $RESULT|cut -d '|' -f 1)" PACKAGES_NAMES="$PACKAGES_NAMES $(echo $RESULT|cut -d '|' -f 2)" let "TOTAL+=1" fi done PACKAGE_IDS=$(echo $PACKAGE_IDS) case $TOTAL in 1) PACKAGES_TXT="package" ;; *) PACKAGES_TXT="packages" ;; esac } write_page() { echo "$1" >> $PAGE } set_icon() { # icons taken from tango-icon-theme (0.8.90-5) # licenced under http://creativecommons.org/licenses/publicdomain/ STATE_TARGET_NAME="$1" case "$1" in reproducible) ICON=weather-clear.png ;; unreproducible|FTBR) ICON=weather-showers-scattered.png ;; FTBFS) ICON=weather-storm.png ;; 404) ICON=weather-severe-alert.png ;; not_for_us|"not for us") ICON=weather-few-clouds-night.png STATE_TARGET_NAME="not_for_us" ;; blacklisted) ICON=error.png ;; *) ICON="" esac } write_icon() { # ICON and STATE_TARGET_NAME are set by set_icon() write_page "\"${STATE_TARGET_NAME}" } write_page_header() { rm -f $PAGE MAINVIEW="stats" ALLSTATES="reproducible FTBR FTBFS 404 not_for_us blacklisted" ALLVIEWS="issues notes no_notes scheduled last_24h last_48h all_abc notify dd-list pkg_sets suite_stats repositories stats" GLOBALVIEWS="issues scheduled notify repositories stats" SUITEVIEWS="dd-list suite_stats" SPOKENTARGET["issues"]="issues" SPOKENTARGET["notes"]="packages with notes" SPOKENTARGET["no_notes"]="packages without notes" SPOKENTARGET["scheduled"]="currently scheduled" SPOKENTARGET["last_24h"]="packages tested in the last 24h" SPOKENTARGET["last_48h"]="packages tested in the last 48h" SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" SPOKENTARGET["notify"]="⚑" SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" SPOKENTARGET["pkg_sets"]="package sets" SPOKENTARGET["suite_stats"]="suite: $SUITE" SPOKENTARGET["repositories"]="repositories overview" SPOKENTARGET["stats"]="reproducible stats" write_page "" write_page "" write_page "" write_page "$2" if [ "$1" != "$MAINVIEW" ] ; then write_page "

$2

" else write_page "

$2

" write_page "

These pages are showing the prospects of reproducible builds of Debian packages." write_page " The results shown were obtained from several jobs running on" write_page " jenkins.debian.net." write_page " Thanks to Profitbricks for donating the virtual machine this is running on!

" fi if [ "$1" = "dd-list" ] || [ "$1" = "stats" ] ; then write_page "

Join #debian-reproducible on OFTC" write_page " or send us an email" write_page " to get support for making sure your packages build reproducibly too. Also, we care about free software in general, so if you are an upstream developer or working on another distribution, we'd love to hear from you! Just now we've started to programatically test coreboot and OpenWRT!" write_page "

" fi write_page "
  • Have a look at:
    • " for MY_STATE in $ALLSTATES ; do set_icon $MY_STATE write_page "
  • " write_icon write_page "
    • " done for TARGET in $ALLVIEWS ; do if [ "$TARGET" = "pkg_sets" ] && [ "$SUITE" = "experimental" ] ; then # no pkg_sets are tested in experimental continue fi SPOKEN_TARGET=${SPOKENTARGET[$TARGET]} BASEURL="/$SUITE/$ARCH" local i for i in $GLOBALVIEWS ; do if [ "$TARGET" = "$i" ] ; then BASEURL="" fi done for i in ${SUITEVIEWS} ; do if [ "$TARGET" = "$i" ] ; then BASEURL="/$SUITE" fi done if [ "$TARGET" = "suite_stats" ] ; then for i in $SUITES ; do write_page "
  • suite: $i
    • " done elif [ "$TARGET" = "notify" ] ; then write_page "
  • ${SPOKEN_TARGET}
    • " else write_page "
  • ${SPOKEN_TARGET}
    • " fi done write_page "
  • wiki
    • " write_page "
" if [ "$1" = "$MAINVIEW" ] ; then write_page "
" write_page "https://reproducible.debian.net/" write_page "" fi write_page "
" } write_page_footer() { write_page "

There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014-2015 Holger Levsen and others, GPL2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library." if [ "$1" = "coreboot" ] ; then write_page "The Coreboot logo is Copyright © 2008 by Konsult Stuge and coresystems GmbH and can be freely used to refer to the Coreboot project." fi write_page "

" } write_page_meta_sign() { write_page "

A package name displayed with a bold font is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue.
" write_page "A # sign after the name of a package indicates that a bug is filed against it. Likewise, a + sign indicates there is a patch available. # indicates a closed bug. In cases of several bugs, the symbol is repeated.

" } write_explaination_table() { write_page "

" write_page "" if [ "$1" = "debian" ] ; then write_page "" write_page "" else write_page "" write_page "" fi export CAPTURE_ENVIRONMENT="I capture the environment" write_page "" write_page "" write_page "" write_page "" write_page "" if [ "$1" = "debian" ] ; then write_page "" write_page "" write_page "" write_page "" write_page "" write_page "" write_page "" else write_page "" write_page "" write_page "" write_page "" fi write_page "" write_page "" write_page "" write_page "" if [ "$1" = "debian" ] ; then write_page "" write_page "" else write_page "" write_page "" fi write_page "
variationfirst buildsecond build
hostname$(hostname)i-capture-the-hostname
domainname$(hostname -d)i-capture-the-domainname
hostname is not yet varied between rebuilds of $1.
domainname is not yet varied between rebuilds of $1.
env CAPTURE_ENVIRONMENTnot setCAPTURE_ENVIRONMENT=\"I capture the environment\"
env TZTZ=\"/usr/share/zoneinfo/Etc/GMT+12\"TZ=\"/usr/share/zoneinfo/Etc/GMT-14\"
env LANGLANG=\"en_GB.UTF-8\"LANG=\"fr_CH.UTF-8\"
env LC_ALLunsetLC_ALL=\"fr_CH.UTF-8\"
env PATHPATH=\"/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:\"PATH=\"/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path\"
env BUILDUSERIDBUILDUSERID=\"1111\"BUILDUSERID=\"2222\"
env BUILDUSERNAMEBUILDUSERNAME=\"pbuilder1\"BUILDUSERNAME=\"pbuilder2\"
env USERUSER=\"pbuilder1\"USER=\"pbuilder2\"
uiduid=1111uid=2222
gidgid=1111gid=2222
env DEB_BUILD_OPTIONSDEB_BUILD_OPTIONS=\"parallel=$NUM_CPU\"DEB_BUILD_OPTIONS=\"parallel=$(echo $NUM_CPU-1|bc)\"
(using a different number of cores is on the agenda)
UTS namespaceshared with the hostmodified using /usr/bin/unshare --uts
env USER is not yet varied between rebuilds of $1.
uid is not yet varied between rebuilds of $1.
gid is not yet varied between rebuilds of $1.
UTS namespace is not yet varied between rebuilds of $1.
kernel version, modified using /usr/bin/linux64 --uname-2.6$(uname -sr)$(/usr/bin/linux64 --uname-2.6 uname -sr)
umask00220002
CPU type$(cat /proc/cpuinfo|grep 'model name'|head -1|cut -d ":" -f2-)same for both builds (currently, work in progress)
year, month, datetoday ($DATE)same for both builds (currently, work in progress)
hour, minutehour is usually the same...usually, the minute differs... (currently, work in progress)
everything else...is likely the same. So far, this is just about the prospects of reproducible builds of Debian - there will be more variations in the wild.
hour, minutehour is usually the same...the minute differs... (currently, work in progress)
everything else...is likely the same. There will be more variations in the wild.

" } publish_page() { if [ "$1" = "" ] ; then TARGET=$PAGE else TARGET=$1/$PAGE fi cp $PAGE $BASE/$TARGET rm $PAGE echo "Enjoy $REPRODUCIBLE_URL/$TARGET" } link_packages() { set +x local i for (( i=1; i<$#+1; i=i+400 )) ; do local string='[' local delimiter='' local j for (( j=0; j<400; j++)) ; do local item=$(( $j+$i )) if (( $item < $#+1 )) ; then string+="${delimiter}\"${!item}\"" delimiter=',' fi done string+=']' cd /srv/jenkins/bin DATA=" $(python3 -c "from reproducible_common import link_packages; \ print(link_packages(${string}, '$SUITE', '$ARCH'))" 2> /dev/null)" cd - > /dev/null write_page "$DATA" done if "$DEBUG" ; then set -x ; fi } gen_packages_html() { local suite="$1" shift CWD=$(pwd) cd /srv/jenkins/bin local i for (( i=1; i<$#+1; i=i+100 )) ; do local string='[' local delimiter='' local j for (( j=0; j<100; j++)) ; do local item=$(( $j+$i )) if (( $item < $#+1 )) ; then string+="${delimiter}\"${!item}\"" delimiter=',' fi done string+=']' python3 -c "from reproducible_html_packages import gen_packages_html; gen_packages_html(${string}, suite=\"${suite}\", no_clean=True)" || echo "Warning: cannot update html pages for ${string} in ${suite}" done cd "$CWD" } calculate_build_duration() { END=$(date +'%s') DURATION=$(( $END - $START )) } print_out_duration() { local HOUR=$(echo "$DURATION/3600"|bc) local MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc) local SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc) echo "$(date) - total duration: ${HOUR}h ${MIN}m ${SEC}s." | tee -a ${RBUILDLOG} } irc_message() { local MESSAGE="$@" kgb-client --conf /srv/jenkins/kgb/debian-reproducible.conf --relay-msg "$MESSAGE" || true # don't fail the whole job }