#!/bin/bash # Copyright 2014-2016 Holger Levsen # released under the GPLv=2 DEBUG=false . /srv/jenkins/bin/common-functions.sh common_init "$@" # common code defining db access . /srv/jenkins/bin/reproducible_common.sh set -e # build for these architectures MACHINES="sparc64 amd64" cleanup_tmpdirs() { cd rm -r $TMPDIR rm -r $TMPBUILDDIR } create_results_dirs() { mkdir -p $BASE/netbsd/dbd } save_netbsd_results() { local RUN=$1 local MACHINE=$2 mkdir -p $TMPDIR/$RUN/${MACHINE} cp -pr obj/releasedir/${MACHINE} $TMPDIR/$RUN/ find $TMPDIR/$RUN/ -name MD5 -o -name SHA512 | xargs -r rm -v } # # main # TMPBUILDDIR=$(mktemp --tmpdir=/srv/workspace/chroots/ -d -t rbuild-netbsd-build-XXXXXXXX) # used to build on tmpfs TMPDIR=$(mktemp --tmpdir=/srv/reproducible-results -d -t rbuild-netbsd-results-XXXXXXXX) # accessable in schroots, used to compare results DATE=$(date -u +'%Y-%m-%d') START=$(date +'%s') trap cleanup_tmpdirs INT TERM EXIT cd $TMPBUILDDIR echo "=============================================================================" echo "$(date -u) - Cloning the NetBSD git repository (which is synced with the NetBSD CVS repository)" echo "=============================================================================" git clone --depth 1 https://github.com/jsonn/src mv src netbsd cd netbsd NETBSD="$(git log -1)" NETBSD_VERSION=$(git describe --always) echo "This is netbsd $NETBSD_VERSION." echo git log -1 # from $src/share/mk/bsd.README: # MKREPRO If "yes", create reproducable builds. This enables # different switches to make two builds from the same source tree # result in the same build results. # also see http://man.netbsd.org/HEAD/usr/share/man/html5/mk.conf.html export MKREPRO="yes" # MKREPRO_TIMESTAMP is set to SOURCE_DATE_EPOCH of netbsd.git # see http://cvsweb.netbsd.org/bsdweb.cgi/~checkout~/src/BUILDING SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) MKREPRO_TIMESTAMP=${SOURCE_DATE_EPOCH} echo "=============================================================================" echo "$(date -u) - Building NetBSD ${NETBSD_VERSION} - first build run." echo "=============================================================================" export TZ="/usr/share/zoneinfo/Etc/GMT+12" # actually build everything for MACHINE in $MACHINES ; do ionice -c 3 \ ./build.sh -j $NUM_CPU -V MKREPRO_TIMESTAMP=$MKREPRO_TIMESTAMP -U -u -m ${MACHINE} release # save results in b1 save_netbsd_results b1 ${MACHINE} # cleanup and explicitly delete old tooldir to force re-creation for the next $MACHINE type ./build.sh -U -m ${MACHINE} cleandir rm obj/tooldir.* -rf echo "${MACHINE} done, first time." done echo "=============================================================================" echo "$(date -u) - Building NetBSD ${NETBSD_VERSION} - cleaning up between builds." echo "=============================================================================" rm obj/releasedir -r rm obj/destdir.* -r # we keep the toolchain(s) echo "=============================================================================" echo "$(date -u) - Building NetBSD - second build run." echo "=============================================================================" export TZ="/usr/share/zoneinfo/Etc/GMT-14" export LANG="fr_CH.UTF-8" export LC_ALL="fr_CH.UTF-8" export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" export CAPTURE_ENVIRONMENT="I capture the environment" umask 0002 # use allmost all cores for second build NEW_NUM_CPU=$(echo $NUM_CPU-1|bc) for MACHINE in $MACHINES ; do ionice -c 3 \ linux64 --uname-2.6 \ ./build.sh -j $NEW_NUM_CPU -V MKREPRO_TIMESTAMP=$MKREPRO_TIMESTAMP -U -u -m ${MACHINE} release # save results in b2 save_netbsd_results b2 ${MACHINE} # cleanup and explicitly delete old tooldir to force re-creation for the next $MACHINE type ./build.sh -U -m ${MACHINE} cleandir rm obj/tooldir.* -r echo "${MACHINE} done, second time." done # reset environment to default values again export LANG="en_GB.UTF-8" unset LC_ALL export TZ="/usr/share/zoneinfo/UTC" export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:" umask 0022 # clean up builddir to save space on tmpfs rm -r $TMPBUILDDIR/netbsd # temporarily enable more debug set -x # run diffoscope on the results TIMEOUT="30m" DIFFOSCOPE="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --version 2>&1)" echo "=============================================================================" echo "$(date -u) - Running $DIFFOSCOPE on NetBSD build results..." echo "=============================================================================" FILES_HTML=$(mktemp --tmpdir=$TMPDIR) GOOD_FILES_HTML=$(mktemp --tmpdir=$TMPDIR) BAD_FILES_HTML=$(mktemp --tmpdir=$TMPDIR) GOOD_SECTION_HTML=$(mktemp --tmpdir=$TMPDIR) BAD_SECTION_HTML=$(mktemp --tmpdir=$TMPDIR) GOOD_FILES=0 ALL_FILES=0 SIZE="" create_results_dirs cd $TMPDIR/b1 tree . for i in * ; do cd $i for j in $(find * -type f |sort -u ) ; do let ALL_FILES+=1 call_diffoscope $i $j get_filesize $j if [ -f $TMPDIR/$i/$j.html ] ; then mkdir -p $BASE/netbsd/dbd/$i/$(dirname $j) mv $TMPDIR/$i/$j.html $BASE/netbsd/dbd/$i/$j.html echo " \"unreproducible $j ($SIZE) is unreproducible." >> $BAD_FILES_HTML else SHASUM=$(sha256sum $j|cut -d " " -f1) echo " \"reproducible $j ($SHASUM, $SIZE) is reproducible." >> $GOOD_FILES_HTML let GOOD_FILES+=1 rm -f $BASE/netbsd/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed fi # for debugging #842250 if [ "$(basename $TMPDIR/b1/$i/$j)" = "base.tgz" ] ; then for k in b1 b2 ; do mkdir -p $BASE/netbsd/artifacts/$k/$i cp $TMPDIR/$k/$i/$j $BASE/netbsd/artifacts/$k/$i/$j done echo "artifacts published for debugging #842250 in https://tests.reproducible-builds.org/netbsd/artifacts/ - please dont forget to delete this directory later…" fi done cd .. if [ -s $GOOD_FILES_HTML ] ; then echo " " >> $GOOD_SECTION_HTML cat $GOOD_FILES_HTML >> $GOOD_SECTION_HTML echo "
Reproducible artifacts for $i
" >> $GOOD_SECTION_HTML rm $GOOD_FILES_HTML fi if [ -s $BAD_FILES_HTML ] ; then echo " " >> $BAD_SECTION_HTML cat $BAD_FILES_HTML >> $BAD_SECTION_HTML echo "
Unreproducible artifacts for $i
" >> $BAD_SECTION_HTML rm $BAD_FILES_HTML fi done GOOD_PERCENT=$(echo "scale=1 ; ($GOOD_FILES*100/$ALL_FILES)" | bc) # are we there yet? if [ "$GOOD_PERCENT" = "100.0" ] ; then MAGIC_SIGN="!" else MAGIC_SIGN="?" fi # # finally create the webpage # cd $TMPDIR ; mkdir netbsd PAGE=netbsd/netbsd.html cat > $PAGE <<- EOF Reproducible NetBSD $MAGIC_SIGN
EOF write_page_intro NetBSD write_page "

$GOOD_FILES ($GOOD_PERCENT%) out of $ALL_FILES built NetBSD files were reproducible in our test setup" if [ "$GOOD_PERCENT" = "100.0" ] ; then write_page "!" else write_page "." fi write_page " These tests were last run on $DATE for version ${NETBSD_VERSION} with MKREPRO=yes and MKREPRO_TIMESTAMP=$MKREPRO_TIMESTAMP and were compared using ${DIFFOSCOPE}.

" write_variation_table NetBSD cat $BAD_SECTION_HTML >> $PAGE cat $GOOD_SECTION_HTML >> $PAGE write_page "

"
echo -n "$NETBSD" >> $PAGE
write_page "     

" write_page "
" write_page_footer NetBSD publish_page rm -f $FILES_HTML $GOOD_FILES_HTML $BAD_FILES_HTML $GOOD_SECTION_HTML $BAD_SECTION_HTML # the end calculate_build_duration print_out_duration irc_message reproducible-builds "$REPRODUCIBLE_URL/netbsd/ has been updated. ($GOOD_PERCENT% reproducible)" echo "=============================================================================" # remove everything, we don't need it anymore... cleanup_tmpdirs trap - INT TERM EXIT