#!/bin/bash # Copyright 2014-2015 Holger Levsen # © 2015 Mattia Rizzolo # released under the GPLv=2 DEBUG=false . /srv/jenkins/bin/common-functions.sh common_init "$@" # common code defining db access . /srv/jenkins/bin/reproducible_common.sh set -e cleanup_tmpdirs() { cd rm -r $TMPDIR rm -r $TMPBUILDDIR } create_results_dirs() { mkdir -p $BASE/openwrt/dbd } call_debbindiff() { mkdir -p $TMPDIR/$1/$(dirname $2) local TMPLOG=(mktemp --tmpdir=$TMPDIR) local msg="" set +e ( timeout $TIMEOUT schroot \ --directory $TMPDIR \ -c source:jenkins-reproducible-${DBDSUITE}-debbindiff \ debbindiff -- \ --html $TMPDIR/$1/$2.html \ $TMPDIR/b1/$1/$2 \ $TMPDIR/b2/$1/$2 2>&1 \ ) 2>&1 >> $TMPLOG RESULT=$? if ! "$DEBUG" ; then set +x ; fi set -e cat $TMPLOG # print dbd output rm -f $TMPLOG case $RESULT in 0) echo "$(date -u) - $1/$2 is reproducible, yay!" ;; 1) echo "$(date -u) - $DBDVERSION found issues, please investigate $1/$2" ;; 2) msg="$(date -u) - $DBDVERSION had trouble comparing the two builds. Please investigate $1/$2" ;; 124) if [ ! -s $TMPDIR/$1.html ] ; then msg="$(date -u) - $DBDVERSION produced no output for $1/$2 and was killed after running into timeout after ${TIMEOUT}..." else msg="$DBDVERSION was killed after running into timeout after $TIMEOUT, but there is still $TMPDIR/$1/$2.html" fi ;; *) msg="$(date -u) - Something weird happened when running $DBDVERSION on $1/$2 (which exited with $RESULT) and I don't know how to handle it." ;; esac if [ ! -z "$msg" ] ; then echo $msg | tee -a $TMPDIR/$1/$2.html fi } save_openwrt_results(){ RUN=$1 cd bin for i in * ; do cd $i # save images mkdir -p $TMPDIR/$RUN/$i for j in $(find * -name "*.bin" -o -name "*.squashfs") ; do cp -p $j $TMPDIR/$RUN/$i/ done # save packages cd packages for j in $(find * -name "*.ipk") ; do mkdir -p $TMPDIR/$RUN/$i/$(dirname $j) cp -p $j $TMPDIR/$RUN/$i/$(dirname $j)/ done cd ../.. done cd .. } openwrt_config() { CONFIG=$1 printf "$CONFIG" > .config make defconfig } openwrt_build_toolchain() { echo "=============================================================================" echo "$(date -u) - Building the toolchain." echo "=============================================================================" ionice -c 3 nice \ make -j $NUM_CPU tools/install ionice -c 3 nice \ make -j $NUM_CPU toolchain/install } openwrt_build() { RUN=$1 TARGET=$2 echo "=============================================================================" echo "$(date -u) - Building OpenWrt ${OPENWRT_VERSION} ($TARGET) - $RUN build run." echo "=============================================================================" ionice -c 3 nice \ $MAKE -j $NUM_CPU target/compile ionice -c 3 nice \ $MAKE -j $NUM_CPU package/cleanup ionice -c 3 nice \ $MAKE -j $NUM_CPU package/compile || true # don't let some packages fail the whole build ionice -c 3 nice \ $MAKE -j $NUM_CPU package/install ionice -c 3 nice \ $MAKE -j $NUM_CPU target/install ionice -c 3 nice \ $MAKE -j $NUM_CPU package/index # temporary workaround. we see openssl _sometimes_ failing. # build it explicitely again with verbose output ionice -c 3 nice \ $MAKE package/openssl/{clean,compile} -j1 V=99 } openwrt_cleanup() { rm build_dir/target-* -r rm staging_dir/target-* -r rm bin/* -r } build_two_times() { TARGET=$1 CONFIG=$2 openwrt_config $CONFIG openwrt_build_toolchain # FIRST BUILD export TZ="/usr/share/zoneinfo/Etc/GMT+12" MAKE=make openwrt_build "first" "$TARGET" # save results in b1 save_openwrt_results b1 # clean up between builds openwrt_cleanup # SECOND BUILD export TZ="/usr/share/zoneinfo/Etc/GMT-14" export LANG="fr_CH.UTF-8" export LC_ALL="fr_CH.UTF-8" export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path" export CAPTURE_ENVIRONMENT="I capture the environment" umask 0002 # use allmost all cores for second build NEW_NUM_CPU=$(echo $NUM_CPU-1|bc) MAKE="linux64 --uname-2.6 make" openwrt_build "second" "$TARGET" # save results in b2 save_openwrt_results b2 # reset environment to default values again export LANG="en_GB.UTF-8" unset LC_ALL export TZ="/usr/share/zoneinfo/UTC" export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:" umask 0022 # clean up again openwrt_cleanup } # # main # TMPBUILDDIR=$(mktemp --tmpdir=/srv/workspace/chroots/ -d -t openwrt-XXXXXXXX) # used to build on tmpfs TMPDIR=$(mktemp --tmpdir=/srv/reproducible-results -d) # accessable in schroots, used to compare results DATE=$(date -u +'%Y-%m-%d') START=$(date +'%s') trap cleanup_tmpdirs INT TERM EXIT cd $TMPBUILDDIR echo "=============================================================================" echo "$(date -u) - Cloning the OpenWrt git repository." echo "=============================================================================" git clone git://git.openwrt.org/openwrt.git cd openwrt OPENWRT="$(git log -1)" OPENWRT_VERSION=$(git describe --always) echo "This is openwrt $OPENWRT_VERSION." echo git log -1 echo "=============================================================================" echo "$(date -u) - Updating package feeds." echo "=============================================================================" ./scripts/feeds update -a ./scripts/feeds install -a build_two_times ar71xx_generic_ARCHERC7 "CONFIG_TARGET_ar71xx_generic=y\nCONFIG_TARGET_ar71xx_generic_ARCHERC7=y\n" build_two_times x86_64 "CONFIG_TARGET_x86=y\nCONFIG_TARGET_x86_64=y\n" build_two_times ramips_rt288x_RTN15 "CONFIG_TARGET_ramips=y\nCONFIG_TARGET_ramips_rt288x=y\nCONFIG_TARGET_ramips_rt288x_RTN15=y\n" # # create html about toolchain used # TOOLCHAIN_HTML=$(mktemp --tmpdir=$TMPDIR) echo "" > $TOOLCHAIN_HTML for i in $(ls -1d staging_dir/toolchain*|cut -d "-" -f2-|xargs echo) ; do echo " " >> $TOOLCHAIN_HTML done echo "
Target toolchains built
$i
" >> $TOOLCHAIN_HTML echo "" >> $TOOLCHAIN_HTML for i in $(ls -1 build_dir/host/) ; do echo " " >> $TOOLCHAIN_HTML done echo "
Contents of build_dir/host/
$i
" >> $TOOLCHAIN_HTML echo "" >> $TOOLCHAIN_HTML for i in $(ls -1 dl/) ; do echo " " >> $TOOLCHAIN_HTML done echo "
Downloaded software
$i
" >> $TOOLCHAIN_HTML echo "" >> $TOOLCHAIN_HTML for i in gcc binutils bzip2 flex python perl make findutils grep diffutils unzip gawk util-linux zlib1g-dev libc6-dev git subversion ; do echo " " >> $TOOLCHAIN_HTML done echo "
Debian $(cat /etc/debian_version) package on $(dpkg --print-architecture)installed version
$i" >> $TOOLCHAIN_HTML dpkg -s $i|grep '^Version'|cut -d " " -f2 >> $TOOLCHAIN_HTML echo "
" >> $TOOLCHAIN_HTML # get banner BANNER_HTML=$(mktemp --tmpdir=$TMPDIR) cat $(find build_dir/ -name banner | grep etc/banner|head -1) >> $BANNER_HTML # clean up builddir to save space on tmpfs rm -r $TMPBUILDDIR/openwrt # run debbindiff on the results # (this needs refactoring rather badly) TIMEOUT="30m" DBDSUITE="unstable" DBDVERSION="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-debbindiff debbindiff -- --version 2>&1)" echo "=============================================================================" echo "$(date -u) - Running $DBDVERSION on OpenWrt images and packages." echo "=============================================================================" DBD_HTML=$(mktemp --tmpdir=$TMPDIR) # run debbindiff on the images GOOD_IMAGES=0 ALL_IMAGES=0 create_results_dirs cd $TMPDIR/b1 for i in * ; do cd $i echo " " >> $DBD_HTML for j in $(find * -name "*.bin" -o -name "*.squashfs" |sort -u ) ; do let ALL_IMAGES+=1 call_debbindiff $i $j SIZE="$(du -h -b $j | cut -f1)" SIZE="$(echo $SIZE/1024|bc)" if [ -f $TMPDIR/$i/$j.html ] ; then mkdir -p $BASE/openwrt/dbd/$i mv $TMPDIR/$i/$j.html $BASE/openwrt/dbd/$i/$j.html echo " " >> $DBD_HTML else SHASUM=$(sha256sum $j|cut -d " " -f1) echo " " >> $DBD_HTML let GOOD_IMAGES+=1 rm -f $BASE/openwrt/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed fi done cd .. echo "
Images for $i
\"unreproducible $j (${SIZE}K) is unreproducible.
\"reproducible $j ($SHASUM, ${SIZE}K) is reproducible.
" >> $DBD_HTML done GOOD_PERCENT_IMAGES=$(echo "scale=1 ; ($GOOD_IMAGES*100/$ALL_IMAGES)" | bc) # run debbindiff on the packages GOOD_PACKAGES=0 ALL_PACKAGES=0 create_results_dirs cd $TMPDIR/b1 tree . for i in * ; do cd $i echo " " >> $DBD_HTML for j in $(find * -name "*.ipk" |sort -u ) ; do let ALL_PACKAGES+=1 call_debbindiff $i $j SIZE="$(du -h -b $j | cut -f1)" SIZE="$(echo $SIZE/1024|bc)" if [ -f $TMPDIR/$i/$j.html ] ; then mkdir -p $BASE/openwrt/dbd/$i/$(dirname $j) mv $TMPDIR/$i/$j.html $BASE/openwrt/dbd/$i/$j.html echo " " >> $DBD_HTML else SHASUM=$(sha256sum $j|cut -d " " -f1) echo " " >> $DBD_HTML let GOOD_PACKAGES+=1 rm -f $BASE/openwrt/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed fi done cd .. echo "
Packages for $i
\"unreproducible $j (${SIZE}K) is unreproducible.
\"reproducible $j ($SHASUM, ${SIZE}K) is reproducible.
" >> $DBD_HTML done GOOD_PERCENT_PACKAGES=$(echo "scale=1 ; ($GOOD_PACKAGES*100/$ALL_PACKAGES)" | bc) # are we there yet? if [ "$GOOD_PERCENT_IMAGES" = "100.0" ] || [ "$GOOD_PERCENT_PACKAGES" = "100.0" ]; then MAGIC_SIGN="!" else MAGIC_SIGN="?" fi # # finally create the webpage # cd $TMPDIR ; mkdir openwrt PAGE=openwrt/openwrt.html cat > $PAGE <<- EOF openwrt
" write_page "

OpenWrt - reproducible wireless freedom$MAGIC_SIGN

" write_page_intro OpenWrt write_page "

$GOOD_IMAGES ($GOOD_PERCENT_IMAGES%) out of $ALL_IMAGES built images and $GOOD_PACKAGES ($GOOD_PERCENT_PACKAGES%) out of $ALL_PACKAGES built packages were reproducible in our test setup." write_page " These tests were last run on $DATE for version ${OPENWRT_VERSION}.

" write_explaination_table OpenWrt cat $DBD_HTML >> $PAGE write_page "
git commit built
" echo -n "$OPENWRT" >> $PAGE write_page "
" cat $TOOLCHAIN_HTML >> $PAGE write_page "
" write_page_footer OpenWrt publish_page rm -f $DBD_HTML $TOOLCHAIN_HTML $BANNER_HTML # the end calculate_build_duration print_out_duration irc_message "$REPRODUCIBLE_URL/openwrt/ has been updated. ($GOOD_PERCENT_IMAGES% images and $GOOD_PERCENT_PACKAGES% reproducible)" echo "=============================================================================" # remove everything, we don't need it anymore... cleanup_tmpdirs trap - INT TERM EXIT