diff options
| -rw-r--r-- | TODO | 2 | ||||
| -rwxr-xr-x | bin/reproducible_coreboot.sh | 2 | ||||
| -rwxr-xr-x | bin/reproducible_openwrt.sh | 149 | ||||
| -rw-r--r-- | job-cfg/reproducible.yaml | 9 |
4 files changed, 99 insertions, 63 deletions
@@ -251,8 +251,6 @@ egrep -R -l '(debbindiff had trouble comparing|maybe there is still )' /var/lib/ ---- <nbd> wenn man einmal make defconfig ausgeführt hat, gibt's paket und target metadaten in tmp/.packageinfo und tmp/.targetinfo <nbd> man kann ne .config mit der target auswahl und CONFIG_ALL=y erstellen und mit make defconfig auffüllen - <nbd> macht wahrscheinlich sinn vom prozess her das bauen von paketen und das bauen der toolchain zusammen laufen zu lassen - <nbd> weil openwrt das ja alles in einem durchgang macht <nbd> reicht ja erstmal, mit dem basis paketsatz anzufangen <nbd> das wird für ein target deutlich weniger als 12h brauchen <nbd> default config ohne CONFIG_ALL dauert bei meinem laptop in der regel <30 min diff --git a/bin/reproducible_coreboot.sh b/bin/reproducible_coreboot.sh index f612c13d..972bb8a3 100755 --- a/bin/reproducible_coreboot.sh +++ b/bin/reproducible_coreboot.sh @@ -276,7 +276,7 @@ rm -f $ROMS_HTML $TOOLCHAIN_HTML # the end calculate_build_duration print_out_duration -irc_message "$REPRODUCIBLE_URL/coreboot/ has been updated." +irc_message "$REPRODUCIBLE_URL/coreboot/ has been updated. ($GOOD_PERCENT% reproducible)" echo "=============================================================================" # remove everything, we don't need it anymore... diff --git a/bin/reproducible_openwrt.sh b/bin/reproducible_openwrt.sh index 322c5e39..82e8bc34 100755 --- a/bin/reproducible_openwrt.sh +++ b/bin/reproducible_openwrt.sh @@ -22,6 +22,7 @@ create_results_dirs() { } call_debbindiff() { + mkdir -p $TMPDIR/$1 local TMPLOG=(mktemp --tmpdir=$TMPDIR) local msg="" set +e @@ -29,9 +30,9 @@ call_debbindiff() { --directory $TMPDIR \ -c source:jenkins-reproducible-${DBDSUITE}-debbindiff \ debbindiff -- \ - --html $TMPDIR/$1.html \ - $TMPDIR/b1/$1/openwrt.rom \ - $TMPDIR/b2/$1/openwrt.rom 2>&1 \ + --html $TMPDIR/$1/$2.html \ + $TMPDIR/b1/$1/$2 \ + $TMPDIR/b2/$1/$2 2>&1 \ ) 2>&1 >> $TMPLOG RESULT=$? if ! "$DEBUG" ; then set +x ; fi @@ -39,27 +40,27 @@ call_debbindiff() { cat $TMPLOG # print dbd output rm -f $TMPLOG case $RESULT in - 0) echo "$(date -u) - $1/openwrt.rom is reproducible, yay!" + 0) echo "$(date -u) - $1/$2 is reproducible, yay!" ;; 1) - echo "$(date -u) - $DBDVERSION found issues, please investigate $1/openwrt.rom" + echo "$(date -u) - $DBDVERSION found issues, please investigate $1/$2" ;; 2) - msg="$(date -u) - $DBDVERSION had trouble comparing the two builds. Please investigate $1/openwrt.rom" + msg="$(date -u) - $DBDVERSION had trouble comparing the two builds. Please investigate $1/$2" ;; 124) if [ ! -s $TMPDIR/$1.html ] ; then - msg="$(date -u) - $DBDVERSION produced no output for $1/openwrt.rom and was killed after running into timeout after ${TIMEOUT}..." + msg="$(date -u) - $DBDVERSION produced no output for $1/$2 and was killed after running into timeout after ${TIMEOUT}..." else - msg="$DBDVERSION was killed after running into timeout after $TIMEOUT, but there is still $TMPDIR/$1.html" + msg="$DBDVERSION was killed after running into timeout after $TIMEOUT, but there is still $TMPDIR/$1/$2.html" fi ;; *) - msg="$(date -u) - Something weird happened when running $DBDVERSION on $1/openwrt.rom (which exited with $RESULT) and I don't know how to handle it." + msg="$(date -u) - Something weird happened when running $DBDVERSION on $1/$2 (which exited with $RESULT) and I don't know how to handle it." ;; esac if [ ! -z $msg ] ; then - echo $msg | tee -a $TMPDIR/$1.html + echo $msg | tee -a $TMPDIR/$1/$2.html fi } @@ -96,14 +97,22 @@ make -j $NUM_CPU tools/install # create html about toolchain used # TOOLCHAIN_HTML=$(mktemp) -echo "<table><tr><th>Debian $(cat /etc/debian_version) package on $(dpkg --print-architecture)</th><th>installed version</th></tr>" > $TOOLCHAIN_HTML +TARGET=$(ls -1d staging_dir/toolchain*|cut -d "-" -f2-) +echo "<table><tr><th>Contents of <pre>build_dir/host/</pre></th></tr>" > $TOOLCHAIN_HTML +for i in $(ls -1 build_dir/host/) ; do + echo " <tr><td>$i</td></tr>" >> $TOOLCHAIN_HTML +echo "</table>" >> $TOOLCHAIN_HTML +echo "<table><tr><th>Downloaded software built for <pre>$TARGET</pre></th></tr>" >> $TOOLCHAIN_HTML +for i in $(ls -1 dl/) ; do + echo " <tr><td>$i</td></tr>" >> $TOOLCHAIN_HTML +echo "</table>" >> $TOOLCHAIN_HTML +echo "<table><tr><th>Debian $(cat /etc/debian_version) package on $(dpkg --print-architecture)</th><th>installed version</th></tr>" >> $TOOLCHAIN_HTML for i in gcc binutils bzip2 flex python perl make findutils grep diff unzip gawk util-linux zlib1g-dev libc6-dev git subversion ; do echo " <tr><td>$i</td><td>" >> $TOOLCHAIN_HTML dpkg -s $i|grep '^Version'|cut -d " " -f2 >> $TOOLCHAIN_HTML echo " </td></tr>" >> $TOOLCHAIN_HTML done echo "</table>" >> $TOOLCHAIN_HTML -cd ../../.. echo "=============================================================================" echo "$(date -u) - Building openwrt ${OPENWRT_VERSION} images now - first build run." @@ -116,19 +125,30 @@ nice ionice -c 3 \ make -j $NUM_CPU package/cleanup nice ionice -c 3 \ make -j $NUM_CPU package/compile +nice ionice -c 3 \ + make -j $NUM_CPU package/install +nice ionice -c 3 \ + make -j $NUM_CPU target/install +nice ionice -c 3 \ + make -j $NUM_CPU package/index -cd openwrt-builds +cd bin for i in * ; do - # abuild and sharedutils are build results but not the results we are looking for... - if [ "$i" != "abuild" ] && [ "$i" != "sharedutils" ] ; then - mkdir $TMPDIR/b1/$i - if [ -f $i/openwrt.rom ] ; then - cp -p $i/openwrt.rom $TMPDIR/b1/$i/ - fi - fi + cd $i + mkdir $TMPDIR/b1/$i + for j in $(find . -name "*.bin") ; do + cp -p $j $TMPDIR/b1/$i/ + done + cd .. done cd .. -rm openwrt-builds -rf +rm bin -r + +# +# clean up between builds +# +rm staging_dir -r +rm target -r echo "=============================================================================" echo "$(date -u) - Building openwrt images now - second build run." @@ -150,6 +170,15 @@ nice ionice -c 3 \ nice ionice -c 3 \ linux64 --uname-2.6 \ make -j $NEW_NUM_CPU package/compile +nice ionice -c 3 \ + linux64 --uname-2.6 \ + make -j $NEW_NUM_CPU package/install +nice ionice -c 3 \ + linux64 --uname-2.6 \ + make -j $NEW_NUM_CPU target/install +nice ionice -c 3 \ + linux64 --uname-2.6 \ + make -j $NEW_NUM_CPU package/index # reset environment to default values again export LANG="en_GB.UTF-8" @@ -158,15 +187,17 @@ export TZ="/usr/share/zoneinfo/UTC" export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:" umask 0022 -cd openwrt-builds +cd bin for i in * ; do - if [ -f $i/openwrt.rom ] ; then - mkdir $TMPDIR/b2/$i - cp -p $i/openwrt.rom $TMPDIR/b2/$i/ - fi + cd $i + mkdir $TMPDIR/b2/$i + for j in $(find . -name "*.bin") ; do + cp -p $j $TMPDIR/b2/$i/ + done + cd .. done cd .. -rm openwrt-builds -r +rm bin -r # run debbindiff on the results TIMEOUT="30m" @@ -175,36 +206,34 @@ DBDVERSION="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE echo "=============================================================================" echo "$(date -u) - Running $DBDVERSION on openwrt images now" echo "=============================================================================" -ROMS_HTML=$(mktemp) -echo " <ul>" > $ROMS_HTML -BAD_ROMS=0 -GOOD_ROMS=0 -ALL_ROMS=0 +IMAGES_HTML=$(mktemp) +echo " <ul>" > $IMAGES_HTML +GOOD_IMAGES=0 +ALL_IMAGES=0 create_results_dirs cd $TMPDIR/b1 for i in * ; do - let ALL_ROMS+=1 - if [ -f $i/openwrt.rom ] ; then - call_debbindiff $i - SIZE="$(du -h -b $i/openwrt.rom | cut -f1)" + cd $i + for j in $(find . -name "*.bin") ; do + let ALL_IMAGES+=1 + call_debbindiff $i $j + SIZE="$(du -h -b $j | cut -f1)" SIZE="$(echo $SIZE/1024|bc)" - if [ -f $TMPDIR/$i.html ] ; then - mv $TMPDIR/$i.html $BASE/openwrt/dbd/$i.html - echo " <li><a href=\"dbd/$i.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $i</a> (${SIZE}K) is unreproducible.</li>" >> $ROMS_HTML + if [ -f $TMPDIR/$i/$j.html ] ; then + mkdir -p $BASE/openwrt/dbd/$i + mv $TMPDIR/$i/$j.html $BASE/openwrt/dbd/$i/$j.html + echo " <li><a href=\"dbd/$i/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> (${SIZE}K) is unreproducible.</li>" >> $IMAGES_HTML else - SHASUM=$(sha256sum $i/openwrt.rom|cut -d " " -f1) - echo " <li><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $i ($SHASUM, ${SIZE}K) is reproducible.</li>" >> $ROMS_HTML - let GOOD_ROMS+=1 - rm -f $BASE/openwrt/dbd/$i.html # cleanup from previous (unreproducible) tests - if needed + SHASUM=$(sha256sum $j|cut -d " " -f1) + echo " <li><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, ${SIZE}K) is reproducible.</li>" >> $IMAGES_HTML + let GOOD_IMAGES+=1 + rm -f $BASE/openwrt/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed fi - else - echo " <li><img src=\"/userContent/static/weather-storm.png\" alt=\"FTBFS icon\" /> $i <a href=\"${BUILD_URL}console\">failed to build</a> from source.</li>" >> $ROMS_HTML - let BAD_ROMS+=1 - fi + done + cd .. done -echo " </ul>" >> $ROMS_HTML -GOOD_PERCENT=$(echo "scale=1 ; ($GOOD_ROMS*100/$ALL_ROMS)" | bc) -BAD_PERCENT=$(echo "scale=1 ; ($BAD_ROMS*100/$ALL_ROMS)" | bc) +echo " </ul>" >> $IMAGES_HTML +GOOD_PERCENT=$(echo "scale=1 ; ($GOOD_IMAGES*100/$ALL_IMAGES)" | bc) # # finally create the webpage @@ -218,40 +247,40 @@ cat > $PAGE <<- EOF <meta charset="UTF-8"> <meta name="viewport" content="width=device-width"> <title>openwrt</title> - <link rel='stylesheet' id='twentyfourteen-style-css' href='landing_style.css?ver=4.0' type='text/css' media='all' /> + <!-- link rel='stylesheet' id='twentyfourteen-style-css' href='landing_style.css?ver=4.0' type='text/css' media='all' /--> </head> <body> <div class="content"> <div class="page-content"> <p> </p> - <p><center><img src="openwrt.png" width="300" class="alignnone size-medium wp-image-6" alt="openwrt" height="231" /><br /> + <p><center><img src="logo.png" alt="openwrt logo"" /><br /> <blockquote> <br /> - <strong>openwrt™</strong>: fast, flexible <em>and reproducible</em> Open Source firmware? + <strong>OpenWRT</strong>: <em>reproducible</em> wireless freedom? </blockquote> </center></p> EOF write_page " <h1>Reproducible OpenWRT</h1>" write_page " <p><em>Reproducible builds</em> enable anyone to reproduce bit by bit identical binary packages from a given source, so that anyone can verify that a given binary derived from the source it was said to be derived. There is a lot more information about <a href=\"https://wiki.debian.org/ReproducibleBuilds\">reproducible builds on the Debian wiki</a> and on <a href=\"https://reproducible.debian.net\">https://reproducible.debian.net</a>. The wiki has a lot more information, eg. why this is useful, what common issues exist and which workarounds and solutions are known.<br />" -write_page " <em>Reproducible OpenWRT</em> is an effort to apply this to openwrt. Thus each openwrt.rom is build twice (without payloads), with a few varitations added and then those two ROMs are compared using <a href=\"https://tracker.debian.org/debbindiff\">debbindiff</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>" -write_page " <p>There is a monthly run <a href=\"https://jenkins.debian.net/view/reproducible/job/reproducible_openwrt/\">jenkins job</a> to test the <code>master</code> branch of <a href=\"https://review.openwrt.org/p/openwrt.git\">openwrt.git</a>. Currently this job is triggered more often though, because this is still under development and brand new. The jenkins job is simply running <a href=\"http://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/tree/bin/reproducible_openwrt.sh\">reproducible_openwrt.sh</a> in a Debian environemnt and this script is solely responsible for creating this page. Feel invited to join <code>#debian-reproducible</code> (on irc.oftc.net) to request job runs whenever sensible. Patches and other <a href=\"mailto:reproducible-builds@lists.alioth.debian.org\">feedback</a> are very much appreciated!</p>" -write_page " <p>$GOOD_ROMS ($GOOD_PERCENT%) out of $ALL_ROMS built openwrt images were reproducible in our test setup, while $BAD_ROMS ($BAD_PERCENT%) failed to build from source." +write_page " <em>Reproducible OpenWRT</em> is an effort to apply this to OpenWRT. Thus each OpenWR target is build twice, with a few varitations added and then the resulting images from the two builds are compared using <a href=\"https://tracker.debian.org/debbindiff\">debbindiff</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>" +write_page " <p>There is a monthly run <a href=\"https://jenkins.debian.net/view/reproducible/job/reproducible_openwrt/\">jenkins job</a> to test the <code>master</code> branch of <a href=\"git://git.openwrt.org/openwrt.git\">openwrt.git</a>. Currently this job is triggered more often though, because this is still under development and brand new. The jenkins job is simply running <a href=\"http://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/tree/bin/reproducible_openwrt.sh\">reproducible_openwrt.sh</a> in a Debian environemnt and this script is solely responsible for creating this page. Feel invited to join <code>#debian-reproducible</code> (on irc.oftc.net) to request job runs whenever sensible. Patches and other <a href=\"mailto:reproducible-builds@lists.alioth.debian.org\">feedback</a> are very much appreciated!</p>" +write_page " <p>$GOOD_IMAGES ($GOOD_PERCENT%) out of $ALL_IMAGES built openwrt images were reproducible in our test setup." write_page " These tests were last run on $DATE for version ${OPENWRT_VERSION}.</p>" write_explaination_table openwrt -cat $ROMS_HTML >> $PAGE +cat $IMAGES_HTML >> $PAGE write_page " <p><pre>" echo -n "$OPENWRT" >> $PAGE write_page " </pre></p>" cat $TOOLCHAIN_HTML >> $PAGE write_page " </div></div>" -write_page_footer openwrt +write_page_footer OpenWRT publish_page -rm -f $ROMS_HTML $TOOLCHAIN_HTML +rm -f $IMAGES_HTML $TOOLCHAIN_HTML # the end calculate_build_duration print_out_duration -irc_message "$REPRODUCIBLE_URL/openwrt/ has been updated." +irc_message "$REPRODUCIBLE_URL/openwrt/ has been updated. ($GOOD_PERCENT% reproducible)" echo "=============================================================================" # remove everything, we don't need it anymore... diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index 1187724d..5b52fbd2 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -266,6 +266,10 @@ defaults: reproducible_other_projects name: '{name}_coreboot' +- job-template: + defaults: reproducible_other_projects + name: '{name}_openwrt' + - project: name: reproducible jobs: @@ -410,4 +414,9 @@ my_timed: '23 42 23 * *' my_shell: '/srv/jenkins/bin/reproducible_coreboot.sh' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' + - '{name}_coreboot': + my_description: 'Try to rebuild OpenWRT (some targets, some packages) twice (with variations) and investigate them regarding reproducibility.' + my_timed: '23 42 23 * *' + my_shell: '/srv/jenkins/bin/reproducible_openwrt.sh' + my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' |