diff options
| -rw-r--r-- | README | 2 | ||||
| -rw-r--r-- | TODO | 2 | ||||
| -rwxr-xr-x | bin/reproducible_breakages.py | 20 | ||||
| -rwxr-xr-x | bin/reproducible_build.sh | 16 | ||||
| -rwxr-xr-x | bin/reproducible_common.sh | 8 | ||||
| -rwxr-xr-x | bin/reproducible_coreboot.sh | 4 | ||||
| -rwxr-xr-x | bin/reproducible_freebsd.sh | 4 | ||||
| -rwxr-xr-x | bin/reproducible_html_packages.py | 2 | ||||
| -rwxr-xr-x | bin/reproducible_html_repository_comparison.sh | 6 | ||||
| -rwxr-xr-x | bin/reproducible_netbsd.sh | 4 | ||||
| -rwxr-xr-x | bin/reproducible_openwrt.sh | 10 | ||||
| -rwxr-xr-x | bin/schroot-create.sh | 8 |
12 files changed, 43 insertions, 43 deletions
@@ -152,7 +152,7 @@ Installation tests inside chroot environments. * There are suite specific jobs to create the pbuilder base.tgz's per suite, which have the reproducible apt repo added. Similarly there's another job per suite to create the schroots used by the builder jobs to download the packages sources to build. -* Then there are two more jobs to create sid and testing schroots to run debbindiff on the the two results. This is necessary since to investigate haskell binaries, debbindiff needs access to the same haskell compiler version as the investigated packages have been built with. +* Then there are two more jobs to create sid and testing schroots to run diffoscope on the the two results. This is necessary since to investigate haskell binaries, diffoscope needs access to the same haskell compiler version as the investigated packages have been built with. * For making sure things are considerably under control at any time, there is a maintenance job running every 4h, mostly doing cleanups. @@ -173,7 +173,7 @@ properties: * higher prio: ** fix https://jenkins.debian.net/munin/debian.net/jenkins.debian.net/jenkins_builds.html which is broken since jessie upgrade ** repo-comparison: check for binaries without source -** s#debbindiff#diffoscope#g and s#dbd#ds#g everywhere and manually rename the jobs accordingly... +** s#debbindiff#diffoscope#g and s#dbd#ds#g everywhere and manually rename the jobs accordingly... also the .debbindiff.html+txt files need to be renamed, as well as the dbd directories... * lesser prio ** document (in README) the multihost setup diff --git a/bin/reproducible_breakages.py b/bin/reproducible_breakages.py index 0551c9e9..4cbfa70b 100755 --- a/bin/reproducible_breakages.py +++ b/bin/reproducible_breakages.py @@ -27,14 +27,14 @@ def unrep_with_dbd_issues(): if not os.access(dbd, os.R_OK): without_dbd.append((pkg, version, suite, arch)) log.warning(pkg + '/' + suite + ' (' + version + ') is ' - 'unreproducible without debbindiff file.') + 'unreproducible without diffoscope file.') else: log.debug(dbd + ' found.') data = open(dbd, 'br').read(3) if b'<' not in data: bad_dbd.append((pkg, version, suite, arch)) log.warning(pkg + '/' + suite + ' (' + version + ') has ' - 'debbindiff output, but it does not seem to ' + 'diffoscope output, but it does not seem to ' 'be an html page.') return without_dbd, bad_dbd @@ -54,7 +54,7 @@ def not_unrep_with_dbd_file(): if os.access(dbd, os.R_OK): bad_pkgs.append((pkg, version, suite, arch)) log.warning(pkg + '/' + suite + ' (' + version + ') has a ' - 'debbindiff file but it\'s not unreproducible.') + 'diffoscope file but it\'s not unreproducible.') return bad_pkgs @@ -262,28 +262,28 @@ def gen_html(): # files that should not be there (e.g. removed package without cleanup) html += _gen_section('log files that should not be there', None, entries=alien_log()) - html += _gen_section('debbindiff files that should not be there:', None, + html += _gen_section('diffoscope files that should not be there:', None, entries=alien_dbd()) html += _gen_section('rb-pkg pages that should not be there:', None, entries=alien_rbpkg()) html += _gen_section('buildinfo files that should not be there:', None, entries=alien_buildinfo()) - # debbindiff report where it shouldn't be + # diffoscope report where it shouldn't be html += _gen_section('are not marked as unreproducible, but they ' + - 'have a debbindiff file:', not_unrep_with_dbd_file()) + 'have a diffoscope file:', not_unrep_with_dbd_file()) # missing files html += _gen_section('are built but don\'t have a buildlog:', lack_rbuild()) html += _gen_section('are built but don\'t have a .buildinfo file:', lack_buildinfo()) - # debbindiff troubles + # diffoscope troubles without_dbd, bad_dbd = unrep_with_dbd_issues() html += _gen_section('are marked as unreproducible, but without ' + - 'debbindiff output - so probably debbindiff ' + + 'diffoscope output - so probably diffoscope ' + 'crashed:', without_dbd) html += _gen_section('are marked as unreproducible, but their ' + - 'debbindiff output does not seem to be an html ' + - 'file - so probably debbindiff ran into a ' + + 'diffoscope output does not seem to be an html ' + + 'file - so probably diffoscope ran into a ' + 'timeout:', bad_dbd) # pbuilder-satisfydepends failed html += _gen_section('failed to match their build-dependencies:', diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh index 8b142ab2..dc4690c2 100755 --- a/bin/reproducible_build.sh +++ b/bin/reproducible_build.sh @@ -81,7 +81,7 @@ save_artifacts() { # irc message if [ ! -z "$NOTIFY" ] ; then local MESSAGE="$URL published" - if [ "$NOTIFY" = "debbindiff" ] ; then + if [ "$NOTIFY" = "diffoscope" ] ; then MESSAGE="$MESSAGE, $DBDVERSION had troubles with these..." fi irc_message "$MESSAGE" @@ -118,8 +118,8 @@ update_db_and_html() { MESSAGE="${REPRODUCIBLE_URL}/${SUITE}/${ARCH}/${SRCPACKAGE} : reproducible ➤ ${STATUS}" echo "\n$MESSAGE" | tee -a ${RBUILDLOG} irc_message "$MESSAGE" - # disable ("regular") irc notification unless it's due to debbindiff problems - if [ ! -z "$NOTIFY" ] && [ "$NOTIFY" != "debbindiff" ] ; then + # disable ("regular") irc notification unless it's due to diffoscope problems + if [ ! -z "$NOTIFY" ] && [ "$NOTIFY" != "diffoscope" ] ; then NOTIFY="" fi fi @@ -255,7 +255,7 @@ handle_reproducible() { calculate_build_duration update_db_and_html "reproducible" elif [ -f ./$DBDREPORT ] ; then - echo "Debbindiff says the build is reproducible, but there is a debbindiff file. Please investigate" | tee -a $RBUILDLOG + echo "Debbindiff says the build is reproducible, but there is a diffoscope file. Please investigate" | tee -a $RBUILDLOG handle_ftbr elif [ ! -f b1/$BUILDINFO ] ; then echo "Debbindiff says the build is reproducible, but there is no .buildinfo file. Please investigate" | tee -a $RBUILDLOG @@ -271,11 +271,11 @@ dbd_timeout() { msg="$msg, but there is still $REPRODUCIBLE_URL/dbd/$SUITE/$ARCH/$DDBREPORT" fi SAVE_ARTIFACTS=1 - NOTIFY="debbindiff" + NOTIFY="diffoscope" handle_ftbr "$msg" } -call_debbindiff_on_changes_files() { +call_diffoscope_on_changes_files() { local TMPLOG=(mktemp --tmpdir=$TMPDIR) echo | tee -a ${RBUILDLOG} local TIMEOUT="30m" @@ -314,7 +314,7 @@ call_debbindiff_on_changes_files() { ;; 2) SAVE_ARTIFACTS=1 - NOTIFY="debbindiff" + NOTIFY="diffoscope" handle_ftbr "$DBDVERSION had trouble comparing the two builds. Please investigate $REPRODUCIBLE_URL/rbuild/${SUITE}/${ARCH}/${SRCPACKAGE}_${EVERSION}.rbuild.log" ;; 124) @@ -561,7 +561,7 @@ update_rbuildlog if [ $FTBFS -eq 1 ] ; then handle_ftbfs elif [ $FTBFS -eq 0 ] ; then - call_debbindiff_on_changes_files # defines DBDVERSION, update_db_and_html defines STATUS + call_diffoscope_on_changes_files # defines DBDVERSION, update_db_and_html defines STATUS fi check_for_race_conditions diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 1ce6f630..a87bc81d 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -237,15 +237,15 @@ write_page_header() { write_page_intro() { write_page " <p><em>Reproducible builds</em> enable anyone to reproduce bit by bit identical binary packages from a given source, so that anyone can verify that a given binary derived from the source it was said to be derived. There is a lot more information about <a href=\"https://wiki.debian.org/ReproducibleBuilds\">reproducible builds on the Debian wiki</a> and on <a href=\"https://reproducible.debian.net\">https://reproducible.debian.net</a>. The wiki explains in more depth why this is useful, what common issues exist and which workarounds and solutions are known.<br />" if [ "$1" = "coreboot" ] ; then - write_page " <em>Reproducible Coreboot</em> is an effort to apply this to coreboot. Thus each coreboot.rom is build twice (without payloads), with a few varitations added and then those two ROMs are compared using <a href=\"https://tracker.debian.org/debbindiff\">debbindiff</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>" + write_page " <em>Reproducible Coreboot</em> is an effort to apply this to coreboot. Thus each coreboot.rom is build twice (without payloads), with a few varitations added and then those two ROMs are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>" local PROJECTNAME="$1" local PROJECTURL="https://review.coreboot.org/p/coreboot.git" elif [ "$1" = "OpenWrt" ] ; then - write_page " <em>Reproducible OpenWrt</em> is an effort to apply this to OpenWrt. Thus each OpenWrt target is build twice, with a few varitations added and then the resulting images and packages from the two builds are compared using <a href=\"https://tracker.debian.org/debbindiff\">debbindiff</a>, <em>which currently cannot detect <code>.bin</code> files as squashfs filesystems.</em> Thus the resulting debbindiff output is not nearly as clear as it could be - hopefully this limitation will be overcome soon. Also please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>" + write_page " <em>Reproducible OpenWrt</em> is an effort to apply this to OpenWrt. Thus each OpenWrt target is build twice, with a few varitations added and then the resulting images and packages from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>, <em>which currently cannot detect <code>.bin</code> files as squashfs filesystems.</em> Thus the resulting diffoscope output is not nearly as clear as it could be - hopefully this limitation will be overcome soon. Also please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>" local PROJECTNAME="openwrt" local PROJECTURL="git://git.openwrt.org/openwrt.git" elif [ "$1" = "NetBSD" ] ; then - write_page " <em>Reproducible NetBSD</em> is an effort to apply this to NetBSD. Thus each NetBSD target is build twice, with a few varitations added and then the resulting files from the two builds are compared using <a href=\"https://tracker.debian.org/debbindiff\">debbindiff</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>" + write_page " <em>Reproducible NetBSD</em> is an effort to apply this to NetBSD. Thus each NetBSD target is build twice, with a few varitations added and then the resulting files from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>" local PROJECTNAME="netbsd" local PROJECTURL="https://github.com/jsonn/src" fi @@ -371,7 +371,7 @@ irc_message() { kgb-client --conf /srv/jenkins/kgb/debian-reproducible.conf --relay-msg "$MESSAGE" || true # don't fail the whole job } -call_debbindiff() { +call_diffoscope() { mkdir -p $TMPDIR/$1/$(dirname $2) local TMPLOG=(mktemp --tmpdir=$TMPDIR) local msg="" diff --git a/bin/reproducible_coreboot.sh b/bin/reproducible_coreboot.sh index cb368803..fb7e011b 100755 --- a/bin/reproducible_coreboot.sh +++ b/bin/reproducible_coreboot.sh @@ -148,7 +148,7 @@ save_coreboot_results b2 # clean up builddir to save space on tmpfs rm -r $TMPBUILDDIR/coreboot -# run debbindiff on the results +# run diffoscope on the results TIMEOUT="30m" DBDSUITE="unstable" DBDVERSION="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-debbindiff debbindiff -- --version 2>&1)" @@ -166,7 +166,7 @@ cd $TMPDIR/b1 for i in $(ls -1d *| sort -u) ; do let ALL_ROMS+=1 if [ -f $i/coreboot.rom ] ; then - call_debbindiff $i coreboot.rom + call_diffoscope $i coreboot.rom get_filesize $i/coreboot.rom if [ -f $TMPDIR/$i.html ] ; then mv $TMPDIR/$i.html $BASE/coreboot/dbd/$i.html diff --git a/bin/reproducible_freebsd.sh b/bin/reproducible_freebsd.sh index 5f08ce59..b4d90df8 100755 --- a/bin/reproducible_freebsd.sh +++ b/bin/reproducible_freebsd.sh @@ -111,7 +111,7 @@ umask 0022 # cleanup on the node $RSSH 'sudo rm -r /usr/src /usr/obj' -# run debbindiff on the results +# run diffoscope on the results TIMEOUT="30m" DBDSUITE="unstable" DBDVERSION="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-debbindiff debbindiff -- --version 2>&1)" @@ -131,7 +131,7 @@ for i in * ; do echo " <table><tr><th>Release files for <code>$i</code></th></tr>" >> $FILES_HTML for j in $(find * -type f |sort -u ) ; do let ALL_FILES+=1 - call_debbindiff $i $j + call_diffoscope $i $j get_filesize $j if [ -f $TMPDIR/$i/$j.html ] ; then mkdir -p $BASE/freebsd/dbd/$i/$(dirname $j) diff --git a/bin/reproducible_html_packages.py b/bin/reproducible_html_packages.py index 92b9a43a..0d46cce2 100755 --- a/bin/reproducible_html_packages.py +++ b/bin/reproducible_html_packages.py @@ -116,7 +116,7 @@ def gen_extra_links(package, version, suite, arch, status): log.debug('debbindiff not detetected at ' + dbd) if status == 'unreproducible' and not args.ignore_missing_files: log.critical(REPRODUCIBLE_URL + '/' + suite + '/' + arch + '/' + package + - ' is unreproducible, but without debbindiff output.') + ' is unreproducible, but without diffoscope output.') if os.access(dbdtxt, os.R_OK): url = DBDTXT_URI + '/' + suite + '/' + arch + '/' + package + '_' + \ eversion + '.debbindiff.txt' diff --git a/bin/reproducible_html_repository_comparison.sh b/bin/reproducible_html_repository_comparison.sh index 8be5bb4d..af3a2ca2 100755 --- a/bin/reproducible_html_repository_comparison.sh +++ b/bin/reproducible_html_repository_comparison.sh @@ -147,7 +147,7 @@ for PKG in $SOURCEPKGS ; do write_row "<tr><td><pre>src:$PKG</pre></td>" write_row " <td>" case $PKG in - strip-nondeterminism|debbindiff) + strip-nondeterminism|debbindiff|diffoscope) URL="http://anonscm.debian.org/cgit/reproducible/$PKG.git" ;; *) URL="http://anonscm.debian.org/cgit/reproducible/$PKG.git/?h=pu/reproducible_builds" ;; @@ -177,7 +177,7 @@ for PKG in $SOURCEPKGS ; do fi else write_row "<a href=\"$URL\">$PKG.git</a>" - if [ "$PKG" != "strip-nondeterminism" ] && [ "$PKG" != "debbindiff" ] ; then + if [ "$PKG" != "strip-nondeterminism" ] && [ "$PKG" != "diffoscope" ] ; then if $OBSOLETE_IN_TESTING && $OBSOLETE_IN_SID && $OBSOLETE_IN_EXP ; then write_row "<br />(unused?" write_row "<br /><span class=\"purple\">Then the branch should probably renamed.</span>" @@ -187,7 +187,7 @@ for PKG in $SOURCEPKGS ; do elif $OBSOLETE_IN_EXP ; then write_row "<br />(only used in testing and unstable, fixed in experimental)" fi - elif ( [ "$PKG" = "strip-nondeterminism" ] || [ "$PKG" = "debbindiff" ] ) && $OBSOLETE_IN_SID ; then + elif ( [ "$PKG" = "strip-nondeterminism" ] || [ "$PKG" = "diffoscope" ] ) && $OBSOLETE_IN_SID ; then write_row "<br />(this repo is always used)" fi fi diff --git a/bin/reproducible_netbsd.sh b/bin/reproducible_netbsd.sh index 2ad6d6a0..151c4acc 100755 --- a/bin/reproducible_netbsd.sh +++ b/bin/reproducible_netbsd.sh @@ -117,7 +117,7 @@ umask 0022 # clean up builddir to save space on tmpfs rm -r $TMPBUILDDIR/netbsd -# run debbindiff on the results +# run diffoscope on the results TIMEOUT="30m" DBDSUITE="unstable" DBDVERSION="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-debbindiff debbindiff -- --version 2>&1)" @@ -137,7 +137,7 @@ for i in * ; do echo " <table><tr><th>Release files for <code>$i</code></th></tr>" >> $FILES_HTML for j in $(find * -type f |sort -u ) ; do let ALL_FILES+=1 - call_debbindiff $i $j + call_diffoscope $i $j get_filesize $j if [ -f $TMPDIR/$i/$j.html ] ; then mkdir -p $BASE/netbsd/dbd/$i/$(dirname $j) diff --git a/bin/reproducible_openwrt.sh b/bin/reproducible_openwrt.sh index cf73f03f..d9daca64 100755 --- a/bin/reproducible_openwrt.sh +++ b/bin/reproducible_openwrt.sh @@ -201,7 +201,7 @@ cat $(find build_dir/ -name banner | grep etc/banner|head -1) >> $BANNER_HTML # clean up builddir to save space on tmpfs rm -r $TMPBUILDDIR/openwrt -# run debbindiff on the results +# run diffoscope on the results # (this needs refactoring rather badly) TIMEOUT="30m" DBDSUITE="unstable" @@ -210,7 +210,7 @@ echo "========================================================================== echo "$(date -u) - Running $DBDVERSION on OpenWrt images and packages." echo "=============================================================================" DBD_HTML=$(mktemp --tmpdir=$TMPDIR) -# run debbindiff on the images +# run diffoscope on the images GOOD_IMAGES=0 ALL_IMAGES=0 SIZE="" @@ -234,7 +234,7 @@ for i in * ; do rm -f $BASE/openwrt/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed continue fi - call_debbindiff $i $j + call_diffoscope $i $j get_filesize $j if [ -f $TMPDIR/$i/$j.html ] ; then mkdir -p $BASE/openwrt/dbd/$i @@ -251,7 +251,7 @@ for i in * ; do echo " </table>" >> $DBD_HTML done GOOD_PERCENT_IMAGES=$(echo "scale=1 ; ($GOOD_IMAGES*100/$ALL_IMAGES)" | bc) -# run debbindiff on the packages +# run diffoscope on the packages GOOD_PACKAGES=0 ALL_PACKAGES=0 create_results_dirs @@ -273,7 +273,7 @@ for i in * ; do rm -f $BASE/openwrt/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed continue fi - call_debbindiff $i $j + call_diffoscope $i $j get_filesize $j if [ -f $TMPDIR/$i/$j.html ] ; then mkdir -p $BASE/openwrt/dbd/$i/$(dirname $j) diff --git a/bin/schroot-create.sh b/bin/schroot-create.sh index 25e7fa48..adecb729 100755 --- a/bin/schroot-create.sh +++ b/bin/schroot-create.sh @@ -127,17 +127,17 @@ bootstrap() { done set -x sudo chroot $CHROOT_TARGET apt-get update - # install debbindiff with all recommends... + # install diffoscope with all recommends... if [ "$1" = "debbindiff" ] ; then sudo chroot $CHROOT_TARGET apt-get install -y --install-recommends debbindiff fi sudo chroot $CHROOT_TARGET apt-get install -y --no-install-recommends "$@" sudo - # always use debbindiff from unstable + # always use diffoscope from unstable if [ "$SUITE" = "testing" ] && [ "$1" = "debbindiff" ] ; then echo "deb $MIRROR unstable main" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list > /dev/null sudo chroot $CHROOT_TARGET apt-get update - # install debbindiff from unstable without re-adding all recommends... - sudo chroot $CHROOT_TARGET apt-get install -y -t unstable --no-install-recommends debbindiff || echo "Warning: debbindiff from unstable is uninstallable at the moment." + # install diffoscope from unstable without re-adding all recommends... + sudo chroot $CHROOT_TARGET apt-get install -y -t unstable --no-install-recommends debbindiff || echo "Warning: diffoscope from unstable is uninstallable at the moment." fi if ! $DEBUG ; then set +x ; fi if [ "$1" = "debbindiff" ] ; then |