#!/bin/bash # Copyright 2015-2016 Holger Levsen # released under the GPLv=2 DEBUG=false . /srv/jenkins/bin/common-functions.sh common_init "$@" # common code defining db access . /srv/jenkins/bin/reproducible_common.sh # overwrite defaults as we need this order here ARCHS="amd64 i386 armhf" VIEW=repositories PAGE=index_${VIEW}.html SOURCES=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) PACKAGES=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) TMPFILE=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) TABLE_TODO=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) TABLE_DONE=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) MODIFIED_IN_SID=0 MODIFIED_IN_EXP=0 BINNMUS_NEEDED=0 write_row() { echo "$1" >> $ROW } custom_curl() { echo -n "$(date -u) - downloading $1 to $2 - " curl -s $1 > $2 local SIZE=$(ls -la $2 |cut -d " " -f5) echo "$SIZE bytes." } ARCH="amd64" SUITE="unstable" echo "$(date -u) - starting to write $PAGE page. Downloading Sources and Packages files from our repository." write_page_header $VIEW "Comparison between the reproducible builds apt repository and regular Debian suites" write_page "

These source packages (and their binaries packages) are different from unstable in our apt repository on alioth. They are available for testing using these sources.lists entries:

"
write_page "deb http://reproducible.alioth.debian.org/debian/ ./"
write_page "deb-src http://reproducible.alioth.debian.org/debian/ ./"
write_page "

" write_page "

" custom_curl http://reproducible.alioth.debian.org/debian/Sources $SOURCES custom_curl http://reproducible.alioth.debian.org/debian/Packages $PACKAGES SOURCEPKGS=$(grep-dctrl -n -s Package -r -FPackage . $SOURCES | sort -u) echo for PKG in $SOURCEPKGS ; do echo "$(date -u) - Processing $PKG..." if [ "${PKG:0:3}" = "lib" ] ; then PREFIX=${PKG:0:4} else PREFIX=${PKG:0:1} fi VERSIONS=$(grep-dctrl -n -s version -S $PKG $SOURCES|sort -u) CRUFT="" BET="" OBSOLETE_IN_SID=false OBSOLETE_IN_TESTING=false OBSOLETE_IN_EXP=false # # gather versions of a package # for VERSION in ${VERSIONS} ; do if [ "$BET" = "" ] ; then BET=${VERSION} continue elif dpkg --compare-versions "$BET" lt "${VERSION}" ; then BET=${VERSION} fi done SID=$(rmadison -s unstable $PKG | egrep -v '^(debian|new):' | cut -d "|" -f2|xargs echo) for VERSION in ${VERSIONS} ; do if [ "${VERSION}" != "$BET" ] ; then CRUFT="$CRUFT ${VERSION}" fi done TESTING=$(rmadison -s testing $PKG | egrep -v '^(debian|new):' | cut -d "|" -f2|xargs echo) EXPERIMENTAL=$(rmadison -s experimental $PKG | egrep -v '^(debian|new):' | cut -d "|" -f2|xargs echo) # # format output # CSID="" CTEST="" CEXP="" if [ ! -z "$TESTING" ] ; then for i in $TESTING ; do if dpkg --compare-versions "$i" gt "$BET" ; then CTEST="$CTEST$i
" OBSOLETE_IN_TESTING=true OBSOLETE_IN_SID=true OBSOLETE_IN_EXP=true else CTEST="$CTEST$i
" fi done fi if [ ! -z "$EXPERIMENTAL" ] ; then for i in $EXPERIMENTAL ; do if dpkg --compare-versions "$i" gt "$BET" ; then CEXP="$CEXP$i
" OBSOLETE_IN_EXP=true else CEXP="$CEXP$i
" fi done fi for i in $SID ; do if dpkg --compare-versions "$i" gt "$BET" ; then CSID="$CSID$i
" if [ ! -z "$BET" ] ; then CRUFT="$BET $CRUFT" BET="" OBSOLETE_IN_SID=true OBSOLETE_IN_EXP=true fi else CSID="$CSID$i
" fi done CBINARIES="" if [ ! -z "$BET" ] ; then ONLYALL=true for ARCH in all ${ARCHS} ; do i="$(grep-dctrl -n -s Package \( -X -FPackage $PKG --or -X -FSource $PKG \) --and -FVersion $BET --and -FArchitecture $ARCH $PACKAGES|sort -u|xargs -r echo)" if [ "$ARCH" != "all" ] && [ ! -z "$i" ] ; then ONLYALL=false fi echo " $ARCH: $i" done for ARCH in all ${ARCHS} ; do i="$(grep-dctrl -n -s Package \( -X -FPackage $PKG --or -X -FSource $PKG \) --and -FVersion $BET --and -FArchitecture $ARCH $PACKAGES|sort -u|xargs -r echo)" if [ ! -z "$i" ] ; then i="$ARCH: $i" elif [ -z "$i" ] && [ "$ARCH" != "all" ] && ! $ONLYALL ; then i="no binaries for $ARCH" let "BINNMUS_NEEDED+=1" fi CBINARIES="$CBINARIES
$i" done BET="$BET" ROW=$TABLE_TODO else BET=" " ROW=$TABLE_DONE fi if [ ! -z "$CRUFT" ] ; then CRUFT="$(echo $CRUFT|sed 's# #
#g')" fi # # write output # write_row "" write_row " " write_row " " URL="https://bugs.debian.org/cgi-bin/pkgreport.cgi?src=$PKG&users=reproducible-builds@lists.alioth.debian.org&archive=both" for TAG in $USERTAGS ; do URL="$URL&tag=$TAG" done write_row " " write_row " " write_row " " write_row " " write_row " " write_row " " write_row "" echo "$(date -u) - Package $PKG done." echo done cat $TABLE_TODO >> $PAGE write_page "
packagegit repoPTS linkusertagged bugold versions in our repo
(needed for reproducing old builds)
version in our repo
(available binary packages per architecture)
version in 'testing'version in 'unstable'version in 'experimental'
src:$PKG
" GIT="$PKG.git" case $PKG in debbindiff) URL="https://anonscm.debian.org/git/reproducible/diffoscope.git" GIT="diffoscope.git" ;; strip-nondeterminism|diffoscope|disorderfs) URL="https://anonscm.debian.org/git/reproducible/$GIT" ;; *) URL="https://anonscm.debian.org/git/reproducible/$GIT/?h=pu/reproducible_builds" ;; esac custom_curl $URL $TMPFILE if [ "$(grep "'error'>No repositories found" $TMPFILE 2>/dev/null)" ] ; then write_row "no git repository found:
$URL" elif [ "$(grep "'error'>Invalid branch" $TMPFILE 2>/dev/null)" ] ; then URL="https://anonscm.debian.org/git/reproducible/$GIT/?h=merged/reproducible_builds" custom_curl $URL $TMPFILE if [ "$(grep "'error'>Invalid branch" $TMPFILE 2>/dev/null)" ] ; then if ! $OBSOLETE_IN_SID ; then write_row "$GIT
non-standard branch" else write_row "$GIT
non-standard branch (but that is ok, our package ain't used in unstable)" fi else write_row "$GIT" MERGEINFO="" if $OBSOLETE_IN_TESTING ; then MERGEINFO=" and available in testing and unstable" elif $OBSOLETE_IN_SID ; then MERGEINFO=" and available in unstable" elif $OBSOLETE_IN_EXP ; then MERGEINFO=" and available in experimental" fi write_row "
(merged$MERGEINFO)" fi else write_row "$GIT" if [ "$PKG" != "strip-nondeterminism" ] && [ "$PKG" != "diffoscope" ] && [ "$PKG" != "debbindiff" ] && [ "$PKG" != "disorderfs" ] ; then if $OBSOLETE_IN_TESTING && $OBSOLETE_IN_SID && $OBSOLETE_IN_EXP ; then write_row "
(unused?" write_row "
Then the branch should probably renamed.)" elif $OBSOLETE_IN_SID && $OBSOLETE_IN_EXP ; then write_row "
(only used in testing, fixed in sid," write_row "
branch probably either should be renamed to merged/reproducible_builds or a new upload to our repo is needed?)" elif $OBSOLETE_IN_EXP ; then write_row "
(only used in testing and unstable, fixed in experimental)" fi elif [ "$PKG" = "disorderfs" ] ; then write_row "
(only used to modify the build environment in the 2nd build)" elif [ "$PKG" = "debbindiff" ] && $OBSOLETE_IN_SID ; then write_row "
(debbindiff has been renamed to diffoscope)" fi fi if ! $OBSOLETE_IN_SID ; then let "MODIFIED_IN_SID+=1" fi if ! $OBSOLETE_IN_EXP ; then let "MODIFIED_IN_EXP+=1" fi write_row "
PTSbugs$CRUFT$BET $CBINARIES$CTEST$CSID$CEXP

" write_page "

" cat $TABLE_DONE >> $PAGE write_page "
package (obsolete in our repo)git repoPTS linkusertagged bugold versions in our repo
(needed for reproducing old builds)
version in our repo
(available binary packages per architecture)
version in 'testing'version in 'unstable'version in 'experimental'

" write_page_footer publish_page echo "$MODIFIED_IN_SID" > /srv/reproducible-results/modified_in_sid.txt echo "$MODIFIED_IN_EXP" > /srv/reproducible-results/modified_in_exp.txt echo "$BINNMUS_NEEDED" > /srv/reproducible-results/binnmus_needed.txt # cleanup rm $SOURCES $PACKAGES $TMPFILE rm $TABLE_TODO $TABLE_DONE