#!/bin/bash # Copyright 2015-2017 Holger Levsen # released under the GPLv=2 DEBUG=false . /srv/jenkins/bin/common-functions.sh common_init "$@" # common code defining db access . /srv/jenkins/bin/reproducible_common.sh VIEW=repositories PAGE=index_${VIEW}.html SOURCES=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) PACKAGES=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) TMPFILE=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) TABLE_TODO=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) TABLE_DONE=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) MODIFIED_IN_SID=0 MODIFIED_IN_EXP=0 BINNMUS_NEEDED=0 write_row() { echo "$1" >> $ROW } custom_curl() { echo -n "$(date -u) - downloading $1 to $2 - " curl -s $1 > $2 local SIZE=$(ls -la $2 |cut -d " " -f5) echo "$SIZE bytes." } ARCH="amd64" SUITE="unstable" echo "$(date -u) - starting to write $PAGE page. Downloading Sources and Packages files from our repository." write_page_header $VIEW "Comparison between the reproducible builds apt repository and regular Debian suites" write_page "

These source packages (and their binaries packages) are different from unstable in our apt repository on alioth. They are available for testing using these sources.lists entries:

"
write_page "deb http://reproducible.alioth.debian.org/debian/ ./"
write_page "deb-src http://reproducible.alioth.debian.org/debian/ ./"
write_page "

" custom_curl http://reproducible.alioth.debian.org/debian/Sources $SOURCES custom_curl http://reproducible.alioth.debian.org/debian/Packages $PACKAGES SOURCEPKGS=$(grep-dctrl -n -s Package -r -FPackage . $SOURCES | sort -u) echo for PKG in $SOURCEPKGS ; do echo "$(date -u) - Processing $PKG..." if [ "${PKG:0:3}" = "lib" ] ; then PREFIX=${PKG:0:4} else PREFIX=${PKG:0:1} fi VERSIONS=$(grep-dctrl -n -s version -S $PKG $SOURCES|sort -u) CRUFT="" BET="" OBSOLETE_IN_SID=false OBSOLETE_IN_STRETCH=false OBSOLETE_IN_EXP=false # # gather versions of a package # for VERSION in ${VERSIONS} ; do if [ "$BET" = "" ] ; then BET=${VERSION} continue elif dpkg --compare-versions "$BET" lt "${VERSION}" ; then BET=${VERSION} fi done SID=$(rmadison -s unstable $PKG | egrep -v '^(debian|new):' | cut -d "|" -f2|xargs echo) for VERSION in ${VERSIONS} ; do if [ "${VERSION}" != "$BET" ] ; then CRUFT="$CRUFT ${VERSION}" fi done STRETCH=$(rmadison -s stretch $PKG | egrep -v '^(debian|new):' | cut -d "|" -f2|xargs echo) EXPERIMENTAL=$(rmadison -s experimental $PKG | egrep -v '^(debian|new):' | cut -d "|" -f2|xargs echo) # # format output # CSID="" CSTRETCH="" CEXP="" if [ ! -z "$STRETCH" ] ; then for i in $STRETCH ; do if dpkg --compare-versions "$i" gt "$BET" ; then CSTRETCH="$CSTRETCH$i
" OBSOLETE_IN_STRETCH=true OBSOLETE_IN_SID=true OBSOLETE_IN_EXP=true else CSTRETCH="$CSTRETCH$i
" fi done fi if [ ! -z "$EXPERIMENTAL" ] ; then for i in $EXPERIMENTAL ; do if dpkg --compare-versions "$i" gt "$BET" ; then CEXP="$CEXP$i
" OBSOLETE_IN_EXP=true else CEXP="$CEXP$i
" fi done fi for i in $SID ; do if dpkg --compare-versions "$i" gt "$BET" ; then CSID="$CSID$i
" if [ ! -z "$BET" ] ; then CRUFT="$BET $CRUFT" BET="" OBSOLETE_IN_SID=true OBSOLETE_IN_EXP=true fi else CSID="$CSID$i
" fi done CBINARIES="" if [ ! -z "$BET" ] ; then ONLYALL=true for ARCH in all ${ARCHS} ; do i="$(grep-dctrl -n -s Package \( -X -FPackage $PKG --or -X -FSource $PKG \) --and -FVersion $BET --and -FArchitecture $ARCH $PACKAGES|sort -u|xargs -r echo)" if [ "$ARCH" != "all" ] && [ ! -z "$i" ] ; then ONLYALL=false fi echo " $ARCH: $i" done for ARCH in all ${ARCHS} ; do i="$(grep-dctrl -n -s Package \( -X -FPackage $PKG --or -X -FSource $PKG \) --and -FVersion $BET --and -FArchitecture $ARCH $PACKAGES|sort -u|xargs -r echo)" if [ ! -z "$i" ] ; then i="$ARCH: $i" elif [ -z "$i" ] && [ "$ARCH" != "all" ] && ! $ONLYALL ; then i="no binaries for $ARCH" let "BINNMUS_NEEDED+=1" fi CBINARIES="$CBINARIES
$i" done BET="$BET" ROW=$TABLE_TODO else BET=" " ROW=$TABLE_DONE fi if [ ! -z "$CRUFT" ] ; then CRUFT="$(echo $CRUFT|sed 's# #
#g')" fi # # write output # write_row "
src:$PKG
" write_row " " GIT="$PKG.git" case $PKG in debbindiff) URL="https://anonscm.debian.org/git/reproducible/diffoscope.git" GIT="diffoscope.git" ;; strip-nondeterminism|diffoscope|disorderfs) URL="https://anonscm.debian.org/git/reproducible/$GIT" ;; *) URL="https://anonscm.debian.org/git/reproducible/$GIT/?h=pu/reproducible_builds" ;; esac custom_curl $URL $TMPFILE if [ "$(grep "'error'>No repositories found" $TMPFILE 2>/dev/null)" ] ; then write_row "no git repository found:
$URL" elif [ "$(grep "'error'>Invalid branch" $TMPFILE 2>/dev/null)" ] ; then URL="https://anonscm.debian.org/git/reproducible/$GIT/?h=merged/reproducible_builds" custom_curl $URL $TMPFILE if [ "$(grep "'error'>Invalid branch" $TMPFILE 2>/dev/null)" ] ; then if ! $OBSOLETE_IN_SID ; then write_row "$GIT
non-standard branch" else write_row "$GIT
non-standard branch (but that is ok, our package ain't used in unstable)" fi else write_row "$GIT" MERGEINFO="" if $OBSOLETE_IN_STRETCH ; then MERGEINFO=" and available in stretch and unstable" elif $OBSOLETE_IN_SID ; then MERGEINFO=" and available in unstable" elif $OBSOLETE_IN_EXP ; then MERGEINFO=" and available in experimental" fi write_row "
(merged$MERGEINFO)" fi else write_row "$GIT" if [ "$PKG" != "strip-nondeterminism" ] && [ "$PKG" != "diffoscope" ] && [ "$PKG" != "debbindiff" ] && [ "$PKG" != "disorderfs" ] ; then if $OBSOLETE_IN_STRETCH && $OBSOLETE_IN_SID && $OBSOLETE_IN_EXP ; then write_row "
(unused?" write_row "
Then the branch should probably renamed.)" elif $OBSOLETE_IN_SID && $OBSOLETE_IN_EXP ; then write_row "
(only used in stretch, fixed in sid," write_row "
branch probably either should be renamed to merged/reproducible_builds or a new upload to our repo is needed?)" elif $OBSOLETE_IN_EXP ; then write_row "
(only used in stretch and unstable, fixed in experimental)" fi elif [ "$PKG" = "disorderfs" ] ; then write_row "
(only used to modify the build environment in the 2nd build)" elif [ "$PKG" = "debbindiff" ] && $OBSOLETE_IN_SID ; then write_row "
(debbindiff has been renamed to diffoscope)" fi fi if ! $OBSOLETE_IN_SID ; then let "MODIFIED_IN_SID+=1" fi if ! $OBSOLETE_IN_EXP ; then let "MODIFIED_IN_EXP+=1" fi write_row " " write_row " PTS" URL="https://bugs.debian.org/cgi-bin/pkgreport.cgi?src=$PKG&users=reproducible-builds@lists.alioth.debian.org&archive=both" for TAG in $USERTAGS ; do URL="$URL&tag=$TAG" done write_row " bugs" write_row " $CRUFT" write_row " $BET $CBINARIES" write_row " $CSTRETCH" write_row " $CSID" write_row " $CEXP" write_row "" echo "$(date -u) - Package $PKG done." echo done if [ -s $TABLE_TODO ] ; then write_page "

" cat $TABLE_TODO >> $PAGE write_page "
packagegit repoPTS linkusertagged bug(s)old versions in our repo
(needed for reproducing old builds)
version in our repo
(available binary packages per architecture)
version in 'stretch'version in 'unstable'version in 'experimental'

" else write_page "

Congratulations! There are no modified packages in our repository compared to unstable. (Yes, that means our repository is obsolete now.)" fi if [ -s $TABLE_DONE ] ; then write_page "

" cat $TABLE_DONE >> $PAGE write_page "
obsoleted package,
version in sid higher than in our repo
git repoPTS linkusertagged bug(s)old version(s) in our repo
(needed for reproducing old builds)
version in our repo
(available binary packages per architecture)
version in 'stretch'version in 'unstable'version in 'experimental'

" fi write_page_footer publish_page debian echo "$MODIFIED_IN_SID" > /srv/reproducible-results/modified_in_sid.txt echo "$MODIFIED_IN_EXP" > /srv/reproducible-results/modified_in_exp.txt echo "$BINNMUS_NEEDED" > /srv/reproducible-results/binnmus_needed.txt # cleanup rm $SOURCES $PACKAGES $TMPFILE rm $TABLE_TODO $TABLE_DONE