From 8a0a4485edff9d21fffb02f763c65afd14c97769 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Fri, 11 Dec 2015 13:53:55 +0100 Subject: reproducible archlinux: refactor to support multiple repositories --- bin/reproducible_build_archlinux_pkg.sh | 36 ++++++++-------- bin/reproducible_html_archlinux.sh | 73 +++++++++++++++++---------------- 2 files changed, 57 insertions(+), 52 deletions(-) diff --git a/bin/reproducible_build_archlinux_pkg.sh b/bin/reproducible_build_archlinux_pkg.sh index 64c49f4f..17cddcd9 100755 --- a/bin/reproducible_build_archlinux_pkg.sh +++ b/bin/reproducible_build_archlinux_pkg.sh @@ -44,23 +44,24 @@ choose_package() { # every 2 days we check for new archlinux packages touch -d "$(date -d '2 days ago' '+%Y-%m-%d') 00:00 UTC" $DUMMY if [ ! -f $ARCHLINUX_PKGS ] || [ $DUMMY -nt $ARCHLINUX_PKGS ] ; then - echo "$(date -u ) - updating list of available packages." + REPOSITORY="core" + echo "$(date -u ) - updating list of available packages in repository '$REPOSITORY'." local SESSION="archlinux-scheduler-$RANDOM" schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux - schroot --run-session -c $SESSION --directory /var/abs/core -- ls -1|sort -R|xargs echo > $ARCHLINUX_PKGS + schroot --run-session -c $SESSION --directory /var/abs/$REPOSITORY -- ls -1|sort -R|xargs echo > $ARCHLINUX_PKGS schroot --end-session -c $SESSION fi rm $DUMMY > /dev/null - echo "$(date -u ) - these packages are known to us:" + echo "$(date -u ) - these packages in repository '$REPOSITORY' are known to us:" cat $ARCHLINUX_PKGS for PKG in $(cat $ARCHLINUX_PKGS) ; do # build package if it has never build or at least a week ago - if [ ! -d $BASE/archlinux/$PKG ] || [ ! -z $(find $BASE/archlinux/ -name $PKG -mtime +6) ] ; then + if [ ! -d $BASE/archlinux/$REPOSITORY/$PKG ] || [ ! -z $(find $BASE/archlinux/$REPOSITORY/ -name $PKG -mtime +6) ] ; then SRCPACKAGE=$PKG - echo "$(date -u ) - building package $PKG now..." + echo "$(date -u ) - building package $PKG from '$REPOSITORY' now..." # very simple locking… - mkdir -p $BASE/archlinux/$PKG - touch $BASE/archlinux/$PKG + mkdir -p $BASE/archlinux/$REPOSITORY/$PKG + touch $BASE/archlinux/$REPOSITORY/$PKG break fi done @@ -77,7 +78,7 @@ choose_package() { first_build() { echo "=============================================================================" - echo "Building ${SRCPACKAGE} for Arch Linux on $(hostname -f) now." + echo "Building ${SRCPACKAGE} from repository '$REPOSITORY' for Arch Linux on $(hostname -f) now." echo "Date: $(date)" echo "Date UTC: $(date -u)" echo "=============================================================================" @@ -88,7 +89,7 @@ first_build() { schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux echo "MAKEFLAGS=-j$NUM_CPU" | schroot --run-session -c $SESSION --directory /tmp -u root -- tee -a /etc/makepkg.conf schroot --run-session -c $SESSION --directory /tmp -- mkdir $BUILDDIR - schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/core/$SRCPACKAGE $BUILDDIR/ + schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/$REPOSITORY/$SRCPACKAGE $BUILDDIR/ # just set timezone in the 1st build echo 'export TZ="/usr/share/zoneinfo/Etc/GMT+12"' | schroot --run-session -c $SESSION --directory /tmp -- tee -a /var/lib/jenkins/.bashrc # nicely run makepkg with a timeout of 4h @@ -104,7 +105,7 @@ first_build() { second_build() { echo "=============================================================================" - echo "Re-Building ${SRCPACKAGE} for Arch Linux on $(hostname -f) now." + echo "Re-Building ${SRCPACKAGE} from repository '$REPOSITORY' for Arch Linux on $(hostname -f) now." echo "Date: $(date)" echo "Date UTC: $(date -u)" echo "=============================================================================" @@ -116,7 +117,7 @@ second_build() { schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux echo "MAKEFLAGS=-j$NEW_NUM_CPU" | schroot --run-session -c $SESSION --directory /tmp -u root -- tee -a /etc/makepkg.conf schroot --run-session -c $SESSION --directory /tmp -- mkdir $BUILDDIR - schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/core/$SRCPACKAGE $BUILDDIR/ + schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/$REPOSITORY/$SRCPACKAGE $BUILDDIR/ # add more variations in the 2nd build: TZ, LANG, LC_ALL, umask schroot --run-session -c $SESSION --directory /tmp -- tee -a /var/lib/jenkins/.bashrc <<-__END__ export TZ="/usr/share/zoneinfo/Etc/GMT-14" @@ -213,13 +214,14 @@ fi # main - only used in master-mode # delay_start # randomize start times -# first, we need to choose a package… +# first, we need to choose a package from a repository… +REPOSITORY="" SRCPACKAGE="" choose_package # build package twice mkdir b1 b2 remote_build 1 -# only do the 2nd build if the 1st produced some results +# only do the 2nd build if the 1st produced results if [ ! -z "$(ls $TMPDIR/b1/$SRCPACKAGE/*.pkg.tar.xz 2>/dev/null|| true)" ] ; then remote_build 2 # run diffoscope on the results @@ -232,15 +234,15 @@ if [ ! -z "$(ls $TMPDIR/b1/$SRCPACKAGE/*.pkg.tar.xz 2>/dev/null|| true)" ] ; the call_diffoscope $SRCPACKAGE $ARTIFACT # publish page if [ -f $TMPDIR/$SRCPACKAGE/$ARTIFACT.html ] ; then - cp $TMPDIR/$SRCPACKAGE/$ARTIFACT.html $BASE/archlinux/$SRCPACKAGE/ + cp $TMPDIR/$SRCPACKAGE/$ARTIFACT.html $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/ fi done fi # publish logs cd $TMPDIR/b1/$SRCPACKAGE -cp build1.log $BASE/archlinux/$SRCPACKAGE/ -[ ! -f $TMPDIR/b2/$SRCPACKAGE/build2.log ] || cp $TMPDIR/b2/$SRCPACKAGE/build2.log $BASE/archlinux/$SRCPACKAGE/ -echo "$(date -u) - $REPRODUCIBLE_URL/archlinux/$SRCPACKAGE/ updated." +cp build1.log $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/ +[ ! -f $TMPDIR/b2/$SRCPACKAGE/build2.log ] || cp $TMPDIR/b2/$SRCPACKAGE/build2.log $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/ +echo "$(date -u) - $REPRODUCIBLE_URL/archlinux/$REPOSITORY/$SRCPACKAGE/ updated." cd cleanup_all diff --git a/bin/reproducible_html_archlinux.sh b/bin/reproducible_html_archlinux.sh index aa948f58..0e92e0e9 100755 --- a/bin/reproducible_html_archlinux.sh +++ b/bin/reproducible_html_archlinux.sh @@ -37,45 +37,48 @@ cat > $PAGE <<- EOF EOF write_page_intro 'Arch Linux' write_explaination_table 'Arch Linux' -write_page " " -for PKG in $(find $ARCHBASE/* -maxdepth 1 -type d -exec basename {} \;) ; do - write_page " " - write_page " " - write_page "
source packagetest resulttest date1st build log2nd build log
$PKG" - if [ -z "$(cd $ARCHBASE/$PKG/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then - if [ ! -z "$(grep '==> ERROR: Could not resolve all dependencies' $ARCHBASE/$PKG/build1.log)" ] ; then - write_page " \"depwait could not resolve dependencies" - elif [ ! -z "$(egrep '==> ERROR: .pacman. failed to install missing dependencies.' $ARCHBASE/$PKG/build1.log)" ] ; then - write_page " \"depwait failed to install dependencies" - elif [ ! -z "$(egrep '==> ERROR: A failure occurred in (build|package)' $ARCHBASE/$PKG/build1.log)" ] ; then - write_page " \"ftbfs failed to build from source" - elif [ ! -z "$(egrep '==> ERROR: A failure occurred in check' $ARCHBASE/$PKG/build1.log)" ] ; then - write_page " \"ftbfs failed to build from source, while running tests" - elif [ ! -z "$(egrep '==> ERROR: Failure while downloading' $ARCHBASE/$PKG/build1.log)" ] ; then - write_page " \"404 failed to download source" - elif [ ! -z "$(egrep '==> ERROR: One or more files did not pass the validity check' $ARCHBASE/$PKG/build1.log)" ] ; then - write_page " \"ftbfs failed to verify source" - elif [ ! -z "$(egrep 'makepkg was killed by timeout after 4h' $ARCHBASE/$PKG/build1.log)" ] ; then - write_page " \"ftbfs failed to build, killed by timeout after 4h" +write_page " " +for REPOSITORY in "core" ; do + for PKG in $(find $ARCHBASE/$REPOSITORY/* -maxdepth 1 -type d -exec basename {} \;) ; do + write_page " " + write_page " " + write_page " " + write_page " " + write_page " " + for LOG in build1.log build2.log ; do + if [ -f $ARCHBASE/$REPOSITORY/$PKG/$LOG ] ; then + write_page " " + else + write_page " " + fi done - fi - write_page " " - write_page " " - for LOG in build1.log build2.log ; do - if [ -f $ARCHBASE/$PKG/$LOG ] ; then - write_page " " - else - write_page " " - fi + write_page " " done - write_page " " done write_page "
repositorysource packagetest resulttest date1st build log2nd build log
$REPOSITORY$PKG" + if [ -z "$(cd $ARCHBASE/$REPOSITORY/$PKG/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then + if [ ! -z "$(grep '==> ERROR: Could not resolve all dependencies' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then + write_page " \"depwait could not resolve dependencies" + elif [ ! -z "$(egrep '==> ERROR: .pacman. failed to install missing dependencies.' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then + write_page " \"depwait failed to install dependencies" + elif [ ! -z "$(egrep '==> ERROR: A failure occurred in (build|package)' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then + write_page " \"ftbfs failed to build from source" + elif [ ! -z "$(egrep '==> ERROR: A failure occurred in check' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then + write_page " \"ftbfs failed to build from source, while running tests" + elif [ ! -z "$(egrep '==> ERROR: Failure while downloading' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then + write_page " \"404 failed to download source" + elif [ ! -z "$(egrep '==> ERROR: One or more files did not pass the validity check' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then + write_page " \"ftbfs failed to verify source" + elif [ ! -z "$(egrep 'makepkg was killed by timeout after 4h' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then + write_page " \"ftbfs failed to build, killed by timeout after 4h" + else + write_page " probably failed to build from source, please investigate" + # or is it reproducible??? + fi else - write_page " probably failed to build from source, please investigate" - # or is it reproducible??? + for ARTIFACT in $(cd $ARCHBASE/$REPOSITORY/$PKG/ ; ls *.pkg.tar.xz.html) ; do + write_page " \"unreproducible ${ARTIFACT:0:-5} is unreproducible
" + done fi - else - for ARTIFACT in $(cd $ARCHBASE/$PKG/ ; ls *.pkg.tar.xz.html) ; do - write_page " \"unreproducible ${ARTIFACT:0:-5} is unreproducible
" + write_page " 		$(LANG=C TZ=UTC ls --full-time $ARCHBASE/$REPOSITORY/$PKG/build1.log | cut -d ' ' -f6 )$LOG $(LANG=C TZ=UTC ls --full-time $ARCHBASE/$PKG/build1.log | cut -d ' ' -f6 )$LOG 
" write_page "" -- cgit v1.2.3-54-g00ecf