From 4e1a2de485ea48cb3d9b2090101d37e203a775da Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Sat, 12 Dec 2015 12:06:32 +0100 Subject: reproducible archlinux: include repository stats in html page --- bin/reproducible_build_archlinux_pkg.sh | 1 - bin/reproducible_common.sh | 1 + bin/reproducible_html_archlinux.sh | 43 +++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/bin/reproducible_build_archlinux_pkg.sh b/bin/reproducible_build_archlinux_pkg.sh index 80b0897c..8dc5a5d3 100755 --- a/bin/reproducible_build_archlinux_pkg.sh +++ b/bin/reproducible_build_archlinux_pkg.sh @@ -218,7 +218,6 @@ TIMEOUT=8 # maximum time in hours for a single build DATE=$(date -u +'%Y-%m-%d %H:%M') START=$(date +'%s') BUILDER="${JOB_NAME#reproducible_builder_}/${BUILD_ID}" -ARCHLINUX_PKGS=/srv/reproducible-results/.archlinux_pkgs DUMMY=$(mktemp -t archlinux-dummy-XXXXXXXX) # diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 43349a3b..d4953bc2 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -55,6 +55,7 @@ USERTAGS="toolchain infrastructure timestamps fileordering buildpath username ho # common settings for testing Archlinux ARCHLINUX_BUILD_NODE=profitbricks-build3-amd64 ARCHLINUX_REPOS="core extra" +ARCHLINUX_PKGS=/srv/reproducible-results/.archlinux_pkgs # common settings for testing rpm based distros RPM_BUILD_NODE=profitbricks-build3-amd64 diff --git a/bin/reproducible_html_archlinux.sh b/bin/reproducible_html_archlinux.sh index a333de32..11220893 100755 --- a/bin/reproducible_html_archlinux.sh +++ b/bin/reproducible_html_archlinux.sh @@ -14,6 +14,7 @@ ARCHBASE=$BASE/archlinux # # analyse results to create the webpage # +echo "$(date -u) - starting to analyse build results." HTML_FTBFS=$(mktemp) HTML_FTBR=$(mktemp) HTML_DEPWAIT=$(mktemp) @@ -22,13 +23,24 @@ HTML_GOOD=$(mktemp) HTML_UNKNOWN=$(mktemp) HTML_BUFFER=$(mktemp) HTML_TARGET="" +HTML_REPOSTATS=$(mktemp) for REPOSITORY in $ARCHLINUX_REPOS ; do + echo "$(date -u) - starting to analyse build results for '$REPOSITORY'." + TOTAL=$(cat ${ARCHLINUX_PKGS}_$REPOSITORY | sed -s "s# #\n#g" | wc -l) + TESTED=0 + NR_FTBFS=0 + NR_FTBR=0 + NR_DEPWAIT=0 + NR_404=0 + NR_GOOD=0 + NR_UNKNOWN=0 for PKG in $(find $ARCHBASE/$REPOSITORY/* -maxdepth 1 -type d -exec basename {} \;) ; do if [ -z "$(cd $ARCHBASE/$REPOSITORY/$PKG/ ; ls)" ] ; then # directory exists but is empty: package is building… echo "$(date -u ) - ignoring $PKG from '$REPOSITORY' which is building right now…" continue fi + let TESTED+=1 echo " " >> $HTML_BUFFER echo " $REPOSITORY" >> $HTML_BUFFER echo " $PKG" >> $HTML_BUFFER @@ -36,32 +48,41 @@ for REPOSITORY in $ARCHLINUX_REPOS ; do if [ -z "$(cd $ARCHBASE/$REPOSITORY/$PKG/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then if [ ! -z "$(grep '==> ERROR: Could not resolve all dependencies' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then HTML_TARGET=$HTML_DEPWAIT + let NR_DEPWAIT+=1 echo " \"depwait could not resolve dependencies" >> $HTML_BUFFER elif [ ! -z "$(egrep '==> ERROR: .pacman. failed to install missing dependencies.' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then HTML_TARGET=$HTML_DEPWAIT + let NR_DEPWAIT+=1 echo " \"depwait failed to install dependencies" >> $HTML_BUFFER elif [ ! -z "$(egrep '==> ERROR: A failure occurred in (build|package)' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then HTML_TARGET=$HTML_FTBFS + let NR_FTBFS+=1 echo " \"ftbfs failed to build from source" >> $HTML_BUFFER elif [ ! -z "$(egrep '==> ERROR: A failure occurred in check' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then HTML_TARGET=$HTML_FTBFS + let NR_FTBFS+=1 echo " \"ftbfs failed to build from source, while running tests" >> $HTML_BUFFER elif [ ! -z "$(egrep '==> ERROR: Failure while downloading' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then HTML_TARGET=$HTML_404 + let NR_404+=1 echo " \"404 failed to download source" >> $HTML_BUFFER elif [ ! -z "$(egrep '==> ERROR: One or more files did not pass the validity check' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then HTML_TARGET=$HTML_FTBFS + let NR_FTBFS+=1 echo " \"ftbfs failed to verify source" >> $HTML_BUFFER elif [ ! -z "$(egrep 'makepkg was killed by timeout after' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then HTML_TARGET=$HTML_FTBFS + let NR_FTBFS+=1 echo " \"ftbfs failed to build, killed by timeout" >> $HTML_BUFFER else echo " probably failed to build from source, please investigate" >> $HTML_BUFFER HTML_TARGET=$HTML_UNKNOWN + let NR_UNKNOWN+=1 # or is it reproducible??? fi else HTML_TARGET=$HTML_FTBR + let NR_FTBR+=1 for ARTIFACT in $(cd $ARCHBASE/$REPOSITORY/$PKG/ ; ls *.pkg.tar.xz.html) ; do echo " \"unreproducible ${ARTIFACT:0:-5} is unreproducible
" >> $HTML_BUFFER done @@ -79,6 +100,24 @@ for REPOSITORY in $ARCHLINUX_REPOS ; do cat $HTML_BUFFER >> $HTML_TARGET rm $HTML_BUFFER > /dev/null done + # prepare stats per repository + PERCENT_TOTAL=$(echo "scale=1 ; ($TESTED*100/$TOTAL)" | bc) + if [ $(echo $PERCENT_TOTAL/1|bc) -lt 98 ] ; then + NR_TESTED="$TESTED ($PERCENT_TOTAL% of $TOTAL tested)" + else + NR_TESTED=$TESTED + fi + echo " " >> $HTML_REPOSTATS + echo " $REPOSITORY$NR_TESTED" >> $HTML_REPOSTATS + for i in $NR_GOOD $NR_FTBR $NR_FTBFS $NR_DEPWAIT $NR_404 $NR_UNKNOWN ; do + PERCENT_i=$(echo "scale=1 ; ($i*100/$TOTAL)" | bc) + if [ "$PERCENT_i" != "0" ] ; then + echo " $i ($PERCENT_i%)" >> $HTML_REPOSTATS + else + echo " $i" >> $HTML_REPOSTATS + fi + done + echo " " >> $HTML_REPOSTATS done # # write out the actual webpage @@ -106,6 +145,10 @@ cat > $PAGE <<- EOF EOF write_page_intro 'Arch Linux' write_explaination_table 'Arch Linux' +write_page " " +cat $HTML_REPOSTATS >> $PAGE +rm $HTML_REPOSTATS > /dev/null +write_page "
repositoryall sources packagesreproducible packagesunreproducible packagespackages failing to buildpackages in depwait statepackages 404unknown state
" write_page " " for i in $HTML_UNKNOWN $HTML_FTBFS $HTML_DEPWAIT $HTML_404 $HTML_FTBR $HTML_GOOD ; do cat $i >> $PAGE -- cgit v1.2.3-70-g09d2
repositorysource packagetest resulttest date1st build log2nd build log