#!/bin/bash # Copyright 2014 Holger Levsen # released under the GPLv=2 . /srv/jenkins/bin/common-functions.sh common_init "$@" set +x # define db PACKAGES_DB=/var/lib/jenkins/reproducible.db INIT=/var/lib/jenkins/reproducible.init if [ ! -f $PACKAGES_DB ] ; then echo "$PACKAGES_DB doesn't exist, no stats possible." exit 1 fi declare -A GOOD declare -A BAD declare -A UGLY declare -A SOURCELESS declare -A NOTFORUS declare -A STAR declare -A LINKTARGET declare -A SPOKENTARGET LAST24="AND build_date > datetime('now', '-24 hours') " LAST48="AND build_date > datetime('now', '-48 hours') " SUITE=sid AMOUNT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT amount FROM source_stats WHERE suite = \"$SUITE\"" | xargs echo) #ALLSTATES="reproducible FTBR FTBR_with_buildinfo FTBFS 404 not_for_us blacklisted" ALLSTATES="reproducible FTBR FTBFS 404 not_for_us blacklisted" GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY build_date DESC" | xargs echo) GOOD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) GOOD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) GOOD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY name" | xargs echo) COUNT_GOOD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"reproducible\"") BAD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo) BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo) BAD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo) BAD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY name" | xargs echo) COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"unreproducible\"") UGLY["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY build_date DESC" | xargs echo) UGLY["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST24 ORDER BY build_date DESC" | xargs echo) UGLY["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST48 ORDER BY build_date DESC" | xargs echo) UGLY["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY name" | xargs echo) COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"FTBFS\"") SOURCELESS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY build_date DESC" | xargs echo) SOURCELESS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY name" | xargs echo) COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"404\"" | xargs echo) NOTFORUS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY build_date DESC" | xargs echo) NOTFORUS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY name" | xargs echo) COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"not for us\"" | xargs echo) BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"blacklisted\" ORDER BY name" | xargs echo) COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"blacklisted\"" | xargs echo) COUNT_TOTAL=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages") PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc) PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc) PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc) PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc) PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc) SPOKENTARGET["all"]="all tested packages" SPOKENTARGET["last_24h"]="packages tested in the last 24h" SPOKENTARGET["last_48h"]="packages tested in the last 48h" SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" SPOKENTARGET["dd-list"]="maintainers of unreproducible packages" SPOKENTARGET["notes"]="packages with notes" SPOKENTARGET["reproducible"]="packages which built reproducibly" SPOKENTARGET["FTBR"]="packages which failed to build reproducibly" #SPOKENTARGET["FTBR_with_buildinfo"]="packages which failed to build reproducibly and have a .buildinfo file" SPOKENTARGET["FTBFS"]="packages which fail to build from source" SPOKENTARGET["404"]="packages where the sources failed to downloaded" SPOKENTARGET["not_for_us"]="packages which should not be build on 'amd64'" SPOKENTARGET["blacklisted"]="packages which have been blacklisted" # # gather notes # WORKSPACE=$PWD cd /var/lib/jenkins if [ -d notes.git ] ; then cd notes.git git pull else git clone git://git.debian.org/git/reproducible/notes.git notes.git fi cd $WORKSPACE PACKAGES_YML=/var/lib/jenkins/notes.git/packages.yml ISSUES_YML=/var/lib/jenkins/notes.git/issues.yml NOTES_PATH=/var/lib/jenkins/userContent/notes mkdir -p $NOTES_PATH rm -f $NOTES_PATH/*.html declare -A NOTES_PACKAGE declare -A NOTES_VERSION declare -A NOTES_ISSUES declare -A NOTES_BUGS declare -A NOTES_COMMENTS declare -A ISSUES_DESCRIPTION declare -A ISSUES_URL show_multi_values() { TMPFILE=$(mktemp) echo "$@" > $TMPFILE while IFS= read -r p ; do if [ "$p" = "-" ] || [ "$p" = "" ] ; then continue elif [ "${p:0:2}" = "- " ] ; then p="${p:2}" fi echo " $PROPERTY = $p" done < $TMPFILE unset IFS rm $TMPFILE } ISSUES=$(cat ${ISSUES_YML} | /srv/jenkins/bin/shyaml keys) for ISSUE in ${ISSUES} ; do echo " Issue = ${ISSUE}" for PROPERTY in url description ; do VALUE="$(cat ${ISSUES_YML} | /srv/jenkins/bin/shyaml get-value ${ISSUE}.${PROPERTY} )" if [ "$VALUE" != "" ] ; then case $PROPERTY in url) ISSUES_URL[${ISSUE}]=$VALUE echo " $PROPERTY = $VALUE" ;; description) ISSUES_DESCRIPTION[${ISSUE}]=$VALUE show_multi_values "$VALUE" ;; esac fi done done tag_property_loop() { BEFORE=$1 shift AFTER=$1 shift TMPFILE=$(mktemp) echo "$@" > $TMPFILE while IFS= read -r p ; do if [ "$p" = "-" ] || [ "$p" = "" ] ; then continue elif [ "${p:0:2}" = "- " ] ; then p="${p:2}" fi echo "$BEFORE" >> ${NOTE} if $BUG ; then # turn bugs into links p="#$p" else # turn URLs into links p="$(echo $p |sed -e 's|http[s:]*//[^ ]*|\0|g')" fi echo "$p" >> ${NOTE} echo "$AFTER" >> ${NOTE} done < $TMPFILE unset IFS rm $TMPFILE } issues_loop() { TTMPFILE=$(mktemp) echo "$@" > $TTMPFILE FIRST=true while IFS= read -r p ; do if [ "${p:0:2}" = "- " ] ; then p="${p:2}" fi if ! $FIRST ; then echo " " >> ${NOTE} fi FIRST=false if [ "${ISSUES_URL[$p]}" != "" ] ; then echo "$p" >> ${NOTE} else echo "$p" >> ${NOTE} fi tag_property_loop "" "
" "${ISSUES_DESCRIPTION[$p]}" echo "" >> ${NOTE} done < $TTMPFILE unset IFS rm $TTMPFILE } create_pkg_note() { echo "" > ${NOTE} echo "" >> ${NOTE} echo "" >> ${NOTE} echo "" >> ${NOTE} BUG=false if [ "${NOTES_ISSUES[$1]}" != "" ] ; then echo "" >> ${NOTE} issues_loop "${NOTES_ISSUES[$1]}" fi BUG=true if [ "${NOTES_BUGS[$1]}" != "" ] ; then echo "" >> ${NOTE} echo "" >> ${NOTE} fi BUG=false if [ "${NOTES_COMMENTS[$1]}" != "" ] ; then echo "" >> ${NOTE} echo "" >> ${NOTE} fi echo "" >> ${NOTE} echo "" >> ${NOTE} echo "
Version annotated:${NOTES_VERSION[$1]}
Identified issues:
Bugs noted:
 " >> ${NOTE} tag_property_loop "" "
" "${NOTES_BUGS[$1]}" echo "
Comments:
 " >> ${NOTE} tag_property_loop "" "
" "${NOTES_COMMENTS[$1]}" echo "
 
" >> ${NOTE} echo "Notes are stored in notes.git." >> ${NOTE} echo "
" >> ${NOTE} } PACKAGES_WITH_NOTES=$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml keys) for PKG in $PACKAGES_WITH_NOTES ; do echo " Package = ${PKG}" NOTES_PACKAGE[${PKG}]=" notes " for PROPERTY in version issues bugs comments ; do VALUE="$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml get-value ${PKG}.${PROPERTY} )" if [ "$VALUE" != "" ] ; then case $PROPERTY in version) NOTES_VERSION[${PKG}]=$VALUE echo " $PROPERTY = $VALUE" ;; issues) NOTES_ISSUES[${PKG}]=$VALUE show_multi_values "$VALUE" ;; bugs) NOTES_BUGS[${PKG}]=$VALUE show_multi_values "$VALUE" ;; comments) NOTES_COMMENTS[${PKG}]=$VALUE show_multi_values "$VALUE" ;; esac fi done NOTE=$NOTES_PATH/${PKG}_note.html create_pkg_note $PKG done # # end note parsing # write_summary() { echo "$1" >> $SUMMARY } mkdir -p /var/lib/jenkins/userContent/rb-pkg/ write_pkg_frameset() { FRAMESET="/var/lib/jenkins/userContent/rb-pkg/$1.html" cat > $FRAMESET <<-EOF EOF } set_icon() { # icons taken from tango-icon-theme (0.8.90-5) # licenced under http://creativecommons.org/licenses/publicdomain/ STATE_TARGET_NAME=$1 case "$1" in reproducible) ICON=weather-clear.png ;; unreproducible|FTBR) if [ "$2" != "" ] ; then ICON=weather-showers-scattered.png else ICON=weather-showers.png fi STATE_TARGET_NAME=FTBR ;; FTBFS) ICON=weather-storm.png ;; 404) ICON=weather-severe-alert.png ;; not_for_us) ICON=weather-few-clouds-night.png ;; blacklisted) ICON=error.png ;; *) ICON="" esac } init_navi_frame() { echo "" > $NAVI echo "" >> $NAVI echo "
$1 $2" >> $NAVI set_icon $3 $5 echo " $3" >> $NAVI echo "at $4: " >> $NAVI } append2navi_frame() { echo "$1" >> $NAVI } finish_navi_frame() { echo "notes/bugs/stats for reproducible builds
" >> $NAVI } process_packages() { for PKG in $@ ; do RESULT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT build_date,version,status FROM source_packages WHERE name = \"$PKG\"") BUILD_DATE=$(echo $RESULT|cut -d "|" -f1) VERSION=$(echo $RESULT|cut -d "|" -f2) STATUS=$(echo $RESULT|cut -d "|" -f3) # remove epoch EVERSION=$(echo $VERSION | cut -d ":" -f2) if $BUILDINFO_SIGNS && [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then STAR[$PKG]="β" # used to be a star... fi # only build $PKG pages if they don't exist or are older than $BUILD_DATE NAVI="/var/lib/jenkins/userContent/rb-pkg/${PKG}_navigation.html" FILE=$(find $(dirname $NAVI) -name $(basename $NAVI) ! -newermt "$BUILD_DATE" 2>/dev/null || true) # if no navigation exists, or is older than last build_date or if a note exist... if [ ! -f $NAVI ] || [ "$FILE" != "" ] || [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then MAINLINK="" init_navi_frame "$PKG" "$VERSION" "$STATUS" "$BUILD_DATE" "${STAR[$PKG]}" append2navi_frame "${NOTES_PACKAGE[${PKG}]}" if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then append2navi_frame " buildinfo " MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" fi if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then append2navi_frame " debbindiff " MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" fi RBUILD_LOG="rbuild/${PKG}_${EVERSION}.rbuild.log" if [ -f "/var/lib/jenkins/userContent/${RBUILD_LOG}" ] ; then SIZE=$(du -sh "/var/lib/jenkins/userContent/${RBUILD_LOG}" |cut -f1) append2navi_frame " rbuild ($SIZE) " if [ "$MAINLINK" = "" ] ; then MAINLINK="$JENKINS_URL/userContent/${RBUILD_LOG}" fi fi append2navi_frame " PTS " append2navi_frame " BTS " append2navi_frame " sources " append2navi_frame " debian/rules " if [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then MAINLINK="$JENKINS_URL/userContent/notes/${PKG}_note.html" fi finish_navi_frame write_pkg_frameset "$PKG" "$MAINLINK" fi if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then if [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then NOTED="N" else NOTED="" fi LINKTARGET[$PKG]="$PKG${STAR[$PKG]}$NOTED" else LINKTARGET[$PKG]="$PKG" fi done } force_package_targets() { for PKG in $@ ; do LINKTARGET[$PKG]="$PKG${STAR[$PKG]}" done } link_packages() { for PKG in $@ ; do write_summary " ${LINKTARGET[$PKG]} " done } write_summary_header() { rm -f $SUMMARY write_summary "" write_summary "" write_summary "" write_summary "

$2

" if [ "$1" = "$MAINVIEW" ] ; then write_summary "

These pages are updated every three hours. Results are obtained from several build jobs running on jenkins.debian.net. Thanks to Profitbricks for donating the virtual machine it's running on!

" fi write_summary "

$COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!" if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] ; then write_summary " Join #debian-reproducible on OFTC to get support for making sure your packages build reproducibly too!" fi write_summary "

" write_summary "

" write_summary "
" } write_summary_footer() { write_summary "

Static URL for this page. Last modified: $(date). Copyright 2014 Holger Levsen, GPL-2 licensed. About jenkins.debian.net" write_summary "

" } write_summary_beta_sign() { write_summary "

A β sign after a package which is unreproducible indicates that a .buildinfo file was generated." write_summary "This means the basics for building packages reproducibly are covered :-)

" } publish_summary() { cp $SUMMARY /var/lib/jenkins/userContent/ if [ "$VIEW" = "$MAINVIEW" ] ; then cp $SUMMARY /var/lib/jenkins/userContent/reproducible.html fi rm $SUMMARY } echo "Processing $COUNT_TOTAL packages... this will take a while." BUILDINFO_SIGNS=true process_packages ${BAD["all"]} BUILDINFO_SIGNS=false process_packages ${UGLY["all"]} ${GOOD["all"]} ${SOURCELESS["all"]} ${NOTFORUS["all"]} $BLACKLISTED MAINVIEW="all_abc" ALLVIEWS="last_24h last_48h all all_abc" for VIEW in $ALLVIEWS ; do SUMMARY=index_${VIEW}.html echo "Starting to write $SUMMARY page." write_summary_header $VIEW "Statistics for reproducible builds of ${SPOKENTARGET[$VIEW]}" if [ "${VIEW:0:3}" = "all" ] ; then FINISH=":" else SHORTER_SPOKENTARGET=$(echo ${SPOKENTARGET[$VIEW]} | cut -d "(" -f1) FINISH=", from $SHORTER_SPOKENTARGET these were:" fi write_summary "

$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly in total$FINISH " link_packages ${BAD[$VIEW]} write_summary "

" write_summary write_summary "

$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source in total$FINISH " link_packages ${UGLY[$VIEW]} write_summary "

" if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_SOURCELESS -gt 0 ] ; then write_summary "

For $COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages in total sources could not be downloaded: ${SOURCELESS[$VIEW]}

" fi if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_NOTFORUS -gt 0 ] ; then write_summary "

In total there were $COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': ${NOTFORUS[$VIEW]}

" fi if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_BLACKLISTED -gt 0 ] ; then write_summary "

$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: $BLACKLISTED

" fi write_summary "

$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH " link_packages ${GOOD[$VIEW]} write_summary "

" write_summary_beta_sign write_summary_footer publish_summary done VIEW=dd-list SUMMARY=index_${VIEW}.html echo "Starting to write $SUMMARY page." write_summary_header $VIEW "Statistics for reproducible builds of ${SPOKENTARGET[$VIEW]}" TMPFILE=$(mktemp) echo "${BAD["all"]}" | dd-list -i > $TMPFILE write_summary "

"
while IFS= read -r LINE ; do
	if [ "${LINE:0:3}" = "   " ] ; then
		PACKAGE=$(echo "${LINE:3}" | cut -d " " -f1)
		UPLOADERS=$(echo "${LINE:3}" | cut -d " " -f2-)
		if [ "$UPLOADERS" = "$PACKAGE" ] ; then
			UPLOADERS=""
		fi
		write_summary "   $PACKAGE $UPLOADERS"
	else
		LINE="$(echo $LINE | sed 's#&#\&#g ; s#<#\<#g ; s#>#\>#g')"
		write_summary "$LINE"
	fi
done < $TMPFILE
write_summary "

" rm $TMPFILE write_summary_footer publish_summary VIEW=notes SUMMARY=index_${VIEW}.html echo "Starting to write $SUMMARY page." write_summary_header $VIEW "Statistics for reproducible builds of ${SPOKENTARGET[$VIEW]}" write_summary "

Packages which have notes: " for PKG in $PACKAGES_WITH_NOTES ; do NOTES_PACKAGE[${PKG}]="" done force_package_targets $PACKAGES_WITH_NOTES PACKAGES_WITH_NOTES=$(echo $PACKAGES_WITH_NOTES | sed -s "s# #\n#g" | sort | xargs echo) link_packages $PACKAGES_WITH_NOTES write_summary "

" write_summary "

Notes are stored in notes.git." write_summary_footer publish_summary count_packages() { COUNT=${#@} PERCENT=$(echo "scale=1 ; ($COUNT*100/$COUNT_TOTAL)" | bc) } for STATE in $ALLSTATES ; do SUMMARY=index_${STATE}.html echo "Starting to write $SUMMARY page." write_summary_header $STATE "Statistics for reproducible builds of ${SPOKENTARGET[$STATE]}" case "$STATE" in reproducible) PACKAGES=${GOOD["all"]} ;; FTBR) PACKAGES=${BAD["all"]} ;; FTBFS) PACKAGES=${UGLY["all"]} ;; 404) PACKAGES=${SOURCELESS["all"]} ;; not_for_us) PACKAGES=${NOTFORUS["all"]} ;; blacklisted) PACKAGES=${BLACKLISTED} ;; esac count_packages ${PACKAGES} write_summary "

$COUNT ($PERCENT%)" set_icon $STATE # sets ICON and STATE_TARGET_NAME write_summary "" write_summary " ${SPOKENTARGET[$STATE]}:" link_packages ${PACKAGES} write_summary "

" write_summary if [ "${STATE:0:4}" = "FTBR" ] ; then write_summary_beta_sign fi write_summary_footer publish_summary done echo "Enjoy https://jenkins.debian.net/userContent/reproducible.html"