#!/bin/bash
# Copyright 2014 Holger Levsen These pages are updated every six hours. Results are obtained from several jobs running on jenkins.debian.net. Thanks to Profitbricks for donating the virtual machine it's running on! $COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!"
if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] ; then
write_page " Join There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014 Holger Levsen, GPL-2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library. An underlined package is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue."
if $BETA_SIGN ; then
write_page "A β sign after a package which is unreproducible indicates that a .buildinfo file was generated."
write_page "And that means the basics for building packages reproducibly are covered."
fi
write_page " $COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly in total$FINISH $COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source in total$FINISH For $COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages in total sources could not be downloaded: In total there were $COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64': $COUNT_BLACKLISTED packages are blacklisted and will never be tested here: $COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH Packages which have notes: Broken .yaml files in notes.git could not be parsed, please investigate and fix! Notes are stored in notes.git. Broken .yaml files in notes.git could not be parsed, please investigate and fix! Notes are stored in notes.git. $COUNT ($PERCENT%)"
set_icon $STATE $WITH # sets ICON and STATE_TARGET_NAME
write_page " The following maintainers and uploaders are listed for packages which have built unreproducibly:
"
done < $TTMPFILE
unset IFS
rm $TTMPFILE
}
create_pkg_note() {
BUG=false
rm -f $PAGE
# write_page_header() is not used as it contains the "
if [ "${ISSUES_URL[$p]}" != "" ] ; then
write_page "Identifier: $p "
fi
if [ "${ISSUES_DESCRIPTION[$p]}" != "" ] ; then
write_page "URL ${ISSUES_URL[$p]} "
fi
write_page "Description "
tag_property_loop "" "
" "${ISSUES_DESCRIPTION[$p]}"
write_page " tag...
write_page ""
write_page ""
write_page ""
write_page "
"
write_page "
"
write_page_footer
}
create_issue() {
BUG=false
write_page_header "" "Notes about issue '$1'"
write_page " "
if [ "${NOTES_ISSUES[$1]}" != "" ] ; then
write_page "Version annotated: ${NOTES_VERSION[$1]} "
fi
BUG=true
if [ "${NOTES_BUGS[$1]}" != "" ] ; then
write_page "Identified issues: "
issues_loop "${NOTES_ISSUES[$1]}"
write_page " "
fi
BUG=false
if [ "${NOTES_COMMENTS[$1]}" != "" ] ; then
write_page "Bugs noted: "
write_page ""
tag_property_loop "" "
" "${NOTES_BUGS[$1]}"
write_page " "
fi
write_page "Comments: "
write_page ""
tag_property_loop "" "
" "${NOTES_COMMENTS[$1]}"
write_page " "
write_page " "
write_page "Notes are stored in notes.git."
write_page " "
write_page "
"
write_page_meta_sign
write_page_footer
}
write_issues() {
touch $ISSUES_PATH/stamp
for ISSUE in ${ISSUES} ; do
PAGE=$ISSUES_PATH/${ISSUE}_issue.html
create_issue $ISSUE
done
cd $ISSUES_PATH
for FILE in *.html ; do
# if issue is older than stamp file...
if [ $FILE -ot stamp ] ; then
rm $FILE
fi
done
rm stamp
cd - > /dev/null
}
parse_issues() {
ISSUES=$(cat ${ISSUES_YML} | /srv/jenkins/bin/shyaml keys)
for ISSUE in ${ISSUES} ; do
echo " Issue = ${ISSUE}"
for PROPERTY in url description ; do
VALUE="$(cat ${ISSUES_YML} | /srv/jenkins/bin/shyaml get-value ${ISSUE}.${PROPERTY} )"
if [ "$VALUE" != "" ] ; then
case $PROPERTY in
url) ISSUES_URL[${ISSUE}]=$VALUE
echo " $PROPERTY = $VALUE"
;;
description) ISSUES_DESCRIPTION[${ISSUE}]=$VALUE
show_multi_values "$VALUE"
;;
esac
fi
done
done
}
write_notes() {
touch $NOTES_PATH/stamp
for PKG in $PACKAGES_WITH_NOTES ; do
PAGE=$NOTES_PATH/${PKG}_note.html
create_pkg_note $PKG
done
cd $NOTES_PATH
for FILE in *.html ; do
PKG_FILE=/var/lib/jenkins/userContent/rb-pkg/${FILE:0:-10}.html
# if note was removed...
if [ $FILE -ot stamp ] ; then
# cleanup old notes
rm $FILE
# force re-creation of package file if there was a note
rm ${PKG_FILE}
else
# ... else re-recreate ${PKG_FILE} if it does not contain a link to the note
grep _note.html ${PKG_FILE} > /dev/null || rm ${PKG_FILE}
fi
done
rm stamp
cd - > /dev/null
}
parse_notes() {
PACKAGES_WITH_NOTES=$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml keys)
for PKG in $PACKAGES_WITH_NOTES ; do
echo " Package = ${PKG}"
NOTES_PACKAGE[${PKG}]=" notes "
for PROPERTY in version issues bugs comments ; do
VALUE="$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml get-value ${PKG}.${PROPERTY} )"
if [ "$VALUE" != "" ] ; then
case $PROPERTY in
version) NOTES_VERSION[${PKG}]=$VALUE
echo " $PROPERTY = $VALUE"
;;
issues) NOTES_ISSUES[${PKG}]=$VALUE
show_multi_values "$VALUE"
;;
bugs) NOTES_BUGS[${PKG}]=$VALUE
show_multi_values "$VALUE"
;;
comments) NOTES_COMMENTS[${PKG}]=$VALUE
show_multi_values "$VALUE"
;;
esac
fi
done
done
}
validate_yaml() {
VALID_YAML=true
set +e
cat $1 | /srv/jenkins/bin/shyaml keys > /dev/null 2>&1 || VALID_YAML=false
cat $1 | /srv/jenkins/bin/shyaml get-values > /dev/null 2>&1 || VALID_YAML=false
set -e
echo "$1 is valid yaml: $VALID_YAML"
}
#
# end note parsing functions...
#
mkdir -p /var/lib/jenkins/userContent/rb-pkg/
write_page() {
echo "$1" >> $PAGE
}
set_icon() {
# icons taken from tango-icon-theme (0.8.90-5)
# licenced under http://creativecommons.org/licenses/publicdomain/
STATE_TARGET_NAME="$1"
case "$1" in
reproducible) ICON=weather-clear.png
;;
unreproducible|FTBR*) if [ "$2" != "" ] ; then
ICON=weather-showers-scattered.png
STATE_TARGET_NAME=FTBR_with_buildinfo
else
ICON=weather-showers.png
STATE_TARGET_NAME=FTBR
fi
;;
FTBFS) ICON=weather-storm.png
;;
404) ICON=weather-severe-alert.png
;;
not_for_us|"not for us") ICON=weather-few-clouds-night.png
STATE_TARGET_NAME="not_for_us"
;;
blacklisted) ICON=error.png
;;
*) ICON=""
esac
}
init_pkg_page() {
echo "" > ${PKG_FILE}
echo "" >> ${PKG_FILE}
echo " "
if [ "${ISSUES_URL[$1]}" != "" ] ; then
write_page "Identifier: $1 "
fi
if [ "${ISSUES_DESCRIPTION[$1]}" != "" ] ; then
write_page "URL: ${ISSUES_URL[$1]} "
fi
write_page "Description: "
write_page ""
tag_property_loop "" "
" "${ISSUES_DESCRIPTION[$1]}"
write_page " "
write_page "Packages known to be affected by this issue: "
BETA_SIGN=false
for PKG in $PACKAGES_WITH_NOTES ; do
if [ "${NOTES_ISSUES[$PKG]}" != "" ] ; then
TTMPFILE=$(mktemp)
echo "${NOTES_ISSUES[$PKG]}" > $TTMPFILE
while IFS= read -r p ; do
if [ "${p:0:2}" = "- " ] ; then
p="${p:2}"
fi
if [ "$p" = "$1" ] ; then
write_page " ${LINKTARGET[$PKG]} "
if ! $BETA_SIGN && [ "${STAR[$PKG]}" != "" ] ; then
BETA_SIGN=true
fi
fi
done < $TTMPFILE
unset IFS
rm $TTMPFILE
fi
done
write_page " "
write_page " "
write_page "Notes are stored in notes.git."
write_page "
" >> ${PKG_FILE}
echo "" >> ${PKG_FILE}
echo "" >> ${PKG_FILE}
}
set_package_class() {
if [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then
CLASS="class=\"noted\""
else
CLASS="class=\"package\""
fi
}
process_packages() {
for PKG in $@ ; do
RESULT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT build_date,version,status FROM source_packages WHERE name = \"$PKG\"")
BUILD_DATE=$(echo $RESULT|cut -d "|" -f1)
# version with epoch removed
EVERSION=$(echo $RESULT | cut -d "|" -f2 | cut -d ":" -f2)
if $BUILDINFO_SIGNS && [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
STAR[$PKG]="β" # used to be a star...
fi
# only build $PKG pages if they don't exist or are older than $BUILD_DATE or have a note
PKG_FILE="/var/lib/jenkins/userContent/rb-pkg/${PKG}.html"
OLD_FILE=$(find $(dirname ${PKG_FILE}) -name $(basename ${PKG_FILE}) ! -newermt "$BUILD_DATE" 2>/dev/null || true)
# if no package file exists, or is older than last build_date
if [ ! -f ${PKG_FILE} ] || [ "$OLD_FILE" != "" ] ; then
VERSION=$(echo $RESULT | cut -d "|" -f2)
STATUS=$(echo $RESULT | cut -d "|" -f3)
MAINLINK=""
init_pkg_page "$PKG" "$VERSION" "$STATUS" "$BUILD_DATE" "${STAR[$PKG]}"
append2pkg_page "${NOTES_PACKAGE[${PKG}]}"
if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
append2pkg_page " buildinfo "
MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo"
fi
if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then
append2pkg_page " debbindiff "
MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html"
fi
RBUILD_LOG="rbuild/${PKG}_${EVERSION}.rbuild.log"
if [ -f "/var/lib/jenkins/userContent/${RBUILD_LOG}" ] ; then
SIZE=$(du -sh "/var/lib/jenkins/userContent/${RBUILD_LOG}" |cut -f1)
append2pkg_page " rbuild ($SIZE) "
if [ "$MAINLINK" = "" ] ; then
MAINLINK="$JENKINS_URL/userContent/${RBUILD_LOG}"
fi
fi
append2pkg_page " PTS "
append2pkg_page " BTS "
append2pkg_page " sources "
append2pkg_page " debian/rules "
if [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then
MAINLINK="$JENKINS_URL/userContent/notes/${PKG}_note.html"
fi
finish_pkg_page "$MAINLINK"
fi
if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then
set_package_class
LINKTARGET[$PKG]="$PKG${STAR[$PKG]}"
else
LINKTARGET[$PKG]="$PKG"
fi
done
}
force_package_targets() {
for PKG in $@ ; do
set_package_class
LINKTARGET[$PKG]="$PKG${STAR[$PKG]}"
done
}
link_packages() {
for PKG in $@ ; do
write_page " ${LINKTARGET[$PKG]} "
if ! $BETA_SIGN && [ "${STAR[$PKG]}" != "" ] ; then
BETA_SIGN=true
fi
done
}
write_page_header() {
rm -f $PAGE
write_page ""
write_page ""
write_page ""
write_page "$1 $2" >> ${PKG_FILE}
set_icon "$3" $5
echo " " >> ${PKG_FILE}
echo "at $4: " >> ${PKG_FILE}
}
append2pkg_page() {
echo "$1" >> ${PKG_FILE}
}
finish_pkg_page() {
echo "
reproducible builds $2
"
if [ "$1" = "$MAINVIEW" ] ; then
write_page "#debian-reproducible
on OFTC to get support for making sure your packages build reproducibly too!"
fi
write_page "
"
write_page ""
link_packages ${BAD[$VIEW]}
write_page "
"
link_packages ${UGLY[$VIEW]}
write_page "
${SOURCELESS[$VIEW]}
${NOTFORUS[$VIEW]}
$BLACKLISTED
"
link_packages ${GOOD[$VIEW]}
write_page "
"
force_package_targets ${PACKAGES_WITH_NOTES}
PACKAGES_WITH_NOTES=$(echo $PACKAGES_WITH_NOTES | sed -s "s# #\n#g" | sort | xargs echo)
link_packages $PACKAGES_WITH_NOTES
write_page "
"
ISSUES=$(echo ${ISSUES} | sed -s "s# #\n#g" | sort | xargs echo)
for ISSUE in ${ISSUES} ; do
write_page "
"
else
write_page " "
done
write_page "${ISSUE} "
write_page " ${SPOKENTARGET[$STATE]}:
"
link_packages ${PACKAGES}
write_page "
"
while IFS= read -r LINE ; do
if [ "${LINE:0:3}" = " " ] ; then
PACKAGE=$(echo "${LINE:3}" | cut -d " " -f1)
UPLOADERS=$(echo "${LINE:3}" | cut -d " " -f2-)
if [ "$UPLOADERS" = "$PACKAGE" ] ; then
UPLOADERS=""
fi
write_page " $PACKAGE $UPLOADERS"
else
LINE="$(echo $LINE | sed 's#\&#g ; s#<#\<#g ; s#>#\>#g')"
write_page "$LINE"
fi
done < $TMPFILE
write_page "
$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly.
" write_page "$COUNT_BAD packages ($PERCENT_BAD%) failed to built reproducibly.
" write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source.
" if [ $COUNT_SOURCELESS -gt 0 ] ; then write_page "For $COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages sources could not be downloaded," fi write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64' will never be build here" write_page "and those $COUNT_BLACKLISTED blacklisted packages neither.
" write_page ""
# FIXME: we don't do stats_builds_age.png yet :/
for i in 0 1 ; do
# redo pngs once a day
if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime -1 -name ${TABLE[$i]}.png) ] ; then
redo_png
fi
write_page " "
done
write_page "