#!/bin/bash
# Copyright 2014 Holger Levsen These pages are updated every six hours. Results are obtained from several jobs running on jenkins.debian.net. Thanks to Profitbricks for donating the virtual machine it's running on! $COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!"
if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] ; then
write_summary " Join There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014 Holger Levsen, GPL-2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library. A β sign after a package which is unreproducible indicates that a .buildinfo file was generated."
write_summary "This means the basics for building packages reproducibly are covered :-) $COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly in total$FINISH $COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source in total$FINISH For $COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages in total sources could not be downloaded: In total there were $COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-any' nor 'linux-amd64': $COUNT_BLACKLISTED packages are blacklisted and will never be tested here: $COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH The following maintainers and uploaders are listed for packages which have built unreproducibly: " >> ${NOTE}
done < $TTMPFILE
unset IFS
rm $TTMPFILE
}
create_pkg_note() {
echo "" > ${NOTE}
echo "" >> ${NOTE}
echo " " >> ${NOTE}
fi
FIRST=false
if [ "${ISSUES_URL[$p]}" != "" ] ; then
echo "$p " >> ${NOTE}
else
echo " $p " >> ${NOTE}
fi
tag_property_loop "" "
" "${ISSUES_DESCRIPTION[$p]}"
echo "" >> ${NOTE}
echo "
" >> ${NOTE}
}
parse_notes() {
touch $NOTES_PATH/stamp
PACKAGES_WITH_NOTES=$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml keys)
for PKG in $PACKAGES_WITH_NOTES ; do
echo " Package = ${PKG}"
NOTES_PACKAGE[${PKG}]=" notes "
for PROPERTY in version issues bugs comments ; do
VALUE="$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml get-value ${PKG}.${PROPERTY} )"
if [ "$VALUE" != "" ] ; then
case $PROPERTY in
version) NOTES_VERSION[${PKG}]=$VALUE
echo " $PROPERTY = $VALUE"
;;
issues) NOTES_ISSUES[${PKG}]=$VALUE
show_multi_values "$VALUE"
;;
bugs) NOTES_BUGS[${PKG}]=$VALUE
show_multi_values "$VALUE"
;;
comments) NOTES_COMMENTS[${PKG}]=$VALUE
show_multi_values "$VALUE"
;;
esac
fi
done
NOTE=$NOTES_PATH/${PKG}_note.html
create_pkg_note $PKG
done
cd $NOTES_PATH
for FILE in *.html ; do
PKG_FILE=/var/lib/jenkins/userContent/rb-pkg/${FILE:0:-10}.html
# if note was removed...
if [ $FILE -ot stamp ] ; then
# cleanup old notes
rm $FILE
# force re-creation of package file if there was a note
rm ${PKG_FILE}
else
# ... else re-recreate ${PKG_FILE} if it does not contain a link to the note
grep _note.html ${PKG_FILE} > /dev/null || rm ${PKG_FILE}
fi
done
rm stamp
cd - > /dev/null
}
validate_yaml() {
VALID_YAML=true
set +e
cat $1 | /srv/jenkins/bin/shyaml keys > /dev/null 2>&1 || VALID_YAML=false
cat $1 | /srv/jenkins/bin/shyaml get-values > /dev/null 2>&1 || VALID_YAML=false
set -e
echo "$1 is valid yaml: $VALID_YAML"
}
#
# actually parse the notes
#
validate_yaml ${ISSUES_YML}
validate_yaml ${PACKAGES_YML}
if $VALID_YAML ; then
parse_issues
parse_notes
else
echo "Warning: ${ISSUES_YML} or ${PACKAGES_YML} contains invalid yaml, please fix."
fi
#
# end note parsing
#
mkdir -p /var/lib/jenkins/userContent/rb-pkg/
write_summary() {
echo "$1" >> $SUMMARY
}
set_icon() {
# icons taken from tango-icon-theme (0.8.90-5)
# licenced under http://creativecommons.org/licenses/publicdomain/
STATE_TARGET_NAME="$1"
case "$1" in
reproducible) ICON=weather-clear.png
;;
unreproducible|FTBR*) if [ "$2" != "" ] ; then
ICON=weather-showers-scattered.png
STATE_TARGET_NAME=FTBR_with_buildinfo
else
ICON=weather-showers.png
STATE_TARGET_NAME=FTBR
fi
;;
FTBFS) ICON=weather-storm.png
;;
404) ICON=weather-severe-alert.png
;;
not_for_us|"not for us") ICON=weather-few-clouds-night.png
STATE_TARGET_NAME="not_for_us"
;;
blacklisted) ICON=error.png
;;
*) ICON=""
esac
}
init_pkg_page() {
echo "" > ${PKG_FILE}
echo "" >> ${PKG_FILE}
echo " " >> ${NOTE}
BUG=false
if [ "${NOTES_ISSUES[$1]}" != "" ] ; then
echo "Version annotated: ${NOTES_VERSION[$1]} Identified issues: " >> ${NOTE}
issues_loop "${NOTES_ISSUES[$1]}"
fi
BUG=true
if [ "${NOTES_BUGS[$1]}" != "" ] ; then
echo " " >> ${NOTE}
echo "Bugs noted: " >> ${NOTE}
fi
BUG=false
if [ "${NOTES_COMMENTS[$1]}" != "" ] ; then
echo " " >> ${NOTE}
tag_property_loop "" "
" "${NOTES_BUGS[$1]}"
echo " " >> ${NOTE}
echo "Comments: " >> ${NOTE}
fi
echo "" >> ${NOTE}
echo " " >> ${NOTE}
tag_property_loop "" "
" "${NOTES_COMMENTS[$1]}"
echo " " >> ${NOTE}
echo " " >> ${NOTE}
echo "Notes are stored in notes.git." >> ${NOTE}
echo "
" >> ${PKG_FILE}
echo "" >> ${PKG_FILE}
echo "" >> ${PKG_FILE}
}
set_package_class() {
if [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then
CLASS="class=\"noted\""
else
CLASS="class=\"package\""
fi
}
process_packages() {
for PKG in $@ ; do
RESULT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT build_date,version,status FROM source_packages WHERE name = \"$PKG\"")
BUILD_DATE=$(echo $RESULT|cut -d "|" -f1)
# version with epoch removed
EVERSION=$(echo $RESULT | cut -d "|" -f2 | cut -d ":" -f2)
if $BUILDINFO_SIGNS && [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
STAR[$PKG]="β" # used to be a star...
fi
# only build $PKG pages if they don't exist or are older than $BUILD_DATE or have a note
PKG_FILE="/var/lib/jenkins/userContent/rb-pkg/${PKG}.html"
OLD_FILE=$(find $(dirname ${PKG_FILE}) -name $(basename ${PKG_FILE}) ! -newermt "$BUILD_DATE" 2>/dev/null || true)
# if no package file exists, or is older than last build_date
if [ ! -f ${PKG_FILE} ] || [ "$OLD_FILE" != "" ] ; then
VERSION=$(echo $RESULT | cut -d "|" -f2)
STATUS=$(echo $RESULT | cut -d "|" -f3)
MAINLINK=""
init_pkg_page "$PKG" "$VERSION" "$STATUS" "$BUILD_DATE" "${STAR[$PKG]}"
append2pkg_page "${NOTES_PACKAGE[${PKG}]}"
if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
append2pkg_page " buildinfo "
MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo"
fi
if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then
append2pkg_page " debbindiff "
MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html"
fi
RBUILD_LOG="rbuild/${PKG}_${EVERSION}.rbuild.log"
if [ -f "/var/lib/jenkins/userContent/${RBUILD_LOG}" ] ; then
SIZE=$(du -sh "/var/lib/jenkins/userContent/${RBUILD_LOG}" |cut -f1)
append2pkg_page " rbuild ($SIZE) "
if [ "$MAINLINK" = "" ] ; then
MAINLINK="$JENKINS_URL/userContent/${RBUILD_LOG}"
fi
fi
append2pkg_page " PTS "
append2pkg_page " BTS "
append2pkg_page " sources "
append2pkg_page " debian/rules "
if [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then
MAINLINK="$JENKINS_URL/userContent/notes/${PKG}_note.html"
fi
finish_pkg_page "$MAINLINK"
fi
if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then
set_package_class
LINKTARGET[$PKG]="$PKG${STAR[$PKG]}"
else
LINKTARGET[$PKG]="$PKG"
fi
done
}
force_package_targets() {
for PKG in $@ ; do
set_package_class
LINKTARGET[$PKG]="$PKG${STAR[$PKG]}"
done
}
link_packages() {
for PKG in $@ ; do
write_summary " ${LINKTARGET[$PKG]} "
done
}
write_summary_header() {
rm -f $SUMMARY
write_summary ""
write_summary ""
write_summary ""
write_summary "$1 $2" >> ${PKG_FILE}
set_icon "$3" $5
echo "" >> ${PKG_FILE}
echo "at $4: " >> ${PKG_FILE}
}
append2pkg_page() {
echo "$1" >> ${PKG_FILE}
}
finish_pkg_page() {
echo " reproducible builds $2
"
if [ "$1" = "$MAINVIEW" ] ; then
write_summary "#debian-reproducible
on OFTC to get support for making sure your packages build reproducibly too!"
fi
write_summary "
"
write_summary ""
link_packages ${BAD[$VIEW]}
write_summary "
"
link_packages ${UGLY[$VIEW]}
write_summary "
${SOURCELESS[$VIEW]}
${NOTFORUS[$VIEW]}
$BLACKLISTED
"
link_packages ${GOOD[$VIEW]}
write_summary "
"
while IFS= read -r LINE ; do
if [ "${LINE:0:3}" = " " ] ; then
PACKAGE=$(echo "${LINE:3}" | cut -d " " -f1)
UPLOADERS=$(echo "${LINE:3}" | cut -d " " -f2-)
if [ "$UPLOADERS" = "$PACKAGE" ] ; then
UPLOADERS=""
fi
write_summary " $PACKAGE $UPLOADERS"
else
LINE="$(echo $LINE | sed 's#\&#g ; s#<#\<#g ; s#>#\>#g')"
write_summary "$LINE"
fi
done < $TMPFILE
write_summary "
Packages which have notes: "
force_package_targets $PACKAGES_WITH_NOTES
PACKAGES_WITH_NOTES=$(echo $PACKAGES_WITH_NOTES | sed -s "s# #\n#g" | sort | xargs echo)
link_packages $PACKAGES_WITH_NOTES
write_summary "
Broken .yaml files in notes.git could not be parsed, please investigate and fix!
" fi write_summary "Notes are stored in notes.git.
" write_summary_footer publish_summary count_packages() { COUNT=${#@} PERCENT=$(echo "scale=1 ; ($COUNT*100/$COUNT_TOTAL)" | bc) } for STATE in $ALLSTATES ; do SUMMARY=index_${STATE}.html echo "Starting to write $SUMMARY page." write_summary_header $STATE "Overview of ${SPOKENTARGET[$STATE]}" WITH="" case "$STATE" in reproducible) PACKAGES=${GOOD["all"]} ;; FTBR) CANDIDATES=${BAD["all"]} PACKAGES="" for PKG in $CANDIDATES ; do if [ "${STAR[$PKG]}" = "" ] ; then PACKAGES="$PACKAGES $PKG" fi done ;; FTBR_with_buildinfo) CANDIDATES=${BAD["all"]} PACKAGES="" for PKG in $CANDIDATES ; do if [ "${STAR[$PKG]}" != "" ] ; then PACKAGES="$PACKAGES $PKG" fi done WITH="YES" ;; FTBFS) PACKAGES=${UGLY["all"]} ;; 404) PACKAGES=${SOURCELESS["all"]} ;; not_for_us) PACKAGES=${NOTFORUS["all"]} ;; blacklisted) PACKAGES=${BLACKLISTED} ;; esac count_packages ${PACKAGES} write_summary " $COUNT ($PERCENT%)"
set_icon $STATE $WITH # sets ICON and STATE_TARGET_NAME
write_summary ""
write_summary " ${SPOKENTARGET[$STATE]}:"
link_packages ${PACKAGES}
write_summary "