#!/bin/bash
# Copyright 2014 Holger Levsen These pages are updated every three hours. Results are obtained from several build jobs running on jenkins.debian.net. Thanks to Profitbricks for donating the virtual machine it's running on! $COUNT_TOTAL packages attempted to build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!"
if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] ; then
write_summary " Join " >> ${NOTE}
done < $TTMPFILE
unset IFS
rm $TTMPFILE
}
create_pkg_note() {
echo "" > ${NOTE}
echo "" >> ${NOTE}
echo " " >> ${NOTE}
fi
FIRST=false
if [ "${ISSUES_URL[$p]}" != "" ] ; then
echo "$p " >> ${NOTE}
else
echo " $p " >> ${NOTE}
fi
tag_property_loop "" "
" "${ISSUES_DESCRIPTION[$p]}"
echo "" >> ${NOTE}
echo "
" >> ${NOTE}
}
PACKAGES_WITH_NOTES=$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml keys)
for PKG in $PACKAGES_WITH_NOTES ; do
echo " Package = ${PKG}"
NOTES_PACKAGE[${PKG}]=" notes "
for PROPERTY in version issues bugs comments ; do
VALUE="$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml get-value ${PKG}.${PROPERTY} )"
if [ "$VALUE" != "" ] ; then
case $PROPERTY in
version) NOTES_VERSION[${PKG}]=$VALUE
echo " $PROPERTY = $VALUE"
;;
issues) NOTES_ISSUES[${PKG}]=$VALUE
show_multi_values "$VALUE"
;;
bugs) NOTES_BUGS[${PKG}]=$VALUE
show_multi_values "$VALUE"
;;
comments) NOTES_COMMENTS[${PKG}]=$VALUE
show_multi_values "$VALUE"
;;
esac
fi
done
NOTE=$NOTES_PATH/${PKG}_note.html
create_pkg_note $PKG
done
#
# end note parsing
#
write_summary() {
echo "$1" >> $SUMMARY
}
mkdir -p /var/lib/jenkins/userContent/rb-pkg/
write_pkg_frameset() {
FRAMESET="/var/lib/jenkins/userContent/rb-pkg/$1.html"
cat > $FRAMESET <<-EOF
EOF
}
init_navi_frame() {
echo "" > $NAVI
echo "" >> $NAVI
echo " " >> ${NOTE}
BUG=false
if [ "${NOTES_ISSUES[$1]}" != "" ] ; then
echo "Version annotated: ${NOTES_VERSION[$1]} Identified issues: " >> ${NOTE}
issues_loop "${NOTES_ISSUES[$1]}"
fi
BUG=true
if [ "${NOTES_BUGS[$1]}" != "" ] ; then
echo " " >> ${NOTE}
echo "Bugs noted: " >> ${NOTE}
fi
BUG=false
if [ "${NOTES_COMMENTS[$1]}" != "" ] ; then
echo " " >> ${NOTE}
tag_property_loop "" "
" "${NOTES_BUGS[$1]}"
echo " " >> ${NOTE}
echo "Comments: " >> ${NOTE}
fi
echo "" >> ${NOTE}
echo " " >> ${NOTE}
tag_property_loop "" "
" "${NOTES_COMMENTS[$1]}"
echo " " >> ${NOTE}
echo " " >> ${NOTE}
echo "Notes are stored in notes.git." >> ${NOTE}
echo "
" >> $NAVI
}
process_packages() {
for PKG in $@ ; do
RESULT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT build_date,version,status FROM source_packages WHERE name = \"$PKG\"")
BUILD_DATE=$(echo $RESULT|cut -d "|" -f1)
VERSION=$(echo $RESULT|cut -d "|" -f2)
STATUS=$(echo $RESULT|cut -d "|" -f3)
# remove epoch
EVERSION=$(echo $VERSION | cut -d ":" -f2)
if $BUILDINFO_SIGNS && [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
STAR[$PKG]="β" # used to be a star...
fi
# only build $PKG pages if they don't exist or are older than $BUILD_DATE
NAVI="/var/lib/jenkins/userContent/rb-pkg/${PKG}_navigation.html"
FILE=$(find $(dirname $NAVI) -name $(basename $NAVI) ! -newermt "$BUILD_DATE" 2>/dev/null || true)
# if no navigation exists, or is older than last build_date or if a note exist...
if [ ! -f $NAVI ] || [ "$FILE" != "" ] || [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then
MAINLINK=""
init_navi_frame "$PKG" "$VERSION" "$STATUS" "$BUILD_DATE" "${STAR[$PKG]}"
append2navi_frame "${NOTES_PACKAGE[${PKG}]}"
if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
append2navi_frame " buildinfo "
MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo"
fi
if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then
append2navi_frame " debbindiff "
MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html"
fi
RBUILD_LOG="rbuild/${PKG}_${EVERSION}.rbuild.log"
if [ -f "/var/lib/jenkins/userContent/${RBUILD_LOG}" ] ; then
SIZE=$(du -sh "/var/lib/jenkins/userContent/${RBUILD_LOG}" |cut -f1)
append2navi_frame " rbuild ($SIZE) "
if [ "$MAINLINK" = "" ] ; then
MAINLINK="$JENKINS_URL/userContent/${RBUILD_LOG}"
fi
fi
append2navi_frame " PTS "
append2navi_frame " BTS "
append2navi_frame " sources "
append2navi_frame " debian/rules "
if [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then
MAINLINK="$JENKINS_URL/userContent/notes/${PKG}_note.html"
fi
finish_navi_frame
write_pkg_frameset "$PKG" "$MAINLINK"
fi
if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then
if [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then
NOTED="N"
else
NOTED=""
fi
LINKTARGET[$PKG]="$PKG${STAR[$PKG]}$NOTED"
else
LINKTARGET[$PKG]="$PKG"
fi
done
}
force_package_targets() {
for PKG in $@ ; do
LINKTARGET[$PKG]="$PKG${STAR[$PKG]}"
done
}
link_packages() {
for PKG in $@ ; do
write_summary " ${LINKTARGET[$PKG]} "
done
}
write_summary_header() {
rm -f $SUMMARY
write_summary ""
write_summary ""
write_summary ""
write_summary "$1 $2" >> $NAVI
# icons taken from tango-icon-theme (0.8.90-5)
# licenced under http://creativecommons.org/licenses/publicdomain/
case "$3" in
"reproducible") ICON=weather-clear.png
;;
"unreproducible") if [ "$5" != "" ] ; then
ICON=weather-showers-scattered.png
else
ICON=weather-showers.png
fi
;;
"FTBFS") ICON=weather-storm.png
;;
"404") ICON=weather-severe-alert.png
;;
"not for us") ICON=weather-few-clouds-night.png
;;
"blacklisted") ICON=error.png
;;
esac
echo " $3" >> $NAVI
echo "at $4: " >> $NAVI
}
append2navi_frame() {
echo "$1" >> $NAVI
}
finish_navi_frame() {
echo "
notes/bugs/stats for reproducible builds $2
"
if [ "$1" = "$MAINVIEW" ] ; then
write_summary "#debian-reproducible
on OFTC to get support for making sure your packages build reproducibly too!"
fi
write_summary "Other views for these build results:"
for TARGET in notes $ALLVIEWS dd-list ; do
if [ "$TARGET" = "$1" ] ; then
continue
fi
write_summary "
Static URL for this page. Last modified: $(date). Copyright 2014 Holger Levsen, GPL-2 licensed. About jenkins.debian.net" write_summary "
" } publish_summary() { cp $SUMMARY /var/lib/jenkins/userContent/ if [ "$VIEW" = "$MAINVIEW" ] ; then cp $SUMMARY /var/lib/jenkins/userContent/reproducible.html fi rm $SUMMARY } echo "Processing $COUNT_TOTAL packages... this will take a while." BUILDINFO_SIGNS=true process_packages ${BAD["all"]} BUILDINFO_SIGNS=false process_packages ${UGLY["all"]} ${GOOD["all"]} ${SOURCELESS["all"]} ${NOTFORUS["all"]} $BLACKLISTED MAINVIEW="all_abc" ALLVIEWS="last_24h last_48h all all_abc" for VIEW in $ALLVIEWS ; do SUMMARY=index_${VIEW}.html echo "Starting to write $SUMMARY page." write_summary_header $VIEW "Statistics for reproducible builds of ${SPOKENTARGET[$VIEW]}" if [ "${VIEW:0:3}" = "all" ] ; then FINISH=":" else SHORTER_SPOKENTARGET=$(echo ${SPOKENTARGET[$VIEW]} | cut -d "(" -f1) FINISH=", from $SHORTER_SPOKENTARGET these were:" fi write_summary "$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly in total$FINISH "
link_packages ${BAD[$VIEW]}
write_summary "
$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source in total$FINISH "
link_packages ${UGLY[$VIEW]}
write_summary "
For $COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages in total sources could not be downloaded: ${SOURCELESS[$VIEW]}
In total there were $COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': ${NOTFORUS[$VIEW]}
$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: $BLACKLISTED
$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH "
link_packages ${GOOD[$VIEW]}
write_summary "
A β sign after a package which is unreproducible indicates that a .buildinfo file was generated." write_summary "This means the basics for building packages reproducibly are covered :-)
" write_summary_footer publish_summary done VIEW=dd-list SUMMARY=index_${VIEW}.html echo "Starting to write $SUMMARY page." write_summary_header $VIEW "Statistics for reproducible builds of ${SPOKENTARGET[$VIEW]}" TMPFILE=$(mktemp) echo "${BAD["all"]}" | dd-list -i > $TMPFILE write_summary "" while IFS= read -r LINE ; do if [ "${LINE:0:3}" = " " ] ; then PACKAGE=$(echo "${LINE:3}" | cut -d " " -f1) UPLOADERS=$(echo "${LINE:3}" | cut -d " " -f2-) if [ "$UPLOADERS" = "$PACKAGE" ] ; then UPLOADERS="" fi write_summary " $PACKAGE $UPLOADERS" else LINE="$(echo $LINE | sed 's#\&#g ; s#<#\<#g ; s#>#\>#g')" write_summary "$LINE" fi done < $TMPFILE write_summary "" rm $TMPFILE write_summary_footer publish_summary VIEW=notes SUMMARY=index_${VIEW}.html echo "Starting to write $SUMMARY page." write_summary_header $VIEW "Statistics for reproducible builds of ${SPOKENTARGET[$VIEW]}" write_summary "
Packages which have notes: "
for PKG in $PACKAGES_WITH_NOTES ; do
NOTES_PACKAGE[${PKG}]=""
done
force_package_targets $PACKAGES_WITH_NOTES
PACKAGES_WITH_NOTES=$(echo $PACKAGES_WITH_NOTES | sed -s "s# #\n#g" | sort | xargs echo)
link_packages $PACKAGES_WITH_NOTES
write_summary "
Notes are stored in notes.git." write_summary_footer publish_summary echo "Enjoy https://jenkins.debian.net/userContent/reproducible.html"