#!/bin/bash
# Copyright 2014 Holger Levsen These pages are updated every six hours. Results are obtained from several build jobs running on jenkins.debian.net. Thanks to Profitbricks for donating the virtual machine it's running on! $COUNT_TOTAL packages have been attempted to be build so far, that's $PERCENT_TOTAL% of $AMOUNT source packages in Debian $SUITE currently. Out of these, $PERCENT_GOOD% were successful, so quite wildly guessing this roughy means about $GUESS_GOOD packages should be reproducibly buildable!"
if [ "${1:0:3}" = "all" ] || [ "$1" = "dd-list" ] ; then
write_summary " Join " >> ${NOTE}
done < $TTMPFILE
unset IFS
rm $TTMPFILE
}
create_pkg_note() {
echo "" > ${NOTE}
echo "" >> ${NOTE}
echo " " >> ${NOTE}
fi
FIRST=false
if [ "${ISSUES_URL[$p]}" != "" ] ; then
echo "$p " >> ${NOTE}
else
echo " $p " >> ${NOTE}
fi
tag_property_loop "" "
" "${ISSUES_DESCRIPTION[$p]}"
echo "" >> ${NOTE}
echo "
" >> ${NOTE}
}
PACKAGES_WITH_NOTES=$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml keys)
for PKG in $PACKAGES_WITH_NOTES ; do
echo " Package = ${PKG}"
NOTES_PACKAGE[${PKG}]=" notes "
for PROPERTY in version issues bugs comments ; do
VALUE="$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml get-value ${PKG}.${PROPERTY} )"
if [ "$VALUE" != "" ] ; then
case $PROPERTY in
version) NOTES_VERSION[${PKG}]=$VALUE
echo " $PROPERTY = $VALUE"
;;
issues) NOTES_ISSUES[${PKG}]=$VALUE
show_multi_values "$VALUE"
;;
bugs) NOTES_BUGS[${PKG}]=$VALUE
show_multi_values "$VALUE"
;;
comments) NOTES_COMMENTS[${PKG}]=$VALUE
show_multi_values "$VALUE"
;;
esac
fi
done
NOTE=$NOTES_PATH/${PKG}_note.html
create_pkg_note $PKG
done
#
# end note parsing
#
mkdir -p /var/lib/jenkins/userContent/rb-pkg/
write_summary() {
echo "$1" >> $SUMMARY
}
write_pkg_frameset() {
FRAMESET="/var/lib/jenkins/userContent/rb-pkg/$1.html"
cat > $FRAMESET <<-EOF
" >> ${NOTE}
BUG=false
if [ "${NOTES_ISSUES[$1]}" != "" ] ; then
echo "Version annotated: ${NOTES_VERSION[$1]} Identified issues: " >> ${NOTE}
issues_loop "${NOTES_ISSUES[$1]}"
fi
BUG=true
if [ "${NOTES_BUGS[$1]}" != "" ] ; then
echo " " >> ${NOTE}
echo "Bugs noted: " >> ${NOTE}
fi
BUG=false
if [ "${NOTES_COMMENTS[$1]}" != "" ] ; then
echo " " >> ${NOTE}
tag_property_loop "" "
" "${NOTES_BUGS[$1]}"
echo " " >> ${NOTE}
echo "Comments: " >> ${NOTE}
fi
echo "" >> ${NOTE}
echo " " >> ${NOTE}
tag_property_loop "" "
" "${NOTES_COMMENTS[$1]}"
echo " " >> ${NOTE}
echo " " >> ${NOTE}
echo "Notes are stored in notes.git." >> ${NOTE}
echo "
" >> $NAVI
}
process_packages() {
for PKG in $@ ; do
RESULT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT build_date,version,status FROM source_packages WHERE name = \"$PKG\"")
BUILD_DATE=$(echo $RESULT|cut -d "|" -f1)
VERSION=$(echo $RESULT|cut -d "|" -f2)
STATUS=$(echo $RESULT|cut -d "|" -f3)
# remove epoch
EVERSION=$(echo $VERSION | cut -d ":" -f2)
if $BUILDINFO_SIGNS && [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
STAR[$PKG]="β" # used to be a star...
fi
# only build $PKG pages if they don't exist or are older than $BUILD_DATE
NAVI="/var/lib/jenkins/userContent/rb-pkg/${PKG}_navigation.html"
FILE=$(find $(dirname $NAVI) -name $(basename $NAVI) ! -newermt "$BUILD_DATE" 2>/dev/null || true)
# if no navigation exists, or is older than last build_date or if a note exist...
if [ ! -f $NAVI ] || [ "$FILE" != "" ] || [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then
MAINLINK=""
init_navi_frame "$PKG" "$VERSION" "$STATUS" "$BUILD_DATE" "${STAR[$PKG]}"
append2navi_frame "${NOTES_PACKAGE[${PKG}]}"
if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
append2navi_frame " buildinfo "
MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo"
fi
if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then
append2navi_frame " debbindiff "
MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html"
fi
RBUILD_LOG="rbuild/${PKG}_${EVERSION}.rbuild.log"
if [ -f "/var/lib/jenkins/userContent/${RBUILD_LOG}" ] ; then
SIZE=$(du -sh "/var/lib/jenkins/userContent/${RBUILD_LOG}" |cut -f1)
append2navi_frame " rbuild ($SIZE) "
if [ "$MAINLINK" = "" ] ; then
MAINLINK="$JENKINS_URL/userContent/${RBUILD_LOG}"
fi
fi
append2navi_frame " PTS "
append2navi_frame " BTS "
append2navi_frame " sources "
append2navi_frame " debian/rules "
if [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then
MAINLINK="$JENKINS_URL/userContent/notes/${PKG}_note.html"
fi
finish_navi_frame
write_pkg_frameset "$PKG" "$MAINLINK"
fi
if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then
if [ "${NOTES_PACKAGE[${PKG}]}" != "" ] ; then
NOTED="N"
else
NOTED=""
fi
LINKTARGET[$PKG]="$PKG${STAR[$PKG]}$NOTED"
else
LINKTARGET[$PKG]="$PKG"
fi
done
}
force_package_targets() {
for PKG in $@ ; do
LINKTARGET[$PKG]="$PKG${STAR[$PKG]}"
done
}
link_packages() {
for PKG in $@ ; do
write_summary " ${LINKTARGET[$PKG]} "
done
}
write_summary_header() {
rm -f $SUMMARY
write_summary ""
write_summary ""
write_summary ""
write_summary "$1 $2" >> $NAVI
set_icon "$3" $5
echo "" >> $NAVI
echo "at $4: " >> $NAVI
}
append2navi_frame() {
echo "$1" >> $NAVI
}
finish_navi_frame() {
echo " reproducible builds $2
"
if [ "$1" = "$MAINVIEW" ] ; then
write_summary "#debian-reproducible
on OFTC to get support for making sure your packages build reproducibly too!"
fi
write_summary "Other views for these build results:"
write_summary "
There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $(date +'%Y-%m-%d %H:%M %Z'). Copyright 2014 Holger Levsen, GPL-2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library.
" write_summary "" } write_summary_beta_sign() { write_summary "A β sign after a package which is unreproducible indicates that a .buildinfo file was generated." write_summary "This means the basics for building packages reproducibly are covered :-)
" } publish_summary() { cp $SUMMARY /var/lib/jenkins/userContent/ if [ "$VIEW" = "$MAINVIEW" ] ; then cp $SUMMARY /var/lib/jenkins/userContent/reproducible.html fi rm $SUMMARY } echo "Processing $COUNT_TOTAL packages... this will take a while." BUILDINFO_SIGNS=true process_packages ${BAD["all"]} BUILDINFO_SIGNS=false process_packages ${UGLY["all"]} ${GOOD["all"]} ${SOURCELESS["all"]} ${NOTFORUS["all"]} $BLACKLISTED MAINVIEW="all_abc" ALLVIEWS="last_24h last_48h all_abc" for VIEW in $ALLVIEWS ; do SUMMARY=index_${VIEW}.html echo "Starting to write $SUMMARY page." write_summary_header $VIEW "Overview of reproducible builds of ${SPOKENTARGET[$VIEW]}" if [ "${VIEW:0:3}" = "all" ] ; then FINISH=":" else SHORTER_SPOKENTARGET=$(echo ${SPOKENTARGET[$VIEW]} | cut -d "(" -f1) FINISH=", from $SHORTER_SPOKENTARGET these were:" fi write_summary "$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly in total$FINISH "
link_packages ${BAD[$VIEW]}
write_summary "
$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source in total$FINISH "
link_packages ${UGLY[$VIEW]}
write_summary "
For $COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages in total sources could not be downloaded: ${SOURCELESS[$VIEW]}
In total there were $COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any' nor 'all' nor 'amd64' nor 'linux-amd64': ${NOTFORUS[$VIEW]}
$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: $BLACKLISTED
$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH "
link_packages ${GOOD[$VIEW]}
write_summary "
The following maintainers and uploaders are listed for packages which have built unreproducibly:
" while IFS= read -r LINE ; do if [ "${LINE:0:3}" = " " ] ; then PACKAGE=$(echo "${LINE:3}" | cut -d " " -f1) UPLOADERS=$(echo "${LINE:3}" | cut -d " " -f2-) if [ "$UPLOADERS" = "$PACKAGE" ] ; then UPLOADERS="" fi write_summary " $PACKAGE $UPLOADERS" else LINE="$(echo $LINE | sed 's#\&#g ; s#<#\<#g ; s#>#\>#g')" write_summary "$LINE" fi done < $TMPFILE write_summary "" rm $TMPFILE write_summary_footer publish_summary VIEW=notes SUMMARY=index_${VIEW}.html echo "Starting to write $SUMMARY page." write_summary_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}" write_summary "
Packages which have notes: "
for PKG in $PACKAGES_WITH_NOTES ; do
NOTES_PACKAGE[${PKG}]=""
done
force_package_targets $PACKAGES_WITH_NOTES
PACKAGES_WITH_NOTES=$(echo $PACKAGES_WITH_NOTES | sed -s "s# #\n#g" | sort | xargs echo)
link_packages $PACKAGES_WITH_NOTES
write_summary "
Notes are stored in notes.git.
" write_summary_footer publish_summary count_packages() { COUNT=${#@} PERCENT=$(echo "scale=1 ; ($COUNT*100/$COUNT_TOTAL)" | bc) } for STATE in $ALLSTATES ; do SUMMARY=index_${STATE}.html echo "Starting to write $SUMMARY page." write_summary_header $STATE "Overview of ${SPOKENTARGET[$STATE]}" WITH="" case "$STATE" in reproducible) PACKAGES=${GOOD["all"]} ;; FTBR) CANDIDATES=${BAD["all"]} PACKAGES="" for PKG in $CANDIDATES ; do if [ "${STAR[$PKG]}" = "" ] ; then PACKAGES="$PACKAGES $PKG" fi done ;; FTBR_with_buildinfo) CANDIDATES=${BAD["all"]} PACKAGES="" for PKG in $CANDIDATES ; do if [ "${STAR[$PKG]}" != "" ] ; then PACKAGES="$PACKAGES $PKG" fi done WITH="YES" ;; FTBFS) PACKAGES=${UGLY["all"]} ;; 404) PACKAGES=${SOURCELESS["all"]} ;; not_for_us) PACKAGES=${NOTFORUS["all"]} ;; blacklisted) PACKAGES=${BLACKLISTED} ;; esac count_packages ${PACKAGES} write_summary " $COUNT ($PERCENT%)"
set_icon $STATE $WITH # sets ICON and STATE_TARGET_NAME
write_summary ""
write_summary " ${SPOKENTARGET[$STATE]}:"
link_packages ${PACKAGES}
write_summary "