diff options
| -rwxr-xr-x | bin/reproducible_common.sh | 125 | ||||
| -rwxr-xr-x | bin/reproducible_html_dd_list.sh | 1 | ||||
| -rwxr-xr-x | bin/reproducible_html_notes.sh | 358 | ||||
| -rwxr-xr-x | bin/reproducible_stats.sh | 479 | ||||
| -rw-r--r-- | job-cfg/reproducible.yaml | 8 |
5 files changed, 498 insertions, 473 deletions
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 954aa384..ebd58048 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -100,12 +100,15 @@ fi # shop trailing slash JENKINS_URL=${JENKINS_URL:0:-1} +# we only need them for html creation but we cannot declare them in a function +declare -A SPOKENTARGET +declare -A LINKTARGET + init_html() { SUITE=sid ALLSTATES="reproducible FTBR_with_buildinfo FTBR FTBFS 404 not_for_us blacklisted" MAINVIEW="stats" ALLVIEWS="last_24h last_48h all_abc" - declare -A SPOKENTARGET SPOKENTARGET["last_24h"]="packages tested in the last 24h" SPOKENTARGET["last_48h"]="packages tested in the last 48h" SPOKENTARGET["all_abc"]="all tested packages (sorted alphabetically)" @@ -132,6 +135,7 @@ init_html() { PERCENT_TOTAL=$(echo "scale=1 ; ($COUNT_TOTAL*100/$AMOUNT)" | bc) PERCENT_GOOD=$(echo "scale=1 ; ($COUNT_GOOD*100/$COUNT_TOTAL)" | bc) GUESS_GOOD=$(echo "$PERCENT_GOOD*$AMOUNT/100" | bc) + BUILDINFO_SIGNS=true } write_page() { @@ -173,6 +177,7 @@ write_icon() { write_page_header() { rm -f $PAGE + BUILDINFO_ON_PAGE=false write_page "<!DOCTYPE html><html><head>" write_page "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />" write_page "<link href=\"$JENKINS_URL/userContent/static/style.css\" type=\"text/css\" rel=\"stylesheet\" />" @@ -216,7 +221,7 @@ write_page_footer() { write_page_meta_sign() { write_page "<p style=\"font-size:0.9em;\">An underlined package is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue." - if $BETA_SIGN ; then + if $BUILDINFO_ON_PAGE ; then write_page "A β sign after a package which is unreproducible indicates that a .buildinfo file was generated." write_page "And that means the <a href=\"https://wiki.debian.org/ReproducibleBuilds#The_basics_for_making_packages_build_reproducible\">basics for building packages reproducibly are covered</a>." fi @@ -229,5 +234,121 @@ publish_page() { cp $PAGE /var/lib/jenkins/userContent/reproducible.html fi rm $PAGE + echo "Enjoy $JENKINS_URL/userContent/$PAGE" +} + +set_package_star() { + if [ ! -z "$(find /var/lib/jenkins/userContent/buildinfo/ -name ${PKG}_*_amd64.buildinfo)" ] ; then + STAR="<span class=\"beta\">β</span>" # used to be a star... + else + STAR="" + fi +} + +set_package_class() { + if [ -f ${NOTES_PATH}/${PKG}_note.html ] ; then + CLASS="class=\"noted\"" + else + CLASS="class=\"package\"" + fi +} + +force_package_targets() { + for PKG in $@ ; do + set_package_class + LINKTARGET[$PKG]="<a href=\"$JENKINS_URL/userContent/rb-pkg/$PKG.html\" $CLASS>$PKG</a>" + done +} + +link_packages() { + for PKG in $@ ; do + STAR="" + write_page " ${LINKTARGET[$PKG]}" + if $BUILDINFO_SIGNS ; then + set_package_star + if [ ! -z "$STAR" ] ; then + write_page "$STAR " + fi + fi + done +} + +init_pkg_page() { + echo "<!DOCTYPE html><html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />" > ${PKG_FILE} + echo "<link href=\"../static/style.css\" type=\"text/css\" rel=\"stylesheet\" />" >> ${PKG_FILE} + echo "<title>$1 - reproducible builds results</title></head>" >> ${PKG_FILE} + echo "<body><table class=\"head\"><tr><td><span style=\"font-size:1.2em;\">$1</span> $2" >> ${PKG_FILE} + set_icon "$3" $5 # this sets $STATE_TARGET_NAME and $ICON + echo "<a href=\"$JENKINS_URL/userContent/index_${STATE_TARGET_NAME}.html\" target=\"_parent\"><img src=\"$JENKINS_URL/userContent/static/$ICON\" alt=\"${STATE_TARGET_NAME} icon\" /></a>" >> ${PKG_FILE} + echo "<span style=\"font-size:0.9em;\">at $4:</span> " >> ${PKG_FILE} } +append2pkg_page() { + echo "$1" >> ${PKG_FILE} +} + +finish_pkg_page() { + echo "</td><td style=\"text-align:right; font-size:0.9em;\"><a href=\"$JENKINS_URL/userContent/reproducible.html\" target=\"_parent\">reproducible builds</a></td></tr></table>" >> ${PKG_FILE} + echo "<iframe name=\"main\" src=\"$1\" width=\"100%\" height=\"98%\" frameborder=\"0\">" >> ${PKG_FILE} + echo "<p>Your browser does not support iframes. Use a different one or follow the links above.</p>" >> ${PKG_FILE} + echo "</iframe>" >> ${PKG_FILE} + echo "</body></html>" >> ${PKG_FILE} +} + +process_packages() { + for PKG in $@ ; do + RESULT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT build_date,version,status FROM source_packages WHERE name = \"$PKG\"") + BUILD_DATE=$(echo $RESULT|cut -d "|" -f1) + # version with epoch removed + EVERSION=$(echo $RESULT | cut -d "|" -f2 | cut -d ":" -f2) + set_package_star + # only build $PKG pages if they don't exist or are older than $BUILD_DATE or have a note + PKG_FILE="/var/lib/jenkins/userContent/rb-pkg/${PKG}.html" + OLD_FILE=$(find $(dirname ${PKG_FILE}) -name $(basename ${PKG_FILE}) ! -newermt "$BUILD_DATE" 2>/dev/null || true) + # if no package file exists, or is older than last build_date + if [ ! -f ${PKG_FILE} ] || [ "$OLD_FILE" != "" ] ; then + VERSION=$(echo $RESULT | cut -d "|" -f2) + STATUS=$(echo $RESULT | cut -d "|" -f3) + MAINLINK="" + NOTES_LINK="" + if [ -f ${NOTES_PATH}/${PKG}_note.html ] ; then + NOTES_LINK=" <a href=\"$JENKINS_URL/userContent/notes/${PKG}_note.html\" target=\"main\">notes</a> " + fi + init_pkg_page "$PKG" "$VERSION" "$STATUS" "$BUILD_DATE" "$STAR" + append2pkg_page "${NOTES_LINK}" + if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then + append2pkg_page " <a href=\"$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo\" target=\"main\">buildinfo</a> " + MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" + fi + if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then + append2pkg_page " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html\" target=\"main\">debbindiff</a> " + MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" + fi + RBUILD_LOG="rbuild/${PKG}_${EVERSION}.rbuild.log" + if [ -f "/var/lib/jenkins/userContent/${RBUILD_LOG}" ] ; then + SIZE=$(du -sh "/var/lib/jenkins/userContent/${RBUILD_LOG}" |cut -f1) + append2pkg_page " <a href=\"$JENKINS_URL/userContent/${RBUILD_LOG}\" target=\"main\">rbuild ($SIZE)</a> " + if [ "$MAINLINK" = "" ] ; then + MAINLINK="$JENKINS_URL/userContent/${RBUILD_LOG}" + fi + fi + append2pkg_page " <a href=\"https://packages.qa.debian.org/${PKG}\" target=\"main\">PTS</a> " + append2pkg_page " <a href=\"https://bugs.debian.org/src:${PKG}\" target=\"main\">BTS</a> " + append2pkg_page " <a href=\"https://sources.debian.net/src/${PKG}/\" target=\"main\">sources</a> " + append2pkg_page " <a href=\"https://sources.debian.net/src/${PKG}/${VERSION}/debian/rules\" target=\"main\">debian/rules</a> " + + if [ ! -z "${NOTES_LINK}" ] ; then + MAINLINK="$JENKINS_URL/userContent/notes/${PKG}_note.html" + fi + finish_pkg_page "$MAINLINK" + fi + if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then + set_package_class + LINKTARGET[$PKG]="<a href=\"$JENKINS_URL/userContent/rb-pkg/$PKG.html\" $CLASS>$PKG</a>$STAR" + else + LINKTARGET[$PKG]="$PKG" + fi + done +} + + diff --git a/bin/reproducible_html_dd_list.sh b/bin/reproducible_html_dd_list.sh index 07c6ba5a..60f00902 100755 --- a/bin/reproducible_html_dd_list.sh +++ b/bin/reproducible_html_dd_list.sh @@ -38,4 +38,3 @@ rm $TMPFILE write_page_footer publish_page -echo "Enjoy $JENKINS_URL/userContent/$PAGE" diff --git a/bin/reproducible_html_notes.sh b/bin/reproducible_html_notes.sh new file mode 100755 index 00000000..202dd083 --- /dev/null +++ b/bin/reproducible_html_notes.sh @@ -0,0 +1,358 @@ +#!/bin/bash + +# Copyright 2014 Holger Levsen <holger@layer-acht.org> +# released under the GPLv=2 + +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code defining db access +. /srv/jenkins/bin/reproducible_common.sh + +# FIXME: move to daily cleanup job +# cp db away for backup purposes +cp $PACKAGES_DB /var/lib/jenkins/userContent/reproducible.db + +set +x +init_html + +# +# update git +# + +WORKSPACE=$PWD +cd /var/lib/jenkins +if [ -d notes.git ] ; then + cd notes.git + git pull +else + git clone git://git.debian.org/git/reproducible/notes.git notes.git +fi +cd $WORKSPACE + +# +# declare some variables +# + +PACKAGES_YML=/var/lib/jenkins/notes.git/packages.yml +ISSUES_YML=/var/lib/jenkins/notes.git/issues.yml + +declare -A NOTES_VERSION +declare -A NOTES_ISSUES +declare -A NOTES_BUGS +declare -A NOTES_COMMENTS +declare -A ISSUES_DESCRIPTION +declare -A ISSUES_URL + +# +# declare some functions only used for dealing with notes +# + +show_multi_values() { + TMPFILE=$(mktemp) + echo "$@" > $TMPFILE + while IFS= read -r p ; do + if [ "$p" = "-" ] || [ "$p" = "" ] ; then + continue + elif [ "${p:0:2}" = "- " ] ; then + p="${p:2}" + fi + echo " $PROPERTY = $p" + done < $TMPFILE + unset IFS + rm $TMPFILE +} + +tag_property_loop() { + BEFORE=$1 + shift + AFTER=$1 + shift + TMPFILE=$(mktemp) + echo "$@" > $TMPFILE + while IFS= read -r p ; do + if [ "$p" = "-" ] || [ "$p" = "" ] ; then + continue + elif [ "${p:0:2}" = "- " ] ; then + p="${p:2}" + fi + write_page "$BEFORE" + if $BUG ; then + # turn bugs into links + p="<a href=\"https://bugs.debian.org/$p\">#$p</a>" + else + # turn URLs into links + p="$(echo $p |sed -e 's|http[s:]*//[^ ]*|<a href=\"\0\">\0</a>|g')" + fi + write_page "$p" + write_page "$AFTER" + done < $TMPFILE + unset IFS + rm $TMPFILE +} + +issues_loop() { + TTMPFILE=$(mktemp) + echo "$@" > $TTMPFILE + while IFS= read -r p ; do + if [ "${p:0:2}" = "- " ] ; then + p="${p:2}" + fi + write_page "<table class=\"body\"><tr><td>Identifier:</td><td><a href=\"$JENKINS_URL/userContent/issues/${p}_issue.html\" target=\"_parent\">$p</a></tr>" + if [ "${ISSUES_URL[$p]}" != "" ] ; then + write_page "<tr><td>URL</td><td><a href=\"${ISSUES_URL[$p]}\" target=\"_blank\">${ISSUES_URL[$p]}</a></td></tr>" + fi + if [ "${ISSUES_DESCRIPTION[$p]}" != "" ] ; then + write_page "<tr><td>Description</td><td>" + tag_property_loop "" "<br />" "${ISSUES_DESCRIPTION[$p]}" + write_page "</td></tr>" + fi + write_page "</table>" + done < $TTMPFILE + unset IFS + rm $TTMPFILE +} + +create_pkg_note() { + BUG=false + rm -f $PAGE + # write_page_header() is not used as it contains the <h2> tag... + write_page "<!DOCTYPE html><html><head>" + write_page "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />" + write_page "<link href=\"$JENKINS_URL/userContent/static/style.css\" type=\"text/css\" rel=\"stylesheet\" />" + write_page "<title>Notes for $1</title></head>" + write_page "<body></header>" + write_page "<table class=\"body\">" + + write_page "<tr><td>Version annotated:</td><td colspan=\"2\">${NOTES_VERSION[$1]}</td></tr>" + + if [ "${NOTES_ISSUES[$1]}" != "" ] ; then + write_page "<tr><td colspan=\"2\">Identified issues:</td><td>" + issues_loop "${NOTES_ISSUES[$1]}" + write_page "</td></tr>" + fi + + BUG=true + if [ "${NOTES_BUGS[$1]}" != "" ] ; then + write_page "<tr><td>Bugs noted:</td>" + write_page "<td colspan=\"2\">" + tag_property_loop "" "<br />" "${NOTES_BUGS[$1]}" + write_page "</tr>" + fi + BUG=false + + if [ "${NOTES_COMMENTS[$1]}" != "" ] ; then + write_page "<tr><td>Comments:</td>" + write_page "<td colspan=\"2\">" + tag_property_loop "" "<br />" "${NOTES_COMMENTS[$1]}" + write_page "</tr>" + fi + write_page "<tr><td colspan=\"3\"> </td></tr>" + write_page "<tr><td colspan=\"3\" style=\"text-align:right; font-size:0.9em;\">" + write_page "Notes are stored in <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\">notes.git</a>." + write_page "</td></tr></table>" + write_page_footer +} + +create_issue() { + BUG=false + write_page_header "" "Notes about issue '$1'" + write_page "<table class=\"body\">" + + write_page "<tr><td>Identifier:</td><td colspan=\"2\">$1</td></tr>" + + if [ "${ISSUES_URL[$1]}" != "" ] ; then + write_page "<tr><td>URL:</td><td colspan=\"2\"><a href=\"${ISSUES_URL[$1]}\">${ISSUES_URL[$1]}</a></td></tr>" + fi + if [ "${ISSUES_DESCRIPTION[$1]}" != "" ] ; then + write_page "<tr><td>Description:</td>" + write_page "<td colspan=\"2\">" + tag_property_loop "" "<br />" "${ISSUES_DESCRIPTION[$1]}" + write_page "</td></tr>" + fi + + write_page "<tr><td colspan=\"2\">Packages known to be affected by this issue:</td><td>" + for PKG in $PACKAGES_WITH_NOTES ; do + if [ "${NOTES_ISSUES[$PKG]}" != "" ] ; then + TTMPFILE=$(mktemp) + echo "${NOTES_ISSUES[$PKG]}" > $TTMPFILE + while IFS= read -r p ; do + if [ "${p:0:2}" = "- " ] ; then + p="${p:2}" + fi + if [ "$p" = "$1" ] ; then + write_page " ${LINKTARGET[$PKG]} " + fi + done < $TTMPFILE + unset IFS + rm $TTMPFILE + fi + done + write_page "</td></tr>" + write_page "<tr><td colspan=\"3\"> </td></tr>" + write_page "<tr><td colspan=\"3\" style=\"text-align:right; font-size:0.9em;\">" + write_page "Notes are stored in <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\">notes.git</a>." + write_page "</td></tr></table>" + write_page_meta_sign + write_page_footer +} + +write_issues() { + touch $ISSUES_PATH/stamp + for ISSUE in ${ISSUES} ; do + PAGE=$ISSUES_PATH/${ISSUE}_issue.html + create_issue $ISSUE + done + cd $ISSUES_PATH + for FILE in *.html ; do + # if issue is older than stamp file... + if [ $FILE -ot stamp ] ; then + rm $FILE + fi + done + rm stamp + cd - > /dev/null +} + +parse_issues() { + ISSUES=$(cat ${ISSUES_YML} | /srv/jenkins/bin/shyaml keys) + for ISSUE in ${ISSUES} ; do + echo " Issue = ${ISSUE}" + for PROPERTY in url description ; do + VALUE="$(cat ${ISSUES_YML} | /srv/jenkins/bin/shyaml get-value ${ISSUE}.${PROPERTY} )" + if [ "$VALUE" != "" ] ; then + case $PROPERTY in + url) ISSUES_URL[${ISSUE}]=$VALUE + echo " $PROPERTY = $VALUE" + ;; + description) ISSUES_DESCRIPTION[${ISSUE}]=$VALUE + show_multi_values "$VALUE" + ;; + esac + fi + done + done +} + +write_notes() { + touch $NOTES_PATH/stamp + for PKG in $PACKAGES_WITH_NOTES ; do + PAGE=$NOTES_PATH/${PKG}_note.html + create_pkg_note $PKG + done + cd $NOTES_PATH + for FILE in *.html ; do + PKG_FILE=/var/lib/jenkins/userContent/rb-pkg/${FILE:0:-10}.html + # if note was removed... + if [ $FILE -ot stamp ] ; then + # cleanup old notes + rm $FILE + # force re-creation of package file if there was a note + rm -f ${PKG_FILE} + else + # ... else re-recreate ${PKG_FILE} if it does not contain a link to the note + grep _note.html ${PKG_FILE} > /dev/null 2>&1 || rm -f ${PKG_FILE} + fi + done + rm stamp + cd - > /dev/null +} + +parse_notes() { + PACKAGES_WITH_NOTES=$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml keys) + for PKG in $PACKAGES_WITH_NOTES ; do + echo " Package = ${PKG}" + for PROPERTY in version issues bugs comments ; do + VALUE="$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml get-value ${PKG}.${PROPERTY} )" + if [ "$VALUE" != "" ] ; then + case $PROPERTY in + version) NOTES_VERSION[${PKG}]=$VALUE + echo " $PROPERTY = $VALUE" + ;; + issues) NOTES_ISSUES[${PKG}]=$VALUE + show_multi_values "$VALUE" + ;; + bugs) NOTES_BUGS[${PKG}]=$VALUE + show_multi_values "$VALUE" + ;; + comments) NOTES_COMMENTS[${PKG}]=$VALUE + show_multi_values "$VALUE" + ;; + esac + fi + done + done +} + +validate_yaml() { + VALID_YAML=true + set +e + cat $1 | /srv/jenkins/bin/shyaml keys > /dev/null 2>&1 || VALID_YAML=false + cat $1 | /srv/jenkins/bin/shyaml get-values > /dev/null 2>&1 || VALID_YAML=false + set -e + echo "$1 is valid yaml: $VALID_YAML" +} + +# +# end note parsing functions... +# + +# +# actually validate & parse the notes and then write pages for all notes and issues +# +validate_yaml ${ISSUES_YML} +validate_yaml ${PACKAGES_YML} +if $VALID_YAML ; then + echo "$(date) - processing notes and issues" + parse_issues + parse_notes + echo "$(date) - processing packages with notes" + force_package_targets ${PACKAGES_WITH_NOTES} + write_issues + write_notes + process_packages ${PACKAGES_WITH_NOTES} +else + echo "Warning: ${ISSUES_YML} or ${PACKAGES_YML} contains invalid yaml, please fix." +fi + +# +# write packages with notes page +# +VIEW=notes +PAGE=index_${VIEW}.html +echo "$(date) - starting to write $PAGE page." +write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}" +if $VALID_YAML ; then + write_page "<p>Packages which have notes: <code>" + PACKAGES_WITH_NOTES=$(echo $PACKAGES_WITH_NOTES | sed -s "s# #\n#g" | sort | xargs echo) + link_packages $PACKAGES_WITH_NOTES + write_page "</code></p>" +else + write_page "<p style=\"font-size:1.5em; color: red;\">Broken .yaml files in notes.git could not be parsed, please investigate and fix!</p>" +fi +write_page "<p style=\"font-size:0.9em;\">Notes are stored in <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\">notes.git</a>.</p>" +write_page_meta_sign +write_page_footer +publish_page + +# +# write page with all issues +# +VIEW=issues +PAGE=index_${VIEW}.html +echo "$(date) - starting to write $PAGE page." +write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}" +if $VALID_YAML ; then + write_page "<table class=\"body\">" + ISSUES=$(echo ${ISSUES} | sed -s "s# #\n#g" | sort | xargs echo) + for ISSUE in ${ISSUES} ; do + write_page "<tr><td><a href=\"$JENKINS_URL/userContent/issues/${ISSUE}_issue.html\">${ISSUE}</a></td></tr>" + done + write_page "</table>" +else + write_page "<p style=\"font-size:1.5em; color: red;\">Broken .yaml files in notes.git could not be parsed, please investigate and fix!</p>" +fi +write_page "<p style=\"font-size:0.9em;\">Notes are stored in <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\">notes.git</a>.</p>" +write_page_footer +publish_page + diff --git a/bin/reproducible_stats.sh b/bin/reproducible_stats.sh index 08e1d730..c6188712 100755 --- a/bin/reproducible_stats.sh +++ b/bin/reproducible_stats.sh @@ -21,8 +21,6 @@ declare -A BAD declare -A UGLY declare -A SOURCELESS declare -A NOTFORUS -declare -A STAR -declare -A LINKTARGET LAST24="AND build_date > datetime('now', '-24 hours') " LAST48="AND build_date > datetime('now', '-48 hours') " GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY build_date DESC" | xargs echo) @@ -53,414 +51,14 @@ PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc) PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc) # -# gather notes -# -WORKSPACE=$PWD -cd /var/lib/jenkins -if [ -d notes.git ] ; then - cd notes.git - git pull -else - git clone git://git.debian.org/git/reproducible/notes.git notes.git -fi -cd $WORKSPACE - -PACKAGES_YML=/var/lib/jenkins/notes.git/packages.yml -ISSUES_YML=/var/lib/jenkins/notes.git/issues.yml - -declare -A NOTES_VERSION -declare -A NOTES_ISSUES -declare -A NOTES_BUGS -declare -A NOTES_COMMENTS -declare -A ISSUES_DESCRIPTION -declare -A ISSUES_URL - -show_multi_values() { - TMPFILE=$(mktemp) - echo "$@" > $TMPFILE - while IFS= read -r p ; do - if [ "$p" = "-" ] || [ "$p" = "" ] ; then - continue - elif [ "${p:0:2}" = "- " ] ; then - p="${p:2}" - fi - echo " $PROPERTY = $p" - done < $TMPFILE - unset IFS - rm $TMPFILE -} - -tag_property_loop() { - BEFORE=$1 - shift - AFTER=$1 - shift - TMPFILE=$(mktemp) - echo "$@" > $TMPFILE - while IFS= read -r p ; do - if [ "$p" = "-" ] || [ "$p" = "" ] ; then - continue - elif [ "${p:0:2}" = "- " ] ; then - p="${p:2}" - fi - write_page "$BEFORE" - if $BUG ; then - # turn bugs into links - p="<a href=\"https://bugs.debian.org/$p\">#$p</a>" - else - # turn URLs into links - p="$(echo $p |sed -e 's|http[s:]*//[^ ]*|<a href=\"\0\">\0</a>|g')" - fi - write_page "$p" - write_page "$AFTER" - done < $TMPFILE - unset IFS - rm $TMPFILE -} - -issues_loop() { - TTMPFILE=$(mktemp) - echo "$@" > $TTMPFILE - while IFS= read -r p ; do - if [ "${p:0:2}" = "- " ] ; then - p="${p:2}" - fi - write_page "<table class=\"body\"><tr><td>Identifier:</td><td><a href=\"$JENKINS_URL/userContent/issues/${p}_issue.html\" target=\"_parent\">$p</a></tr>" - if [ "${ISSUES_URL[$p]}" != "" ] ; then - write_page "<tr><td>URL</td><td><a href=\"${ISSUES_URL[$p]}\" target=\"_blank\">${ISSUES_URL[$p]}</a></td></tr>" - fi - if [ "${ISSUES_DESCRIPTION[$p]}" != "" ] ; then - write_page "<tr><td>Description</td><td>" - tag_property_loop "" "<br />" "${ISSUES_DESCRIPTION[$p]}" - write_page "</td></tr>" - fi - write_page "</table>" - done < $TTMPFILE - unset IFS - rm $TTMPFILE -} - -create_pkg_note() { - BUG=false - rm -f $PAGE - # write_page_header() is not used as it contains the <h2> tag... - write_page "<!DOCTYPE html><html><head>" - write_page "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />" - write_page "<link href=\"$JENKINS_URL/userContent/static/style.css\" type=\"text/css\" rel=\"stylesheet\" />" - write_page "<title>Notes for $1</title></head>" - write_page "<body></header>" - write_page "<table class=\"body\">" - - write_page "<tr><td>Version annotated:</td><td colspan=\"2\">${NOTES_VERSION[$1]}</td></tr>" - - if [ "${NOTES_ISSUES[$1]}" != "" ] ; then - write_page "<tr><td colspan=\"2\">Identified issues:</td><td>" - issues_loop "${NOTES_ISSUES[$1]}" - write_page "</td></tr>" - fi - - BUG=true - if [ "${NOTES_BUGS[$1]}" != "" ] ; then - write_page "<tr><td>Bugs noted:</td>" - write_page "<td colspan=\"2\">" - tag_property_loop "" "<br />" "${NOTES_BUGS[$1]}" - write_page "</tr>" - fi - BUG=false - - if [ "${NOTES_COMMENTS[$1]}" != "" ] ; then - write_page "<tr><td>Comments:</td>" - write_page "<td colspan=\"2\">" - tag_property_loop "" "<br />" "${NOTES_COMMENTS[$1]}" - write_page "</tr>" - fi - write_page "<tr><td colspan=\"3\"> </td></tr>" - write_page "<tr><td colspan=\"3\" style=\"text-align:right; font-size:0.9em;\">" - write_page "Notes are stored in <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\">notes.git</a>." - write_page "</td></tr></table>" - write_page_footer -} - -create_issue() { - BUG=false - write_page_header "" "Notes about issue '$1'" - write_page "<table class=\"body\">" - - write_page "<tr><td>Identifier:</td><td colspan=\"2\">$1</td></tr>" - - if [ "${ISSUES_URL[$1]}" != "" ] ; then - write_page "<tr><td>URL:</td><td colspan=\"2\"><a href=\"${ISSUES_URL[$1]}\">${ISSUES_URL[$1]}</a></td></tr>" - fi - if [ "${ISSUES_DESCRIPTION[$1]}" != "" ] ; then - write_page "<tr><td>Description:</td>" - write_page "<td colspan=\"2\">" - tag_property_loop "" "<br />" "${ISSUES_DESCRIPTION[$1]}" - write_page "</td></tr>" - fi - - write_page "<tr><td colspan=\"2\">Packages known to be affected by this issue:</td><td>" - BETA_SIGN=false - for PKG in $PACKAGES_WITH_NOTES ; do - if [ "${NOTES_ISSUES[$PKG]}" != "" ] ; then - TTMPFILE=$(mktemp) - echo "${NOTES_ISSUES[$PKG]}" > $TTMPFILE - while IFS= read -r p ; do - if [ "${p:0:2}" = "- " ] ; then - p="${p:2}" - fi - if [ "$p" = "$1" ] ; then - write_page " ${LINKTARGET[$PKG]} " - if ! $BETA_SIGN && [ "${STAR[$PKG]}" != "" ] ; then - BETA_SIGN=true - fi - fi - done < $TTMPFILE - unset IFS - rm $TTMPFILE - fi - done - write_page "</td></tr>" - write_page "<tr><td colspan=\"3\"> </td></tr>" - write_page "<tr><td colspan=\"3\" style=\"text-align:right; font-size:0.9em;\">" - write_page "Notes are stored in <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\">notes.git</a>." - write_page "</td></tr></table>" - write_page_meta_sign - write_page_footer -} - -write_issues() { - touch $ISSUES_PATH/stamp - for ISSUE in ${ISSUES} ; do - PAGE=$ISSUES_PATH/${ISSUE}_issue.html - create_issue $ISSUE - done - cd $ISSUES_PATH - for FILE in *.html ; do - # if issue is older than stamp file... - if [ $FILE -ot stamp ] ; then - rm $FILE - fi - done - rm stamp - cd - > /dev/null -} - -parse_issues() { - ISSUES=$(cat ${ISSUES_YML} | /srv/jenkins/bin/shyaml keys) - for ISSUE in ${ISSUES} ; do - echo " Issue = ${ISSUE}" - for PROPERTY in url description ; do - VALUE="$(cat ${ISSUES_YML} | /srv/jenkins/bin/shyaml get-value ${ISSUE}.${PROPERTY} )" - if [ "$VALUE" != "" ] ; then - case $PROPERTY in - url) ISSUES_URL[${ISSUE}]=$VALUE - echo " $PROPERTY = $VALUE" - ;; - description) ISSUES_DESCRIPTION[${ISSUE}]=$VALUE - show_multi_values "$VALUE" - ;; - esac - fi - done - done -} - -write_notes() { - touch $NOTES_PATH/stamp - for PKG in $PACKAGES_WITH_NOTES ; do - PAGE=$NOTES_PATH/${PKG}_note.html - create_pkg_note $PKG - done - cd $NOTES_PATH - for FILE in *.html ; do - PKG_FILE=/var/lib/jenkins/userContent/rb-pkg/${FILE:0:-10}.html - # if note was removed... - if [ $FILE -ot stamp ] ; then - # cleanup old notes - rm $FILE - # force re-creation of package file if there was a note - rm ${PKG_FILE} - else - # ... else re-recreate ${PKG_FILE} if it does not contain a link to the note - grep _note.html ${PKG_FILE} > /dev/null || rm ${PKG_FILE} - fi - done - rm stamp - cd - > /dev/null -} - -parse_notes() { - PACKAGES_WITH_NOTES=$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml keys) - for PKG in $PACKAGES_WITH_NOTES ; do - echo " Package = ${PKG}" - for PROPERTY in version issues bugs comments ; do - VALUE="$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml get-value ${PKG}.${PROPERTY} )" - if [ "$VALUE" != "" ] ; then - case $PROPERTY in - version) NOTES_VERSION[${PKG}]=$VALUE - echo " $PROPERTY = $VALUE" - ;; - issues) NOTES_ISSUES[${PKG}]=$VALUE - show_multi_values "$VALUE" - ;; - bugs) NOTES_BUGS[${PKG}]=$VALUE - show_multi_values "$VALUE" - ;; - comments) NOTES_COMMENTS[${PKG}]=$VALUE - show_multi_values "$VALUE" - ;; - esac - fi - done - done -} - -validate_yaml() { - VALID_YAML=true - set +e - cat $1 | /srv/jenkins/bin/shyaml keys > /dev/null 2>&1 || VALID_YAML=false - cat $1 | /srv/jenkins/bin/shyaml get-values > /dev/null 2>&1 || VALID_YAML=false - set -e - echo "$1 is valid yaml: $VALID_YAML" -} - -# -# end note parsing functions... -# - -init_pkg_page() { - echo "<!DOCTYPE html><html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />" > ${PKG_FILE} - echo "<link href=\"../static/style.css\" type=\"text/css\" rel=\"stylesheet\" />" >> ${PKG_FILE} - echo "<title>$1 - reproducible builds results</title></head>" >> ${PKG_FILE} - echo "<body><table class=\"head\"><tr><td><span style=\"font-size:1.2em;\">$1</span> $2" >> ${PKG_FILE} - set_icon "$3" $5 # this sets $STATE_TARGET_NAME and $ICON - echo "<a href=\"$JENKINS_URL/userContent/index_${STATE_TARGET_NAME}.html\" target=\"_parent\"><img src=\"$JENKINS_URL/userContent/static/$ICON\" alt=\"${STATE_TARGET_NAME} icon\" /></a>" >> ${PKG_FILE} - echo "<span style=\"font-size:0.9em;\">at $4:</span> " >> ${PKG_FILE} -} - -append2pkg_page() { - echo "$1" >> ${PKG_FILE} -} - -finish_pkg_page() { - echo "</td><td style=\"text-align:right; font-size:0.9em;\"><a href=\"$JENKINS_URL/userContent/reproducible.html\" target=\"_parent\">reproducible builds</a></td></tr></table>" >> ${PKG_FILE} - echo "<iframe name=\"main\" src=\"$1\" width=\"100%\" height=\"98%\" frameborder=\"0\">" >> ${PKG_FILE} - echo "<p>Your browser does not support iframes. Use a different one or follow the links above.</p>" >> ${PKG_FILE} - echo "</iframe>" >> ${PKG_FILE} - echo "</body></html>" >> ${PKG_FILE} -} - -set_package_class() { - if [ -f ${NOTES_PATH}/${PKG}_note.html ] ; then - CLASS="class=\"noted\"" - else - CLASS="class=\"package\"" - fi -} - -process_packages() { - for PKG in $@ ; do - RESULT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT build_date,version,status FROM source_packages WHERE name = \"$PKG\"") - BUILD_DATE=$(echo $RESULT|cut -d "|" -f1) - # version with epoch removed - EVERSION=$(echo $RESULT | cut -d "|" -f2 | cut -d ":" -f2) - if $BUILDINFO_SIGNS && [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then - STAR[$PKG]="<span class=\"beta\">β</span>" # used to be a star... - fi - # only build $PKG pages if they don't exist or are older than $BUILD_DATE or have a note - PKG_FILE="/var/lib/jenkins/userContent/rb-pkg/${PKG}.html" - OLD_FILE=$(find $(dirname ${PKG_FILE}) -name $(basename ${PKG_FILE}) ! -newermt "$BUILD_DATE" 2>/dev/null || true) - # if no package file exists, or is older than last build_date - if [ ! -f ${PKG_FILE} ] || [ "$OLD_FILE" != "" ] ; then - VERSION=$(echo $RESULT | cut -d "|" -f2) - STATUS=$(echo $RESULT | cut -d "|" -f3) - MAINLINK="" - NOTES_LINK="" - if [ -f ${NOTES_PATH}/${PKG}_note.html ] ; then - NOTES_LINK=" <a href=\"$JENKINS_URL/userContent/notes/${PKG}_note.html\" target=\"main\">notes</a> " - fi - init_pkg_page "$PKG" "$VERSION" "$STATUS" "$BUILD_DATE" "${STAR[$PKG]}" - append2pkg_page "${NOTES_LINK}" - if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then - append2pkg_page " <a href=\"$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo\" target=\"main\">buildinfo</a> " - MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" - fi - if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then - append2pkg_page " <a href=\"$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html\" target=\"main\">debbindiff</a> " - MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" - fi - RBUILD_LOG="rbuild/${PKG}_${EVERSION}.rbuild.log" - if [ -f "/var/lib/jenkins/userContent/${RBUILD_LOG}" ] ; then - SIZE=$(du -sh "/var/lib/jenkins/userContent/${RBUILD_LOG}" |cut -f1) - append2pkg_page " <a href=\"$JENKINS_URL/userContent/${RBUILD_LOG}\" target=\"main\">rbuild ($SIZE)</a> " - if [ "$MAINLINK" = "" ] ; then - MAINLINK="$JENKINS_URL/userContent/${RBUILD_LOG}" - fi - fi - append2pkg_page " <a href=\"https://packages.qa.debian.org/${PKG}\" target=\"main\">PTS</a> " - append2pkg_page " <a href=\"https://bugs.debian.org/src:${PKG}\" target=\"main\">BTS</a> " - append2pkg_page " <a href=\"https://sources.debian.net/src/${PKG}/\" target=\"main\">sources</a> " - append2pkg_page " <a href=\"https://sources.debian.net/src/${PKG}/${VERSION}/debian/rules\" target=\"main\">debian/rules</a> " - - if [ ! -z "${NOTES_LINK}" ] ; then - MAINLINK="$JENKINS_URL/userContent/notes/${PKG}_note.html" - fi - finish_pkg_page "$MAINLINK" - fi - if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then - set_package_class - LINKTARGET[$PKG]="<a href=\"$JENKINS_URL/userContent/rb-pkg/$PKG.html\" $CLASS>$PKG</a>${STAR[$PKG]}" - else - LINKTARGET[$PKG]="$PKG" - fi - done -} - -force_package_targets() { - for PKG in $@ ; do - set_package_class - LINKTARGET[$PKG]="<a href=\"$JENKINS_URL/userContent/rb-pkg/$PKG.html\" $CLASS>$PKG</a>${STAR[$PKG]}" - done -} - -link_packages() { - for PKG in $@ ; do - write_page " ${LINKTARGET[$PKG]} " - if ! $BETA_SIGN && [ "${STAR[$PKG]}" != "" ] ; then - BETA_SIGN=true - fi - done -} - -# -# actually parse the notes -# -validate_yaml ${ISSUES_YML} -validate_yaml ${PACKAGES_YML} -if $VALID_YAML ; then - echo "$(date) - processing notes and issues" - parse_issues - parse_notes - echo "$(date) - processing packages with notes" - process_packages ${PACKAGES_WITH_NOTES} - force_package_targets ${PACKAGES_WITH_NOTES} - write_issues - write_notes -else - echo "Warning: ${ISSUES_YML} or ${PACKAGES_YML} contains invalid yaml, please fix." -fi - -# # actually build the package pages # echo "$(date) - processing $COUNT_TOTAL packages... this will take a while." -BUILDINFO_SIGNS=true process_packages ${BAD["all"]} -BUILDINFO_SIGNS=false process_packages ${UGLY["all"]} ${GOOD["all"]} ${SOURCELESS["all"]} ${NOTFORUS["all"]} $BLACKLISTED for VIEW in $ALLVIEWS ; do - BETA_SIGN=false + BUILDINFO_SIGNS=true PAGE=index_${VIEW}.html echo "$(date) - starting to write $PAGE page." write_page_header $VIEW "Overview of reproducible builds of ${SPOKENTARGET[$VIEW]}" @@ -513,6 +111,7 @@ for VIEW in $ALLVIEWS ; do set_icon reproducible write_icon write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH <code>" + BUILDINFO_SIGNS=false link_packages ${GOOD[$VIEW]} write_page "</code></p>" write_page_meta_sign @@ -520,61 +119,26 @@ for VIEW in $ALLVIEWS ; do publish_page done -VIEW=notes -PAGE=index_${VIEW}.html -echo "$(date) - starting to write $PAGE page." -write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}" -if $VALID_YAML ; then - BETA_SIGN=false - write_page "<p>Packages which have notes: <code>" - force_package_targets ${PACKAGES_WITH_NOTES} - PACKAGES_WITH_NOTES=$(echo $PACKAGES_WITH_NOTES | sed -s "s# #\n#g" | sort | xargs echo) - link_packages $PACKAGES_WITH_NOTES - write_page "</code></p>" -else - write_page "<p style=\"font-size:1.5em; color: red;\">Broken .yaml files in notes.git could not be parsed, please investigate and fix!</p>" -fi -write_page "<p style=\"font-size:0.9em;\">Notes are stored in <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\">notes.git</a>.</p>" -write_page_meta_sign -write_page_footer -publish_page - -VIEW=issues -PAGE=index_${VIEW}.html -echo "$(date) - starting to write $PAGE page." -write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}" -if $VALID_YAML ; then - write_page "<table class=\"body\">" - ISSUES=$(echo ${ISSUES} | sed -s "s# #\n#g" | sort | xargs echo) - for ISSUE in ${ISSUES} ; do - write_page "<tr><td><a href=\"$JENKINS_URL/userContent/issues/${ISSUE}_issue.html\">${ISSUE}</a></td></tr>" - done - write_page "</table>" -else - write_page "<p style=\"font-size:1.5em; color: red;\">Broken .yaml files in notes.git could not be parsed, please investigate and fix!</p>" -fi -write_page "<p style=\"font-size:0.9em;\">Notes are stored in <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\">notes.git</a>.</p>" -write_page_footer -publish_page - count_packages() { COUNT=${#@} PERCENT=$(echo "scale=1 ; ($COUNT*100/$COUNT_TOTAL)" | bc) } for STATE in $ALLSTATES ; do - BETA_SIGN=false + BUILDINFO_SIGNS=true PAGE=index_${STATE}.html echo "$(date) - starting to write $PAGE page." write_page_header $STATE "Overview of ${SPOKENTARGET[$STATE]}" WITH="" case "$STATE" in - reproducible) PACKAGES=${GOOD["all"]} + reproducible) BUILDINFO_SIGNS=false + PACKAGES=${GOOD["all"]} ;; FTBR) CANDIDATES=${BAD["all"]} PACKAGES="" for PKG in $CANDIDATES ; do - if [ "${STAR[$PKG]}" = "" ] ; then + set_package_star + if [ "$STAR" = "" ] ; then PACKAGES="$PACKAGES $PKG" fi done @@ -582,7 +146,8 @@ for STATE in $ALLSTATES ; do FTBR_with_buildinfo) CANDIDATES=${BAD["all"]} PACKAGES="" for PKG in $CANDIDATES ; do - if [ "${STAR[$PKG]}" != "" ] ; then + set_package_star + if [ "$STAR" != "" ] ; then PACKAGES="$PACKAGES $PKG" fi done @@ -610,31 +175,6 @@ for STATE in $ALLSTATES ; do publish_page done -VIEW=dd-list -PAGE=index_${VIEW}.html -echo "$(date) - starting to write $PAGE page." -write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}" -TMPFILE=$(mktemp) -echo "${BAD["all"]}" | dd-list -i > $TMPFILE || true -write_page "<p>The following maintainers and uploaders are listed for packages which have built unreproducibly:</p><p><pre>" -while IFS= read -r LINE ; do - if [ "${LINE:0:3}" = " " ] ; then - PACKAGE=$(echo "${LINE:3}" | cut -d " " -f1) - UPLOADERS=$(echo "${LINE:3}" | cut -d " " -f2-) - if [ "$UPLOADERS" = "$PACKAGE" ] ; then - UPLOADERS="" - fi - write_page " <a href=\"$JENKINS_URL/userContent/rb-pkg/$PACKAGE.html\">$PACKAGE</a> $UPLOADERS" - else - LINE="$(echo $LINE | sed 's#&#\&#g ; s#<#\<#g ; s#>#\>#g')" - write_page "$LINE" - fi -done < $TMPFILE -write_page "</pre></p>" -rm $TMPFILE -write_page_footer -publish_page - # # create stats # @@ -802,4 +342,3 @@ write_page "</p>" write_page_footer publish_page -echo "Enjoy $JENKINS_URL/userContent/reproducible.html" diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index 95572bed..6390fac8 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -89,6 +89,10 @@ name: '{name}_dd_list' - job-template: + defaults: reproducible + name: '{name}_notes' + +- job-template: defaults: reproducible_builder name: '{name}_builder_alpha' @@ -119,6 +123,10 @@ my_description: 'Generates HTML results (dd-list) for reproducible builds.' my_timed: '53 23 * * *' my_shell: '/srv/jenkins/bin/reproducible_html_dd_list.sh' + - '{name}_notes': + my_description: 'Generates HTML results (notes) for reproducible builds, triggered by changes to the notes.git repository.' + my_timed: '' + my_shell: '/srv/jenkins/bin/reproducible_html_notes.sh' - '{name}_builder_alpha': my_description: 'Try to reproducibly build a scheduled package. This is one of several builder jobs.' my_timed: 'H/2 * * * *' |