From 044a78e5a985f110f2bd15a9102f673f23997e09 Mon Sep 17 00:00:00 2001
From: Holger Levsen
"
- done < $TTMPFILE
- unset IFS
- rm $TTMPFILE
-}
-
-create_pkg_note() {
- BUG=false
- rm -f $PAGE
- # write_page_header() is not used as it contains the "
- if [ "${ISSUES_URL[$p]}" != "" ] ; then
- write_page "Identifier: $p "
- fi
- if [ "${ISSUES_DESCRIPTION[$p]}" != "" ] ; then
- write_page "URL ${ISSUES_URL[$p]} "
- fi
- write_page "Description "
- tag_property_loop "" "
" "${ISSUES_DESCRIPTION[$p]}"
- write_page " tag...
- write_page ""
- write_page ""
- write_page ""
- write_page "
"
-
- write_page "
"
- write_page_footer
-}
-
-create_issue() {
- BUG=false
- write_page_header "" "Notes about issue '$1'"
- write_page " "
-
- if [ "${NOTES_ISSUES[$1]}" != "" ] ; then
- write_page "Version annotated: ${NOTES_VERSION[$1]} "
- fi
-
- BUG=true
- if [ "${NOTES_BUGS[$1]}" != "" ] ; then
- write_page "Identified issues: "
- issues_loop "${NOTES_ISSUES[$1]}"
- write_page " "
- fi
- BUG=false
-
- if [ "${NOTES_COMMENTS[$1]}" != "" ] ; then
- write_page "Bugs noted: "
- write_page ""
- tag_property_loop "" "
" "${NOTES_BUGS[$1]}"
- write_page " "
- fi
- write_page "Comments: "
- write_page ""
- tag_property_loop "" "
" "${NOTES_COMMENTS[$1]}"
- write_page " "
- write_page " "
- write_page "Notes are stored in notes.git."
- write_page " "
-
- write_page "
"
- write_page_meta_sign
- write_page_footer
-}
-
-write_issues() {
- touch $ISSUES_PATH/stamp
- for ISSUE in ${ISSUES} ; do
- PAGE=$ISSUES_PATH/${ISSUE}_issue.html
- create_issue $ISSUE
- done
- cd $ISSUES_PATH
- for FILE in *.html ; do
- # if issue is older than stamp file...
- if [ $FILE -ot stamp ] ; then
- rm $FILE
- fi
- done
- rm stamp
- cd - > /dev/null
-}
-
-parse_issues() {
- ISSUES=$(cat ${ISSUES_YML} | /srv/jenkins/bin/shyaml keys)
- for ISSUE in ${ISSUES} ; do
- echo " Issue = ${ISSUE}"
- for PROPERTY in url description ; do
- VALUE="$(cat ${ISSUES_YML} | /srv/jenkins/bin/shyaml get-value ${ISSUE}.${PROPERTY} )"
- if [ "$VALUE" != "" ] ; then
- case $PROPERTY in
- url) ISSUES_URL[${ISSUE}]=$VALUE
- echo " $PROPERTY = $VALUE"
- ;;
- description) ISSUES_DESCRIPTION[${ISSUE}]=$VALUE
- show_multi_values "$VALUE"
- ;;
- esac
- fi
- done
- done
-}
-
-write_notes() {
- touch $NOTES_PATH/stamp
- for PKG in $PACKAGES_WITH_NOTES ; do
- PAGE=$NOTES_PATH/${PKG}_note.html
- create_pkg_note $PKG
- done
- cd $NOTES_PATH
- for FILE in *.html ; do
- PKG_FILE=/var/lib/jenkins/userContent/rb-pkg/${FILE:0:-10}.html
- # if note was removed...
- if [ $FILE -ot stamp ] ; then
- # cleanup old notes
- rm $FILE
- # force re-creation of package file if there was a note
- rm ${PKG_FILE}
- else
- # ... else re-recreate ${PKG_FILE} if it does not contain a link to the note
- grep _note.html ${PKG_FILE} > /dev/null || rm ${PKG_FILE}
- fi
- done
- rm stamp
- cd - > /dev/null
-}
-
-parse_notes() {
- PACKAGES_WITH_NOTES=$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml keys)
- for PKG in $PACKAGES_WITH_NOTES ; do
- echo " Package = ${PKG}"
- for PROPERTY in version issues bugs comments ; do
- VALUE="$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml get-value ${PKG}.${PROPERTY} )"
- if [ "$VALUE" != "" ] ; then
- case $PROPERTY in
- version) NOTES_VERSION[${PKG}]=$VALUE
- echo " $PROPERTY = $VALUE"
- ;;
- issues) NOTES_ISSUES[${PKG}]=$VALUE
- show_multi_values "$VALUE"
- ;;
- bugs) NOTES_BUGS[${PKG}]=$VALUE
- show_multi_values "$VALUE"
- ;;
- comments) NOTES_COMMENTS[${PKG}]=$VALUE
- show_multi_values "$VALUE"
- ;;
- esac
- fi
- done
- done
-}
-
-validate_yaml() {
- VALID_YAML=true
- set +e
- cat $1 | /srv/jenkins/bin/shyaml keys > /dev/null 2>&1 || VALID_YAML=false
- cat $1 | /srv/jenkins/bin/shyaml get-values > /dev/null 2>&1 || VALID_YAML=false
- set -e
- echo "$1 is valid yaml: $VALID_YAML"
-}
-
-#
-# end note parsing functions...
-#
-
-init_pkg_page() {
- echo "" > ${PKG_FILE}
- echo "" >> ${PKG_FILE}
- echo " "
-
- if [ "${ISSUES_URL[$1]}" != "" ] ; then
- write_page "Identifier: $1 "
- fi
- if [ "${ISSUES_DESCRIPTION[$1]}" != "" ] ; then
- write_page "URL: ${ISSUES_URL[$1]} "
- fi
-
- write_page "Description: "
- write_page ""
- tag_property_loop "" "
" "${ISSUES_DESCRIPTION[$1]}"
- write_page " "
- write_page "Packages known to be affected by this issue: "
- BETA_SIGN=false
- for PKG in $PACKAGES_WITH_NOTES ; do
- if [ "${NOTES_ISSUES[$PKG]}" != "" ] ; then
- TTMPFILE=$(mktemp)
- echo "${NOTES_ISSUES[$PKG]}" > $TTMPFILE
- while IFS= read -r p ; do
- if [ "${p:0:2}" = "- " ] ; then
- p="${p:2}"
- fi
- if [ "$p" = "$1" ] ; then
- write_page " ${LINKTARGET[$PKG]} "
- if ! $BETA_SIGN && [ "${STAR[$PKG]}" != "" ] ; then
- BETA_SIGN=true
- fi
- fi
- done < $TTMPFILE
- unset IFS
- rm $TTMPFILE
- fi
- done
- write_page " "
- write_page " "
- write_page "Notes are stored in notes.git."
- write_page "
" >> ${PKG_FILE}
- echo "" >> ${PKG_FILE}
- echo "" >> ${PKG_FILE}
-}
-
-set_package_class() {
- if [ -f ${NOTES_PATH}/${PKG}_note.html ] ; then
- CLASS="class=\"noted\""
- else
- CLASS="class=\"package\""
- fi
-}
-
-process_packages() {
- for PKG in $@ ; do
- RESULT=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT build_date,version,status FROM source_packages WHERE name = \"$PKG\"")
- BUILD_DATE=$(echo $RESULT|cut -d "|" -f1)
- # version with epoch removed
- EVERSION=$(echo $RESULT | cut -d "|" -f2 | cut -d ":" -f2)
- if $BUILDINFO_SIGNS && [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
- STAR[$PKG]="β" # used to be a star...
- fi
- # only build $PKG pages if they don't exist or are older than $BUILD_DATE or have a note
- PKG_FILE="/var/lib/jenkins/userContent/rb-pkg/${PKG}.html"
- OLD_FILE=$(find $(dirname ${PKG_FILE}) -name $(basename ${PKG_FILE}) ! -newermt "$BUILD_DATE" 2>/dev/null || true)
- # if no package file exists, or is older than last build_date
- if [ ! -f ${PKG_FILE} ] || [ "$OLD_FILE" != "" ] ; then
- VERSION=$(echo $RESULT | cut -d "|" -f2)
- STATUS=$(echo $RESULT | cut -d "|" -f3)
- MAINLINK=""
- NOTES_LINK=""
- if [ -f ${NOTES_PATH}/${PKG}_note.html ] ; then
- NOTES_LINK=" notes "
- fi
- init_pkg_page "$PKG" "$VERSION" "$STATUS" "$BUILD_DATE" "${STAR[$PKG]}"
- append2pkg_page "${NOTES_LINK}"
- if [ -f "/var/lib/jenkins/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo" ] ; then
- append2pkg_page " buildinfo "
- MAINLINK="$JENKINS_URL/userContent/buildinfo/${PKG}_${EVERSION}_amd64.buildinfo"
- fi
- if [ -f "/var/lib/jenkins/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html" ] ; then
- append2pkg_page " debbindiff "
- MAINLINK="$JENKINS_URL/userContent/dbd/${PKG}_${EVERSION}.debbindiff.html"
- fi
- RBUILD_LOG="rbuild/${PKG}_${EVERSION}.rbuild.log"
- if [ -f "/var/lib/jenkins/userContent/${RBUILD_LOG}" ] ; then
- SIZE=$(du -sh "/var/lib/jenkins/userContent/${RBUILD_LOG}" |cut -f1)
- append2pkg_page " rbuild ($SIZE) "
- if [ "$MAINLINK" = "" ] ; then
- MAINLINK="$JENKINS_URL/userContent/${RBUILD_LOG}"
- fi
- fi
- append2pkg_page " PTS "
- append2pkg_page " BTS "
- append2pkg_page " sources "
- append2pkg_page " debian/rules "
-
- if [ ! -z "${NOTES_LINK}" ] ; then
- MAINLINK="$JENKINS_URL/userContent/notes/${PKG}_note.html"
- fi
- finish_pkg_page "$MAINLINK"
- fi
- if [ -f "/var/lib/jenkins/userContent/rbuild/${PKG}_${EVERSION}.rbuild.log" ] ; then
- set_package_class
- LINKTARGET[$PKG]="$PKG${STAR[$PKG]}"
- else
- LINKTARGET[$PKG]="$PKG"
- fi
- done
-}
-
-force_package_targets() {
- for PKG in $@ ; do
- set_package_class
- LINKTARGET[$PKG]="$PKG${STAR[$PKG]}"
- done
-}
-
-link_packages() {
- for PKG in $@ ; do
- write_page " ${LINKTARGET[$PKG]} "
- if ! $BETA_SIGN && [ "${STAR[$PKG]}" != "" ] ; then
- BETA_SIGN=true
- fi
- done
-}
-
-#
-# actually parse the notes
-#
-validate_yaml ${ISSUES_YML}
-validate_yaml ${PACKAGES_YML}
-if $VALID_YAML ; then
- echo "$(date) - processing notes and issues"
- parse_issues
- parse_notes
- echo "$(date) - processing packages with notes"
- process_packages ${PACKAGES_WITH_NOTES}
- force_package_targets ${PACKAGES_WITH_NOTES}
- write_issues
- write_notes
-else
- echo "Warning: ${ISSUES_YML} or ${PACKAGES_YML} contains invalid yaml, please fix."
-fi
-
#
# actually build the package pages
#
echo "$(date) - processing $COUNT_TOTAL packages... this will take a while."
-BUILDINFO_SIGNS=true
process_packages ${BAD["all"]}
-BUILDINFO_SIGNS=false
process_packages ${UGLY["all"]} ${GOOD["all"]} ${SOURCELESS["all"]} ${NOTFORUS["all"]} $BLACKLISTED
for VIEW in $ALLVIEWS ; do
- BETA_SIGN=false
+ BUILDINFO_SIGNS=true
PAGE=index_${VIEW}.html
echo "$(date) - starting to write $PAGE page."
write_page_header $VIEW "Overview of reproducible builds of ${SPOKENTARGET[$VIEW]}"
@@ -513,6 +111,7 @@ for VIEW in $ALLVIEWS ; do
set_icon reproducible
write_icon
write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH $1 $2" >> ${PKG_FILE}
- set_icon "$3" $5 # this sets $STATE_TARGET_NAME and $ICON
- echo "" >> ${PKG_FILE}
- echo "at $4: " >> ${PKG_FILE}
-}
-
-append2pkg_page() {
- echo "$1" >> ${PKG_FILE}
-}
-
-finish_pkg_page() {
- echo " reproducible builds "
+ BUILDINFO_SIGNS=false
link_packages ${GOOD[$VIEW]}
write_page "
Packages which have notes: "
- force_package_targets ${PACKAGES_WITH_NOTES}
- PACKAGES_WITH_NOTES=$(echo $PACKAGES_WITH_NOTES | sed -s "s# #\n#g" | sort | xargs echo)
- link_packages $PACKAGES_WITH_NOTES
- write_page "
Broken .yaml files in notes.git could not be parsed, please investigate and fix!
" -fi -write_page "Notes are stored in notes.git.
" -write_page_meta_sign -write_page_footer -publish_page - -VIEW=issues -PAGE=index_${VIEW}.html -echo "$(date) - starting to write $PAGE page." -write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}" -if $VALID_YAML ; then - write_page "${ISSUE} |
Broken .yaml files in notes.git could not be parsed, please investigate and fix!
" -fi -write_page "Notes are stored in notes.git.
" -write_page_footer -publish_page - count_packages() { COUNT=${#@} PERCENT=$(echo "scale=1 ; ($COUNT*100/$COUNT_TOTAL)" | bc) } for STATE in $ALLSTATES ; do - BETA_SIGN=false + BUILDINFO_SIGNS=true PAGE=index_${STATE}.html echo "$(date) - starting to write $PAGE page." write_page_header $STATE "Overview of ${SPOKENTARGET[$STATE]}" WITH="" case "$STATE" in - reproducible) PACKAGES=${GOOD["all"]} + reproducible) BUILDINFO_SIGNS=false + PACKAGES=${GOOD["all"]} ;; FTBR) CANDIDATES=${BAD["all"]} PACKAGES="" for PKG in $CANDIDATES ; do - if [ "${STAR[$PKG]}" = "" ] ; then + set_package_star + if [ "$STAR" = "" ] ; then PACKAGES="$PACKAGES $PKG" fi done @@ -582,7 +146,8 @@ for STATE in $ALLSTATES ; do FTBR_with_buildinfo) CANDIDATES=${BAD["all"]} PACKAGES="" for PKG in $CANDIDATES ; do - if [ "${STAR[$PKG]}" != "" ] ; then + set_package_star + if [ "$STAR" != "" ] ; then PACKAGES="$PACKAGES $PKG" fi done @@ -610,31 +175,6 @@ for STATE in $ALLSTATES ; do publish_page done -VIEW=dd-list -PAGE=index_${VIEW}.html -echo "$(date) - starting to write $PAGE page." -write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}" -TMPFILE=$(mktemp) -echo "${BAD["all"]}" | dd-list -i > $TMPFILE || true -write_page "The following maintainers and uploaders are listed for packages which have built unreproducibly:
" -while IFS= read -r LINE ; do - if [ "${LINE:0:3}" = " " ] ; then - PACKAGE=$(echo "${LINE:3}" | cut -d " " -f1) - UPLOADERS=$(echo "${LINE:3}" | cut -d " " -f2-) - if [ "$UPLOADERS" = "$PACKAGE" ] ; then - UPLOADERS="" - fi - write_page " $PACKAGE $UPLOADERS" - else - LINE="$(echo $LINE | sed 's#\&#g ; s#<#\<#g ; s#>#\>#g')" - write_page "$LINE" - fi -done < $TMPFILE -write_page "" -rm $TMPFILE -write_page_footer -publish_page - # # create stats # @@ -802,4 +342,3 @@ write_page "" write_page_footer publish_page -echo "Enjoy $JENKINS_URL/userContent/reproducible.html" -- cgit v1.2.3-70-g09d2