reproducible: refactor, mv notes+issues html generation into seperate job

author: Holger Levsen <holger@layer-acht.org> 2014-10-17 19:40:32 +0200
committer: Holger Levsen <holger@layer-acht.org> 2014-10-17 19:40:32 +0200
commit: 044a78e5a985f110f2bd15a9102f673f23997e09 (patch)
tree: 97300dae2ba33760b80e215965c869a354ba2440 /bin/reproducible_html_notes.sh
parent: 1aa7e0422d9f0a4561aa952ce6a89a1ba9bedd31 (diff)
download: jenkins.debian.net-044a78e5a985f110f2bd15a9102f673f23997e09.tar.xz
1 files changed, 358 insertions, 0 deletions
diff --git a/bin/reproducible_html_notes.sh b/bin/reproducible_html_notes.sh
new file mode 100755
index 00000000..202dd083
--- /dev/null
+++ b/bin/reproducible_html_notes.sh
@@ -0,0 +1,358 @@
+#!/bin/bash
+
+# Copyright 2014 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv=2
+
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code defining db access
+. /srv/jenkins/bin/reproducible_common.sh
+
+# FIXME: move to daily cleanup job
+# cp db away for backup purposes
+cp $PACKAGES_DB /var/lib/jenkins/userContent/reproducible.db
+
+set +x
+init_html
+
+#
+# update git
+#
+
+WORKSPACE=$PWD
+cd /var/lib/jenkins
+if [ -d notes.git ] ; then
+	cd notes.git
+	git pull
+else
+	git clone git://git.debian.org/git/reproducible/notes.git notes.git
+fi
+cd $WORKSPACE
+
+#
+# declare some variables
+#
+
+PACKAGES_YML=/var/lib/jenkins/notes.git/packages.yml
+ISSUES_YML=/var/lib/jenkins/notes.git/issues.yml
+
+declare -A NOTES_VERSION
+declare -A NOTES_ISSUES
+declare -A NOTES_BUGS
+declare -A NOTES_COMMENTS
+declare -A ISSUES_DESCRIPTION
+declare -A ISSUES_URL
+
+#
+# declare some functions only used for dealing with notes
+#
+
+show_multi_values() {
+	TMPFILE=$(mktemp)
+	echo "$@" > $TMPFILE
+	while IFS= read -r p ; do
+		if [ "$p" = "-" ] || [ "$p" = "" ] ; then
+			continue
+		elif [ "${p:0:2}" = "- " ] ; then
+			p="${p:2}"
+		fi
+		echo "    $PROPERTY = $p"
+	done < $TMPFILE
+	unset IFS
+	rm $TMPFILE
+}
+
+tag_property_loop() {
+	BEFORE=$1
+	shift
+	AFTER=$1
+	shift
+	TMPFILE=$(mktemp)
+	echo "$@" > $TMPFILE
+	while IFS= read -r p ; do
+		if [ "$p" = "-" ] || [ "$p" = "" ] ; then
+			continue
+		elif [ "${p:0:2}" = "- " ] ; then
+			p="${p:2}"
+		fi
+		write_page "$BEFORE"
+		if $BUG ; then
+			# turn bugs into links
+			p="<a href=\"https://bugs.debian.org/$p\">#$p</a>"
+		else
+			# turn URLs into links
+			p="$(echo $p |sed  -e 's|http[s:]*//[^ ]*|<a href=\"\0\">\0</a>|g')"
+		fi
+		write_page "$p"
+		write_page "$AFTER"
+	done < $TMPFILE
+	unset IFS
+	rm $TMPFILE
+}
+
+issues_loop() {
+	TTMPFILE=$(mktemp)
+	echo "$@" > $TTMPFILE
+	while IFS= read -r p ; do
+		if [ "${p:0:2}" = "- " ] ; then
+			p="${p:2}"
+		fi
+		write_page "<table class=\"body\"><tr><td>Identifier:</td><td><a href=\"$JENKINS_URL/userContent/issues/${p}_issue.html\" target=\"_parent\">$p</a></tr>"
+		if [ "${ISSUES_URL[$p]}" != "" ] ; then
+			write_page "<tr><td>URL</td><td><a href=\"${ISSUES_URL[$p]}\" target=\"_blank\">${ISSUES_URL[$p]}</a></td></tr>"
+		fi
+		if [ "${ISSUES_DESCRIPTION[$p]}" != "" ] ; then
+			write_page "<tr><td>Description</td><td>"
+			tag_property_loop "" "<br />" "${ISSUES_DESCRIPTION[$p]}"
+			write_page "</td></tr>"
+		fi
+		write_page "</table>"
+	done < $TTMPFILE
+	unset IFS
+	rm $TTMPFILE
+}
+
+create_pkg_note() {
+	BUG=false
+	rm -f $PAGE
+	# write_page_header() is not used as it contains the <h2> tag...
+	write_page "<!DOCTYPE html><html><head>"
+	write_page "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />"
+	write_page "<link href=\"$JENKINS_URL/userContent/static/style.css\" type=\"text/css\" rel=\"stylesheet\" />"
+	write_page "<title>Notes for $1</title></head>"
+	write_page "<body></header>"
+	write_page "<table class=\"body\">"
+
+	write_page "<tr><td>Version annotated:</td><td colspan=\"2\">${NOTES_VERSION[$1]}</td></tr>"
+
+	if [ "${NOTES_ISSUES[$1]}" != "" ] ; then
+		write_page "<tr><td colspan=\"2\">Identified issues:</td><td>"
+		issues_loop "${NOTES_ISSUES[$1]}"
+		write_page "</td></tr>"
+	fi
+
+	BUG=true
+	if [ "${NOTES_BUGS[$1]}" != "" ] ; then
+		write_page "<tr><td>Bugs noted:</td>"
+		write_page "<td colspan=\"2\">"
+		tag_property_loop "" "<br />" "${NOTES_BUGS[$1]}"
+		write_page "</tr>"
+	fi
+	BUG=false
+
+	if [ "${NOTES_COMMENTS[$1]}" != "" ] ; then
+		write_page "<tr><td>Comments:</td>"
+		write_page "<td colspan=\"2\">"
+		tag_property_loop "" "<br />" "${NOTES_COMMENTS[$1]}"
+		write_page "</tr>"
+	fi
+	write_page "<tr><td colspan=\"3\">&nbsp;</td></tr>"
+	write_page "<tr><td colspan=\"3\" style=\"text-align:right; font-size:0.9em;\">"
+	write_page "Notes are stored in <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\">notes.git</a>."
+	write_page "</td></tr></table>"
+	write_page_footer
+}
+
+create_issue() {
+	BUG=false
+	write_page_header "" "Notes about issue '$1'"
+	write_page "<table class=\"body\">"
+
+	write_page "<tr><td>Identifier:</td><td colspan=\"2\">$1</td></tr>"
+
+	if [ "${ISSUES_URL[$1]}" != "" ] ; then
+		write_page "<tr><td>URL:</td><td colspan=\"2\"><a href=\"${ISSUES_URL[$1]}\">${ISSUES_URL[$1]}</a></td></tr>"
+	fi
+	if [ "${ISSUES_DESCRIPTION[$1]}" != "" ] ; then
+		write_page "<tr><td>Description:</td>"
+		write_page "<td colspan=\"2\">"
+		tag_property_loop "" "<br />" "${ISSUES_DESCRIPTION[$1]}"
+		write_page "</td></tr>"
+	fi
+
+	write_page "<tr><td colspan=\"2\">Packages known to be affected by this issue:</td><td>"
+	for PKG in $PACKAGES_WITH_NOTES ; do
+		if [ "${NOTES_ISSUES[$PKG]}" != "" ] ; then
+			TTMPFILE=$(mktemp)
+			echo "${NOTES_ISSUES[$PKG]}" > $TTMPFILE
+			while IFS= read -r p ; do
+				if [ "${p:0:2}" = "- " ] ; then
+					p="${p:2}"
+				fi
+			if [ "$p" = "$1" ] ; then
+				write_page " ${LINKTARGET[$PKG]} "
+			fi
+			done < $TTMPFILE
+			unset IFS
+			rm $TTMPFILE
+		fi
+	done
+	write_page "</td></tr>"
+	write_page "<tr><td colspan=\"3\">&nbsp;</td></tr>"
+	write_page "<tr><td colspan=\"3\" style=\"text-align:right; font-size:0.9em;\">"
+	write_page "Notes are stored in <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\">notes.git</a>."
+	write_page "</td></tr></table>"
+	write_page_meta_sign
+	write_page_footer
+}
+
+write_issues() {
+	touch $ISSUES_PATH/stamp
+	for ISSUE in ${ISSUES} ; do
+		PAGE=$ISSUES_PATH/${ISSUE}_issue.html
+		create_issue $ISSUE
+	done
+	cd $ISSUES_PATH
+	for FILE in *.html ; do
+		# if issue is older than stamp file...
+		if [ $FILE -ot stamp ] ; then
+			rm $FILE
+		fi
+	done
+	rm stamp
+	cd - > /dev/null
+}
+
+parse_issues() {
+	ISSUES=$(cat ${ISSUES_YML} | /srv/jenkins/bin/shyaml keys)
+	for ISSUE in ${ISSUES} ; do
+		echo " Issue = ${ISSUE}"
+		for PROPERTY in url description ; do
+			VALUE="$(cat ${ISSUES_YML} | /srv/jenkins/bin/shyaml get-value ${ISSUE}.${PROPERTY} )"
+			if [ "$VALUE" != "" ] ; then
+				case $PROPERTY in
+					url)		ISSUES_URL[${ISSUE}]=$VALUE
+							echo "    $PROPERTY = $VALUE"
+							;;
+					description)	ISSUES_DESCRIPTION[${ISSUE}]=$VALUE
+							show_multi_values "$VALUE"
+							;;
+				esac
+			fi
+		done
+	done
+}
+
+write_notes() {
+	touch $NOTES_PATH/stamp
+	for PKG in $PACKAGES_WITH_NOTES ; do
+		PAGE=$NOTES_PATH/${PKG}_note.html
+		create_pkg_note $PKG
+	done
+	cd $NOTES_PATH
+	for FILE in *.html ; do
+		PKG_FILE=/var/lib/jenkins/userContent/rb-pkg/${FILE:0:-10}.html
+		# if note was removed...
+		if [ $FILE -ot stamp ] ; then
+			# cleanup old notes
+			rm $FILE
+			# force re-creation of package file if there was a note
+			rm -f ${PKG_FILE}
+		else
+			# ... else re-recreate ${PKG_FILE} if it does not contain a link to the note
+			grep _note.html ${PKG_FILE} > /dev/null 2>&1 || rm -f ${PKG_FILE}
+		fi
+	done
+	rm stamp
+	cd - > /dev/null
+}
+
+parse_notes() {
+	PACKAGES_WITH_NOTES=$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml keys)
+	for PKG in $PACKAGES_WITH_NOTES ; do
+		echo " Package = ${PKG}"
+		for PROPERTY in version issues bugs comments ; do
+			VALUE="$(cat ${PACKAGES_YML} | /srv/jenkins/bin/shyaml get-value ${PKG}.${PROPERTY} )"
+			if [ "$VALUE" != "" ] ; then
+				case $PROPERTY in
+					version)	NOTES_VERSION[${PKG}]=$VALUE
+							echo "    $PROPERTY = $VALUE"
+							;;
+					issues)		NOTES_ISSUES[${PKG}]=$VALUE
+							show_multi_values "$VALUE"
+							;;
+					bugs)		NOTES_BUGS[${PKG}]=$VALUE
+							show_multi_values "$VALUE"
+							;;
+					comments)	NOTES_COMMENTS[${PKG}]=$VALUE
+							show_multi_values "$VALUE"
+							;;
+				esac
+			fi
+		done
+	done
+}
+
+validate_yaml() {
+	VALID_YAML=true
+	set +e
+	cat $1 | /srv/jenkins/bin/shyaml keys > /dev/null 2>&1 || VALID_YAML=false
+	cat $1 | /srv/jenkins/bin/shyaml get-values > /dev/null 2>&1 || VALID_YAML=false
+	set -e
+	echo "$1 is valid yaml: $VALID_YAML"
+}
+
+#
+# end note parsing functions...
+#
+
+#
+# actually validate & parse the notes and then write pages for all notes and issues
+#
+validate_yaml ${ISSUES_YML}
+validate_yaml ${PACKAGES_YML}
+if $VALID_YAML ; then
+	echo "$(date) - processing notes and issues"
+	parse_issues
+	parse_notes
+	echo "$(date) - processing packages with notes"
+	force_package_targets ${PACKAGES_WITH_NOTES}
+	write_issues
+	write_notes
+	process_packages ${PACKAGES_WITH_NOTES}
+else
+	echo "Warning: ${ISSUES_YML} or ${PACKAGES_YML} contains invalid yaml, please fix."
+fi
+
+#
+# write packages with notes page
+#
+VIEW=notes
+PAGE=index_${VIEW}.html
+echo "$(date) - starting to write $PAGE page."
+write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}"
+if $VALID_YAML ; then
+	write_page "<p>Packages which have notes: <code>"
+	PACKAGES_WITH_NOTES=$(echo $PACKAGES_WITH_NOTES | sed -s "s# #\n#g" | sort | xargs echo)
+	link_packages $PACKAGES_WITH_NOTES
+	write_page "</code></p>"
+else
+	write_page "<p style=\"font-size:1.5em; color: red;\">Broken .yaml files in notes.git could not be parsed, please investigate and fix!</p>"
+fi
+write_page "<p style=\"font-size:0.9em;\">Notes are stored in <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\">notes.git</a>.</p>"
+write_page_meta_sign
+write_page_footer
+publish_page
+
+#
+# write page with all issues
+#
+VIEW=issues
+PAGE=index_${VIEW}.html
+echo "$(date) - starting to write $PAGE page."
+write_page_header $VIEW "Overview of ${SPOKENTARGET[$VIEW]}"
+if $VALID_YAML ; then
+	write_page "<table class=\"body\">"
+	ISSUES=$(echo ${ISSUES} | sed -s "s# #\n#g" | sort | xargs echo)
+	for ISSUE in ${ISSUES} ; do
+		write_page "<tr><td><a href=\"$JENKINS_URL/userContent/issues/${ISSUE}_issue.html\">${ISSUE}</a></td></tr>"
+	done
+	write_page "</table>"
+else
+	write_page "<p style=\"font-size:1.5em; color: red;\">Broken .yaml files in notes.git could not be parsed, please investigate and fix!</p>"
+fi
+write_page "<p style=\"font-size:0.9em;\">Notes are stored in <a href=\"https://anonscm.debian.org/cgit/reproducible/notes.git\">notes.git</a>.</p>"
+write_page_footer
+publish_page
+
author	Holger Levsen <holger@layer-acht.org>	2014-10-17 19:40:32 +0200
committer	Holger Levsen <holger@layer-acht.org>	2014-10-17 19:40:32 +0200
commit	044a78e5a985f110f2bd15a9102f673f23997e09 (patch)
tree	97300dae2ba33760b80e215965c869a354ba2440 /bin/reproducible_html_notes.sh
parent	1aa7e0422d9f0a4561aa952ce6a89a1ba9bedd31 (diff)
download	jenkins.debian.net-044a78e5a985f110f2bd15a9102f673f23997e09.tar.xz