summaryrefslogtreecommitdiffstats
path: root/bin
diff options
context:
space:
mode:
authorHolger Levsen <holger@layer-acht.org>2014-10-17 20:46:14 +0200
committerHolger Levsen <holger@layer-acht.org>2014-10-17 20:46:14 +0200
commit98f55ed3b3c5d25870cca37e4f4e8325a58514f3 (patch)
tree9dc44509f4424c87392e23509a215e4a3c04d953 /bin
parent61fe5629153551e3b38ebc2df56a2a39b2201098 (diff)
downloadjenkins.debian.net-98f55ed3b3c5d25870cca37e4f4e8325a58514f3.tar.xz
reproducible: refactor, split stats script+job in html_indexes and html_graphs.
Diffstat (limited to 'bin')
-rwxr-xr-xbin/reproducible_common.sh19
-rwxr-xr-xbin/reproducible_html_graphs.sh (renamed from bin/reproducible_stats.sh)162
-rwxr-xr-xbin/reproducible_html_indexes.sh165
3 files changed, 180 insertions, 166 deletions
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index 346b8ce4..4808e57d 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -261,15 +261,12 @@ force_package_targets() {
}
link_packages() {
+ STAR=""
for PKG in $@ ; do
- STAR=""
- write_page " ${LINKTARGET[$PKG]}"
if $BUILDINFO_SIGNS ; then
set_package_star
- if [ ! -z "$STAR" ] ; then
- write_page "$STAR "
- fi
fi
+ write_page " ${LINKTARGET[$PKG]} $STAR"
done
}
@@ -351,4 +348,14 @@ process_packages() {
done
}
-
+gather_stats() {
+ COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"unreproducible\"")
+ COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"FTBFS\"")
+ COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"404\"")
+ COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"not for us\"")
+ COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"blacklisted\"")
+ PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc)
+ PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc)
+ PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc)
+ PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc)
+}
diff --git a/bin/reproducible_stats.sh b/bin/reproducible_html_graphs.sh
index 88bd75b5..71d54ee9 100755
--- a/bin/reproducible_stats.sh
+++ b/bin/reproducible_html_graphs.sh
@@ -15,170 +15,12 @@ cp $PACKAGES_DB /var/lib/jenkins/userContent/reproducible.db
set +x
init_html
-
-declare -A GOOD
-declare -A BAD
-declare -A UGLY
-declare -A SOURCELESS
-declare -A NOTFORUS
-LAST24="AND build_date > datetime('now', '-24 hours') "
-LAST48="AND build_date > datetime('now', '-48 hours') "
-GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY build_date DESC" | xargs echo)
-GOOD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo)
-GOOD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo)
-GOOD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY name" | xargs echo)
-BAD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo)
-BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo)
-BAD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo)
-BAD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY name" | xargs echo)
-COUNT_BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"unreproducible\"")
-UGLY["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY build_date DESC" | xargs echo)
-UGLY["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST24 ORDER BY build_date DESC" | xargs echo)
-UGLY["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST48 ORDER BY build_date DESC" | xargs echo)
-UGLY["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY name" | xargs echo)
-COUNT_UGLY=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"FTBFS\"")
-SOURCELESS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY build_date DESC" | xargs echo)
-SOURCELESS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY name" | xargs echo)
-COUNT_SOURCELESS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"404\"" | xargs echo)
-NOTFORUS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY build_date DESC" | xargs echo)
-NOTFORUS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY name" | xargs echo)
-COUNT_NOTFORUS=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"not for us\"" | xargs echo)
-BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"blacklisted\" ORDER BY name" | xargs echo)
-COUNT_BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT COUNT(name) FROM source_packages WHERE status = \"blacklisted\"" | xargs echo)
-PERCENT_BAD=$(echo "scale=1 ; ($COUNT_BAD*100/$COUNT_TOTAL)" | bc)
-PERCENT_UGLY=$(echo "scale=1 ; ($COUNT_UGLY*100/$COUNT_TOTAL)" | bc)
-PERCENT_NOTFORUS=$(echo "scale=1 ; ($COUNT_NOTFORUS*100/$COUNT_TOTAL)" | bc)
-PERCENT_SOURCELESS=$(echo "scale=1 ; ($COUNT_SOURCELESS*100/$COUNT_TOTAL)" | bc)
-
-#
-# actually build the package pages
-#
-echo "$(date) - processing $COUNT_TOTAL packages... this will take a while."
-force_package_targets ${BAD["all"]}
-force_package_targets ${UGLY["all"]} ${GOOD["all"]} ${SOURCELESS["all"]} ${NOTFORUS["all"]} $BLACKLISTED
-
-for VIEW in $ALLVIEWS ; do
- BUILDINFO_SIGNS=true
- PAGE=index_${VIEW}.html
- echo "$(date) - starting to write $PAGE page."
- write_page_header $VIEW "Overview of reproducible builds of ${SPOKENTARGET[$VIEW]}"
- if [ "${VIEW:0:3}" = "all" ] ; then
- FINISH=":"
- else
- SHORTER_SPOKENTARGET=$(echo ${SPOKENTARGET[$VIEW]} | cut -d "(" -f1)
- FINISH=", from $SHORTER_SPOKENTARGET these were:"
- fi
- write_page "<p>"
- set_icon unreproducible with
- write_icon
- set_icon unreproducible
- write_icon
- write_page "$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly in total$FINISH <code>"
- link_packages ${BAD[$VIEW]}
- write_page "</code></p>"
- write_page
- write_page "<p>"
- set_icon FTBFS
- write_icon
- write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source in total$FINISH <code>"
- link_packages ${UGLY[$VIEW]}
- write_page "</code></p>"
- if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_SOURCELESS -gt 0 ] ; then
- write_page "<p>For "
- set_icon 404
- write_icon
- write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages in total sources could not be downloaded: <code>"
- link_packages ${SOURCELESS[$VIEW]}
- write_page "</code></p>"
- fi
- if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_NOTFORUS -gt 0 ] ; then
- write_page "<p>In total there were "
- set_icon not_for_us
- write_icon
- write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64': <code>"
- link_packages ${NOTFORUS[$VIEW]}
- write_page "</code></p>"
- fi
- if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_BLACKLISTED -gt 0 ] ; then
- write_page "<p>"
- set_icon blacklisted
- write_icon
- write_page "$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: <code>"
- link_packages $BLACKLISTED
- write_page "</code></p>"
- fi
- write_page "<p>"
- set_icon reproducible
- write_icon
- write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH <code>"
- BUILDINFO_SIGNS=false
- link_packages ${GOOD[$VIEW]}
- write_page "</code></p>"
- write_page_meta_sign
- write_page_footer
- publish_page
-done
-
-count_packages() {
- COUNT=${#@}
- PERCENT=$(echo "scale=1 ; ($COUNT*100/$COUNT_TOTAL)" | bc)
-}
-
-for STATE in $ALLSTATES ; do
- BUILDINFO_SIGNS=true
- PAGE=index_${STATE}.html
- echo "$(date) - starting to write $PAGE page."
- write_page_header $STATE "Overview of ${SPOKENTARGET[$STATE]}"
- WITH=""
- case "$STATE" in
- reproducible) BUILDINFO_SIGNS=false
- PACKAGES=${GOOD["all"]}
- ;;
- FTBR) CANDIDATES=${BAD["all"]}
- PACKAGES=""
- for PKG in $CANDIDATES ; do
- set_package_star
- if [ "$STAR" = "" ] ; then
- PACKAGES="$PACKAGES $PKG"
- fi
- done
- ;;
- FTBR_with_buildinfo) CANDIDATES=${BAD["all"]}
- PACKAGES=""
- for PKG in $CANDIDATES ; do
- set_package_star
- if [ "$STAR" != "" ] ; then
- PACKAGES="$PACKAGES $PKG"
- fi
- done
- WITH="YES"
- ;;
- FTBFS) PACKAGES=${UGLY["all"]}
- ;;
- 404) PACKAGES=${SOURCELESS["all"]}
- ;;
- not_for_us) PACKAGES=${NOTFORUS["all"]}
- ;;
- blacklisted) PACKAGES=${BLACKLISTED}
- ;;
- esac
- count_packages ${PACKAGES}
- write_page "<p>"
- set_icon $STATE $WITH
- write_icon
- write_page "$COUNT ($PERCENT%) ${SPOKENTARGET[$STATE]}:<code>"
- link_packages ${PACKAGES}
- write_page "</code></p>"
- write_page
- write_page_meta_sign
- write_page_footer
- publish_page
-done
+gather_stats
#
# create stats
#
-# FIXME: we only do stats up until yesterday... we also could do today too but not update the db yet...
+# FIXME?: we only do stats up until yesterday... we also could do today too but not update the db yet...
DATE=$(date -d "1 day ago" '+%Y-%m-%d')
TABLE[0]=stats_pkg_state
TABLE[1]=stats_builds_per_day
diff --git a/bin/reproducible_html_indexes.sh b/bin/reproducible_html_indexes.sh
new file mode 100755
index 00000000..f6ed419f
--- /dev/null
+++ b/bin/reproducible_html_indexes.sh
@@ -0,0 +1,165 @@
+#!/bin/bash
+
+# Copyright 2014 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv=2
+
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code defining db access
+. /srv/jenkins/bin/reproducible_common.sh
+
+set +x
+init_html
+
+declare -A GOOD
+declare -A BAD
+declare -A UGLY
+declare -A SOURCELESS
+declare -A NOTFORUS
+LAST24="AND build_date > datetime('now', '-24 hours') "
+LAST48="AND build_date > datetime('now', '-48 hours') "
+GOOD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY build_date DESC" | xargs echo)
+GOOD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo)
+GOOD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo)
+GOOD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"reproducible\" ORDER BY name" | xargs echo)
+BAD["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY build_date DESC" | xargs echo)
+BAD["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST24 ORDER BY build_date DESC" | xargs echo)
+BAD["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" $LAST48 ORDER BY build_date DESC" | xargs echo)
+BAD["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"unreproducible\" ORDER BY name" | xargs echo)
+UGLY["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY build_date DESC" | xargs echo)
+UGLY["last_24h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST24 ORDER BY build_date DESC" | xargs echo)
+UGLY["last_48h"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" $LAST48 ORDER BY build_date DESC" | xargs echo)
+UGLY["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"FTBFS\" ORDER BY name" | xargs echo)
+SOURCELESS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY build_date DESC" | xargs echo)
+SOURCELESS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"404\" ORDER BY name" | xargs echo)
+NOTFORUS["all"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY build_date DESC" | xargs echo)
+NOTFORUS["all_abc"]=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"not for us\" ORDER BY name" | xargs echo)
+BLACKLISTED=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT name FROM source_packages WHERE status = \"blacklisted\" ORDER BY name" | xargs echo)
+gather_stats
+
+#
+# actually build the package pages
+#
+echo "$(date) - processing $COUNT_TOTAL packages... this will take a while."
+force_package_targets ${BAD["all"]}
+force_package_targets ${UGLY["all"]} ${GOOD["all"]} ${SOURCELESS["all"]} ${NOTFORUS["all"]} $BLACKLISTED
+
+for VIEW in $ALLVIEWS ; do
+ BUILDINFO_SIGNS=true
+ PAGE=index_${VIEW}.html
+ echo "$(date) - starting to write $PAGE page."
+ write_page_header $VIEW "Overview of reproducible builds of ${SPOKENTARGET[$VIEW]}"
+ if [ "${VIEW:0:3}" = "all" ] ; then
+ FINISH=":"
+ else
+ SHORTER_SPOKENTARGET=$(echo ${SPOKENTARGET[$VIEW]} | cut -d "(" -f1)
+ FINISH=", from $SHORTER_SPOKENTARGET these were:"
+ fi
+ write_page "<p>"
+ set_icon unreproducible with
+ write_icon
+ set_icon unreproducible
+ write_icon
+ write_page "$COUNT_BAD packages ($PERCENT_BAD% of $COUNT_TOTAL) failed to built reproducibly in total$FINISH <code>"
+ link_packages ${BAD[$VIEW]}
+ write_page "</code></p>"
+ write_page
+ write_page "<p>"
+ set_icon FTBFS
+ write_icon
+ write_page "$COUNT_UGLY packages ($PERCENT_UGLY%) failed to build from source in total$FINISH <code>"
+ link_packages ${UGLY[$VIEW]}
+ write_page "</code></p>"
+ if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_SOURCELESS -gt 0 ] ; then
+ write_page "<p>For "
+ set_icon 404
+ write_icon
+ write_page "$COUNT_SOURCELESS ($PERCENT_SOURCELESS%) packages in total sources could not be downloaded: <code>"
+ link_packages ${SOURCELESS[$VIEW]}
+ write_page "</code></p>"
+ fi
+ if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_NOTFORUS -gt 0 ] ; then
+ write_page "<p>In total there were "
+ set_icon not_for_us
+ write_icon
+ write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64': <code>"
+ link_packages ${NOTFORUS[$VIEW]}
+ write_page "</code></p>"
+ fi
+ if [ "${VIEW:0:3}" = "all" ] && [ $COUNT_BLACKLISTED -gt 0 ] ; then
+ write_page "<p>"
+ set_icon blacklisted
+ write_icon
+ write_page "$COUNT_BLACKLISTED packages are blacklisted and will never be tested here: <code>"
+ link_packages $BLACKLISTED
+ write_page "</code></p>"
+ fi
+ write_page "<p>"
+ set_icon reproducible
+ write_icon
+ write_page "$COUNT_GOOD packages ($PERCENT_GOOD%) successfully built reproducibly$FINISH <code>"
+ BUILDINFO_SIGNS=false
+ link_packages ${GOOD[$VIEW]}
+ write_page "</code></p>"
+ write_page_meta_sign
+ write_page_footer
+ publish_page
+done
+
+count_packages() {
+ COUNT=${#@}
+ PERCENT=$(echo "scale=1 ; ($COUNT*100/$COUNT_TOTAL)" | bc)
+}
+
+for STATE in $ALLSTATES ; do
+ BUILDINFO_SIGNS=true
+ PAGE=index_${STATE}.html
+ echo "$(date) - starting to write $PAGE page."
+ write_page_header $STATE "Overview of ${SPOKENTARGET[$STATE]}"
+ WITH=""
+ case "$STATE" in
+ reproducible) BUILDINFO_SIGNS=false
+ PACKAGES=${GOOD["all"]}
+ ;;
+ FTBR) CANDIDATES=${BAD["all"]}
+ PACKAGES=""
+ for PKG in $CANDIDATES ; do
+ set_package_star
+ if [ "$STAR" = "" ] ; then
+ PACKAGES="$PACKAGES $PKG"
+ fi
+ done
+ ;;
+ FTBR_with_buildinfo) CANDIDATES=${BAD["all"]}
+ PACKAGES=""
+ for PKG in $CANDIDATES ; do
+ set_package_star
+ if [ "$STAR" != "" ] ; then
+ PACKAGES="$PACKAGES $PKG"
+ fi
+ done
+ WITH="YES"
+ ;;
+ FTBFS) PACKAGES=${UGLY["all"]}
+ ;;
+ 404) PACKAGES=${SOURCELESS["all"]}
+ ;;
+ not_for_us) PACKAGES=${NOTFORUS["all"]}
+ ;;
+ blacklisted) PACKAGES=${BLACKLISTED}
+ ;;
+ esac
+ count_packages ${PACKAGES}
+ write_page "<p>"
+ set_icon $STATE $WITH
+ write_icon
+ write_page "$COUNT ($PERCENT%) ${SPOKENTARGET[$STATE]}:<code>"
+ link_packages ${PACKAGES}
+ write_page "</code></p>"
+ write_page
+ write_page_meta_sign
+ write_page_footer
+ publish_page
+done
+