summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHolger Levsen <holger@layer-acht.org>2015-01-02 12:28:07 +0100
committerHolger Levsen <holger@layer-acht.org>2015-01-02 12:28:07 +0100
commit16ef4e799db5d599cfc29c7516567c1bfac97ba8 (patch)
treee88e6051885d844aae48d6262967cf48ecc529ad
parent00eac7a6f321310ae50ff14594fcfeb96f6b4594 (diff)
downloadjenkins.debian.net-16ef4e799db5d599cfc29c7516567c1bfac97ba8.tar.xz
reproducible: proof-of-concept for graphing and displaying status of package-sets. currently there are two: required + build-essential. more suggestions welcome.
-rwxr-xr-xbin/reproducible_html_graphs.sh95
1 files changed, 60 insertions, 35 deletions
diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_graphs.sh
index 94f6637b..8d005aef 100755
--- a/bin/reproducible_html_graphs.sh
+++ b/bin/reproducible_html_graphs.sh
@@ -1,6 +1,6 @@
#!/bin/bash
-# Copyright 2014 Holger Levsen <holger@layer-acht.org>
+# Copyright 2014-2015 Holger Levsen <holger@layer-acht.org>
# released under the GPLv=2
DEBUG=false
@@ -10,7 +10,6 @@ common_init "$@"
# common code defining db access
. /srv/jenkins/bin/reproducible_common.sh
-set +x
init_html
gather_stats
@@ -51,8 +50,8 @@ if [ -z $RESULT ] ; then
sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[0]} VALUES (\"$DATE\", \"$SUITE\", $UNTESTED, $GOOD, $BAD, $UGLY, $REST)"
sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)"
sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")"
- # FIXME: we don't do 2 / stats_builds_age.png yet :/ (and do 3 later)
- for i in 0 1 4 5 6 ; do
+ # FIXME: we don't do 2 / stats_builds_age.png yet :/ (and do 3 later) and 6 is special anyway
+ for i in 0 1 4 5 ; do
# force regeneration of the image
touch -d "$DATE 00:00" ${TABLE[$i]}.png
done
@@ -65,28 +64,34 @@ if [ -z $RESULT ] ; then
fi
# FIXME: work in progress: meta package state graphs
-META_PKG="required"
-RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"$META_PKG\"")
-if [ -z $RESULT ] ; then
- META_TOTAL=0
- META_REQUIRED="acl attr base-files base-passwd bash coreutils dash debconf debianutils diffutils dpkg e2fsprogs eglibc findutils gcc-4.7 grep gzip hostname liblocale-gettext-perl libselinux libsepol libtext-charwidth-perl libtext-iconv-perl libtext-wrapi18n-perl lsb mawk ncurses pam perl sed sensible-utils shadow sysvinit tar tzdata util-linux xz-utils zlib"
- META_WHERE=""
- for i in $META_REQUIRED ; do
- if [ -z "$META_WHERE" ] ; then
- META_WHERE="name in ('$i'"
- else
- META_WHERE="$META_WHERE, '$i'"
- fi
- let "META_TOTAL=META_TOTAL+1"
- done
- META_WHERE="$META_WHERE)"
- META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
- META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
- META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)<='$DATE' AND $META_WHERE;")
- META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)<='$DATE' AND $META_WHERE;")
- let "META_UNTESTED=META_TOTAL-META_GOOD-META_BAD-META_UGLY-META-REST"
- sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"$META_PKG\", $META_UNTESTED, $META_GOOD, $META_BAD, $META_UGLY, $META_REST)"
-fi
+META_PKG[1]="required"
+META_PKG[2]="build-essential"
+META_LIST[1]="acl attr base-files base-passwd bash coreutils dash debconf debianutils diffutils dpkg e2fsprogs eglibc findutils gcc-4.7 grep gzip hostname liblocale-gettext-perl libselinux libsepol libtext-charwidth-perl libtext-iconv-perl libtext-wrapi18n-perl lsb mawk ncurses pam perl sed sensible-utils shadow sysvinit tar tzdata util-linux xz-utils zlib"
+META_LIST[2]="binutils build-essential dpkg eglibc gcc-defaults make-dfsg patch perl"
+
+for i in $(seq 1 ${#META_PKG[@]}) ; do
+ RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"${META_PKG[$i]}\"")
+ if [ -z $RESULT ] ; then
+ META_TOTAL=0
+ META_WHERE=""
+ for PKG in ${META_LIST[$i]} ; do
+ if [ -z "$META_WHERE" ] ; then
+ META_WHERE="name in ('$PKG'"
+ else
+ META_WHERE="$META_WHERE, '$PKG'"
+ fi
+ let "META_TOTAL=META_TOTAL+1"
+ done
+ META_WHERE="$META_WHERE)"
+ META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
+ META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)<='$DATE' AND $META_WHERE;")
+ META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)<='$DATE' AND $META_WHERE;")
+ META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)<='$DATE' AND $META_WHERE;")
+ let "META_UNTESTED=META_TOTAL-META_GOOD-META_BAD-META_UGLY-META-REST"
+ sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKG[$i]}\", $META_UNTESTED, $META_GOOD, $META_BAD, $META_UGLY, $META_REST)"
+ fi
+ # FIXME: touch -d to force re-creation is missing...
+done
# query bts
USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness"
@@ -139,30 +144,28 @@ MAINLABEL[2]="Age in days of oldest kind of logfile"
MAINLABEL[3]="Bugs with usertags for user reproducible-builds@lists.alioth.debian.org"
MAINLABEL[4]="Packages which have notes"
MAINLABEL[5]="Identified issues"
-MAINLABEL[6]="Package reproducibility status for $META_PKG packages"
YLABEL[0]="Amount (total)"
YLABEL[1]="Amount (per day)"
YLABEL[2]="Age in days"
YLABEL[3]="Amount of bugs"
YLABEL[4]="Amount of packages"
YLABEL[5]="Amount of issues"
-YLABEL[6]="Amount ($META_PKG packages)"
redo_png() {
echo "${FIELDS[$i]}" > ${TABLE[$i]}.csv
# TABLE[3+4+5] don't have a suite column...
# 6 is special anyway
if [ $i -eq 6 ] ; then
- WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$META_PKG'"
+ WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$2'"
elif [ $i -ne 3 ] && [ $i -ne 4 ] && [ $i -ne 5 ] ; then
WHERE_EXTRA="WHERE suite = '$SUITE'"
else
WHERE_EXTRA=""
fi
sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$i]} from ${TABLE[$i]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$i]}.csv
- /srv/jenkins/bin/make_graph.py ${TABLE[$i]}.csv ${TABLE[$i]}.png ${COLOR[$i]} "${MAINLABEL[$i]}" "${YLABEL[$i]}"
+ /srv/jenkins/bin/make_graph.py ${TABLE[$i]}.csv $1 ${COLOR[$i]} "${MAINLABEL[$i]}" "${YLABEL[$i]}"
rm ${TABLE[$i]}.csv
- mv ${TABLE[$i]}.png /var/lib/jenkins/userContent/
+ mv $1 /var/lib/jenkins/userContent/
}
write_usertag_table() {
@@ -220,10 +223,32 @@ for i in 0 3 4 5 6 1 ; do
if [ "$i" = "3" ] ; then
write_usertag_table
fi
- write_page " <a href=\"/userContent/${TABLE[$i]}.png\"><img src=\"/userContent/${TABLE[$i]}.png\" class=\"graph\" alt=\"${MAINLABEL[$i]}\"></a>"
- # redo pngs once a day
- if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then
- redo_png
+ # FIXME: split this out in html_meta_graphs... really.
+ if [ "$i" = "6" ] ; then
+ # FIXME: THIS IS A MESS
+ for j in $(seq 1 ${#META_PKG[@]}) ; do
+ MAINLABEL[6]="Package reproducibility status for ${META_PKG[$j]} packages"
+ YLABEL[6]="Amount (${META_PKG[$j]} packages)"
+ PNG=${TABLE[$i]}_${META_PKG[$j]}.png
+ write_page " <div>"
+ write_page " <a href=\"/userContent/$PNG\"><img src=\"/userContent/$PNG\" class=\"graph\" alt=\"${MAINLABEL[$i]}\"></a>"
+ write_page " <br />The package set '${META_PKG[$j]}' consists of: "
+ # FIXME: split into good/bad/ugly too
+ force_package_targets ${META_LIST[$j]}
+ link_packages ${META_LIST[$j]}
+ write_page " </div>"
+ # redo pngs once a day
+ if [ ! -f /var/lib/jenkins/userContent/$PNG ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name $PNG) ] ; then
+ # FIXME: call redo_png differently here.. sux
+ redo_png $PNG ${META_PKG[$j]}
+ fi
+ done
+ else
+ write_page " <a href=\"/userContent/${TABLE[$i]}.png\"><img src=\"/userContent/${TABLE[$i]}.png\" class=\"graph\" alt=\"${MAINLABEL[$i]}\"></a>"
+ # redo pngs once a day
+ if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then
+ redo_png ${TABLE[$i]}.png
+ fi
fi
done
write_page "</p>"