From 16ef4e799db5d599cfc29c7516567c1bfac97ba8 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Fri, 2 Jan 2015 12:28:07 +0100 Subject: reproducible: proof-of-concept for graphing and displaying status of package-sets. currently there are two: required + build-essential. more suggestions welcome. --- bin/reproducible_html_graphs.sh | 95 ++++++++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 35 deletions(-) (limited to 'bin/reproducible_html_graphs.sh') diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_graphs.sh index 94f6637b..8d005aef 100755 --- a/bin/reproducible_html_graphs.sh +++ b/bin/reproducible_html_graphs.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2014 Holger Levsen +# Copyright 2014-2015 Holger Levsen # released under the GPLv=2 DEBUG=false @@ -10,7 +10,6 @@ common_init "$@" # common code defining db access . /srv/jenkins/bin/reproducible_common.sh -set +x init_html gather_stats @@ -51,8 +50,8 @@ if [ -z $RESULT ] ; then sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[0]} VALUES (\"$DATE\", \"$SUITE\", $UNTESTED, $GOOD, $BAD, $UGLY, $REST)" sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[1]} VALUES (\"$DATE\", \"$SUITE\", $GOOAY, $BAAY, $UGLDAY, $RESDAY)" sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[2]} VALUES (\"$DATE\", \"$SUITE\", \"$DIFFG\", \"$DIFFB\", \"$DIFFU\")" - # FIXME: we don't do 2 / stats_builds_age.png yet :/ (and do 3 later) - for i in 0 1 4 5 6 ; do + # FIXME: we don't do 2 / stats_builds_age.png yet :/ (and do 3 later) and 6 is special anyway + for i in 0 1 4 5 ; do # force regeneration of the image touch -d "$DATE 00:00" ${TABLE[$i]}.png done @@ -65,28 +64,34 @@ if [ -z $RESULT ] ; then fi # FIXME: work in progress: meta package state graphs -META_PKG="required" -RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"$META_PKG\"") -if [ -z $RESULT ] ; then - META_TOTAL=0 - META_REQUIRED="acl attr base-files base-passwd bash coreutils dash debconf debianutils diffutils dpkg e2fsprogs eglibc findutils gcc-4.7 grep gzip hostname liblocale-gettext-perl libselinux libsepol libtext-charwidth-perl libtext-iconv-perl libtext-wrapi18n-perl lsb mawk ncurses pam perl sed sensible-utils shadow sysvinit tar tzdata util-linux xz-utils zlib" - META_WHERE="" - for i in $META_REQUIRED ; do - if [ -z "$META_WHERE" ] ; then - META_WHERE="name in ('$i'" - else - META_WHERE="$META_WHERE, '$i'" - fi - let "META_TOTAL=META_TOTAL+1" - done - META_WHERE="$META_WHERE)" - META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)<='$DATE' AND $META_WHERE;") - META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)<='$DATE' AND $META_WHERE;") - META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)<='$DATE' AND $META_WHERE;") - META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)<='$DATE' AND $META_WHERE;") - let "META_UNTESTED=META_TOTAL-META_GOOD-META_BAD-META_UGLY-META-REST" - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"$META_PKG\", $META_UNTESTED, $META_GOOD, $META_BAD, $META_UGLY, $META_REST)" -fi +META_PKG[1]="required" +META_PKG[2]="build-essential" +META_LIST[1]="acl attr base-files base-passwd bash coreutils dash debconf debianutils diffutils dpkg e2fsprogs eglibc findutils gcc-4.7 grep gzip hostname liblocale-gettext-perl libselinux libsepol libtext-charwidth-perl libtext-iconv-perl libtext-wrapi18n-perl lsb mawk ncurses pam perl sed sensible-utils shadow sysvinit tar tzdata util-linux xz-utils zlib" +META_LIST[2]="binutils build-essential dpkg eglibc gcc-defaults make-dfsg patch perl" + +for i in $(seq 1 ${#META_PKG[@]}) ; do + RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND meta_pkg = \"${META_PKG[$i]}\"") + if [ -z $RESULT ] ; then + META_TOTAL=0 + META_WHERE="" + for PKG in ${META_LIST[$i]} ; do + if [ -z "$META_WHERE" ] ; then + META_WHERE="name in ('$PKG'" + else + META_WHERE="$META_WHERE, '$PKG'" + fi + let "META_TOTAL=META_TOTAL+1" + done + META_WHERE="$META_WHERE)" + META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'reproducible' AND date(build_date)<='$DATE' AND $META_WHERE;") + META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'unreproducible' AND date(build_date)<='$DATE' AND $META_WHERE;") + META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE status = 'FTBFS' AND date(build_date)<='$DATE' AND $META_WHERE;") + META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(status) from source_packages WHERE (status != 'FTBFS' AND status != 'FTBFS' AND status != 'unreproducible' AND status != 'reproducible') AND date(build_date)<='$DATE' AND $META_WHERE;") + let "META_UNTESTED=META_TOTAL-META_GOOD-META_BAD-META_UGLY-META-REST" + sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"${META_PKG[$i]}\", $META_UNTESTED, $META_GOOD, $META_BAD, $META_UGLY, $META_REST)" + fi + # FIXME: touch -d to force re-creation is missing... +done # query bts USERTAGS="toolchain infrastructure timestamps fileordering buildpath username hostname uname randomness" @@ -139,30 +144,28 @@ MAINLABEL[2]="Age in days of oldest kind of logfile" MAINLABEL[3]="Bugs with usertags for user reproducible-builds@lists.alioth.debian.org" MAINLABEL[4]="Packages which have notes" MAINLABEL[5]="Identified issues" -MAINLABEL[6]="Package reproducibility status for $META_PKG packages" YLABEL[0]="Amount (total)" YLABEL[1]="Amount (per day)" YLABEL[2]="Age in days" YLABEL[3]="Amount of bugs" YLABEL[4]="Amount of packages" YLABEL[5]="Amount of issues" -YLABEL[6]="Amount ($META_PKG packages)" redo_png() { echo "${FIELDS[$i]}" > ${TABLE[$i]}.csv # TABLE[3+4+5] don't have a suite column... # 6 is special anyway if [ $i -eq 6 ] ; then - WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$META_PKG'" + WHERE_EXTRA="WHERE suite = '$SUITE' and meta_pkg = '$2'" elif [ $i -ne 3 ] && [ $i -ne 4 ] && [ $i -ne 5 ] ; then WHERE_EXTRA="WHERE suite = '$SUITE'" else WHERE_EXTRA="" fi sqlite3 -init ${INIT} -csv ${PACKAGES_DB} "SELECT ${FIELDS[$i]} from ${TABLE[$i]} ${WHERE_EXTRA} ORDER BY datum" >> ${TABLE[$i]}.csv - /srv/jenkins/bin/make_graph.py ${TABLE[$i]}.csv ${TABLE[$i]}.png ${COLOR[$i]} "${MAINLABEL[$i]}" "${YLABEL[$i]}" + /srv/jenkins/bin/make_graph.py ${TABLE[$i]}.csv $1 ${COLOR[$i]} "${MAINLABEL[$i]}" "${YLABEL[$i]}" rm ${TABLE[$i]}.csv - mv ${TABLE[$i]}.png /var/lib/jenkins/userContent/ + mv $1 /var/lib/jenkins/userContent/ } write_usertag_table() { @@ -220,10 +223,32 @@ for i in 0 3 4 5 6 1 ; do if [ "$i" = "3" ] ; then write_usertag_table fi - write_page " \"${MAINLABEL[$i]}\"" - # redo pngs once a day - if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then - redo_png + # FIXME: split this out in html_meta_graphs... really. + if [ "$i" = "6" ] ; then + # FIXME: THIS IS A MESS + for j in $(seq 1 ${#META_PKG[@]}) ; do + MAINLABEL[6]="Package reproducibility status for ${META_PKG[$j]} packages" + YLABEL[6]="Amount (${META_PKG[$j]} packages)" + PNG=${TABLE[$i]}_${META_PKG[$j]}.png + write_page "
" + write_page " \"${MAINLABEL[$i]}\"" + write_page "
The package set '${META_PKG[$j]}' consists of: " + # FIXME: split into good/bad/ugly too + force_package_targets ${META_LIST[$j]} + link_packages ${META_LIST[$j]} + write_page "
" + # redo pngs once a day + if [ ! -f /var/lib/jenkins/userContent/$PNG ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name $PNG) ] ; then + # FIXME: call redo_png differently here.. sux + redo_png $PNG ${META_PKG[$j]} + fi + done + else + write_page " \"${MAINLABEL[$i]}\"" + # redo pngs once a day + if [ ! -f /var/lib/jenkins/userContent/${TABLE[$i]}.png ] || [ -z $(find /var/lib/jenkins/userContent -maxdepth 1 -mtime +0 -name ${TABLE[$i]}.png) ] ; then + redo_png ${TABLE[$i]}.png + fi fi done write_page "

" -- cgit v1.2.3-54-g00ecf