From 6610b45c60bfe958fb1ce2a351bd4d34fc57cad3 Mon Sep 17 00:00:00 2001 From: Valerie R Young Date: Wed, 22 Jun 2016 17:39:11 -0400 Subject: reproducible debian: rewrite reproducible_html_pkg_sets in python Signed-off-by: Mattia Rizzolo Signed-off-by: Holger Levsen --- bin/reproducible_html_pkg_sets.sh | 219 -------------------------------------- 1 file changed, 219 deletions(-) delete mode 100755 bin/reproducible_html_pkg_sets.sh (limited to 'bin/reproducible_html_pkg_sets.sh') diff --git a/bin/reproducible_html_pkg_sets.sh b/bin/reproducible_html_pkg_sets.sh deleted file mode 100755 index bc875fa6..00000000 --- a/bin/reproducible_html_pkg_sets.sh +++ /dev/null @@ -1,219 +0,0 @@ -#!/bin/bash - -# Copyright 2014-2016 Holger Levsen -# © 2015 Mattia Rizzolo -# released under the GPLv=2 - -DEBUG=false -. /srv/jenkins/bin/common-functions.sh -common_init "$@" - -# common code defining db access -. /srv/jenkins/bin/reproducible_common.sh - -# -# init some variables -# -# we only do stats up until yesterday... we also could do today too but not update the db yet... -DATE=$(date -d "1 day ago" '+%Y-%m-%d') -FORCE_DATE=$(date -d "3 days ago" '+%Y-%m-%d') - -# variables related to the stats we update -FIELDS[6]="datum, reproducible, unreproducible, FTBFS, other" -COLOR[6]=4 - -# -# gather meta pkg stats -# -gather_meta_stats() { - PKGSET_PATH=/srv/reproducible-results/meta_pkgsets-$SUITE/${META_PKGSET[$1]}.pkgset - if [ -f $PKGSET_PATH ] ; then - META_LIST=$(cat $PKGSET_PATH) - if [ ! -z "$META_LIST" ] ; then - META_WHERE="" - # gather data about all packages we know about - # as a result, unknown packages in the package set - # are silently ignored - set +x - for PKG in $META_LIST ; do - if [ -z "$META_WHERE" ] ; then - META_WHERE="s.name in ('$PKG'" - else - META_WHERE="$META_WHERE, '$PKG'" - fi - done - if "$DEBUG" ; then set -x ; fi - META_WHERE="$META_WHERE)" - else - META_WHERE="name = 'meta-name-does-not-exist'" - fi - COUNT_META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;") - COUNT_META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;") - COUNT_META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE;") - COUNT_META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE;") - let META_ALL=COUNT_META_GOOD+COUNT_META_BAD+COUNT_META_UGLY+COUNT_META_REST || META_ALL=1 - PERCENT_META_GOOD=$(echo "scale=1 ; ($COUNT_META_GOOD*100/$META_ALL)" | bc) - PERCENT_META_BAD=$(echo "scale=1 ; ($COUNT_META_BAD*100/$META_ALL)" | bc) - PERCENT_META_UGLY=$(echo "scale=1 ; ($COUNT_META_UGLY*100/$META_ALL)" | bc) - PERCENT_META_REST=$(echo "scale=1 ; ($COUNT_META_REST*100/$META_ALL)" | bc) - # order reproducible packages by name, the rest by build_date - META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY s.name;") - META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") - META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") - META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name AS NAME FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") - else - META_RESULT=false - fi -} - -# -# update meta pkg stats -# -update_meta_pkg_stats() { - for i in $(seq 1 ${#META_PKGSET[@]}) ; do - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND architecture = \"$ARCH\" AND meta_pkg = \"${META_PKGSET[$i]}\"") - if [ -z $RESULT ] ; then - META_RESULT=true - gather_meta_stats $i - if $META_RESULT ; then - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"$ARCH\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)" - echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE/$ARCH on $DATE." - fi - echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..." - touch -d "$FORCE_DATE 00:00 UTC" $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png - fi - done -} - -# -# create pkg set navigation -# -create_pkg_sets_navigation() { - local i - local view=$1 - local CLASS="" - write_page "
  • Tracked package sets in $SUITE/$ARCH:
    • " - write_page "
    " - for i in $(seq 1 ${#META_PKGSET[@]}) ; do - case $i in - 7|10|18|26) write_page "
    " - ;; - *) ;; - esac - if [ "$i" = "$view" ] ; then - CLASS=" class=\"active\"" - else - CLASS="" - fi - - if [ -f $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png ] ; then - THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" - LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" - write_page "
  • " - write_page "${META_PKGSET[$i]}" - write_page "\"$LABEL\"" - write_page "
    • " - fi - done - write_page "
" -} - -# -# create pkg sets pages -# -create_pkg_sets_pages() { - # - # create index page - # - VIEW=pkg_sets - PAGE=index_${VIEW}.html - echo "$(date -u) - starting to write $PAGE page." - write_page_header $VIEW "Package sets in $SUITE/$ARCH for Reproducible builds" - create_pkg_sets_navigation index - write_page_footer - publish_page debian/$SUITE/$ARCH - # - # create individual pages for all the sets - # - local i - for i in $(seq 1 ${#META_PKGSET[@]}) ; do - PAGE="pkg_set_${META_PKGSET[$i]}.html" - echo "$(date -u) - starting to write $PAGE page." - write_page_header $VIEW "Reproducible builds ${META_PKGSET[$i]} package set in $SUITE/$ARCH" - create_pkg_sets_navigation $i - write_page "
" - META_RESULT=true - gather_meta_stats $i - if $META_RESULT ; then - MAINLABEL[6]="Reproducibility status for packages in $SUITE from '${META_PKGSET[$i]}'" - YLABEL[6]="Amount (${META_PKGSET[$i]} packages)" - PNG=${TABLE[6]}_${META_PKGSET[$i]}.png - THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" - # redo pngs once a day - if [ ! -f $DEBIAN_BASE/$SUITE/$ARCH/$PNG ] || [ ! -z $(find $DEBIAN_BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then - create_png_from_table 6 $SUITE/$ARCH/$PNG ${META_PKGSET[$i]} - convert $DEBIAN_BASE/$SUITE/$ARCH/$PNG -adaptive-resize 160x80 $DEBIAN_BASE/$SUITE/$ARCH/$THUMB - fi - LABEL="package set '${META_PKGSET[$j]}' in $SUITE/$ARCH" - write_page "

\"$LABEL\"" - write_page "
The package set '${META_PKGSET[$i]}' in $SUITE/$ARCH consists of:" - write_page " (this set on " - for LINKARCH in ${ARCHS} ; do - if [ "$LINKARCH" = "$ARCH" ] ; then - continue - else - write_page "$LINKARCH " - fi - done - write_page ")
 
" - set_icon unreproducible - write_icon - write_page "$COUNT_META_BAD ($PERCENT_META_BAD%) packages failed to build reproducibly:" - link_packages $META_BAD - write_page "
" - if [ $COUNT_META_UGLY -gt 0 ] ; then - set_icon FTBFS - write_icon - write_page "$COUNT_META_UGLY ($PERCENT_META_UGLY%) packages failed to build from source:" - link_packages $META_UGLY - write_page "
" - fi - if [ $COUNT_META_REST -gt 0 ] ; then - set_icon not_for_us - write_icon - set_icon blacklisted - write_icon - set_icon 404 - write_icon - write_page "$COUNT_META_REST ($PERCENT_META_REST%) packages are either blacklisted, not for us or cannot be downloaded:" - link_packages $META_REST - write_page "
" - fi - write_page " 
" - set_icon reproducible - write_icon - write_page "$COUNT_META_GOOD packages ($PERCENT_META_GOOD%) successfully built reproducibly:" - link_packages $META_GOOD - write_page "
" - write_page "

" - write_page_meta_sign - fi - write_page_footer - publish_page debian/$SUITE/$ARCH - done -} - -# -# main -# -for ARCH in $ARCHS ; do - for SUITE in $SUITES ; do - if [ "$SUITE" = "experimental" ] ; then - # no pkg sets in experimental - continue - fi - update_meta_pkg_stats - create_pkg_sets_pages - done -done - -- cgit v1.2.3-70-g09d2