diff options
| -rwxr-xr-x | bin/reproducible_common.py | 25 | ||||
| -rwxr-xr-x | bin/reproducible_common.sh | 6 | ||||
| -rwxr-xr-x | bin/reproducible_html_pkg_sets.py | 299 | ||||
| -rwxr-xr-x | bin/reproducible_html_pkg_sets.sh | 219 | ||||
| -rw-r--r-- | bin/templates/basic_page.mustache | 4 | ||||
| -rw-r--r-- | bin/templates/main_navigation.mustache | 4 | ||||
| -rw-r--r-- | bin/templates/pkgset_details.mustache | 17 | ||||
| -rw-r--r-- | bin/templates/pkgset_navigation.mustache | 14 | ||||
| -rw-r--r-- | job-cfg/reproducible.yaml | 1 |
9 files changed, 353 insertions, 236 deletions
diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py index 28b4add4..95b652e3 100755 --- a/bin/reproducible_common.py +++ b/bin/reproducible_common.py @@ -24,8 +24,8 @@ import pystache import psycopg2 import html as HTML from string import Template -from subprocess import call from traceback import print_exception +from subprocess import call, check_call from tempfile import NamedTemporaryFile from datetime import datetime, timedelta @@ -49,6 +49,7 @@ BIN_PATH = '/srv/jenkins/bin' BASE = '/var/lib/jenkins/userContent/reproducible' DEBIAN_BASE = '/var/lib/jenkins/userContent/reproducible/debian' TEMPLATE_PATH = BIN_PATH + '/templates' +PKGSET_DEF_PATH = '/srv/reproducible-results' TEMP_PATH="/tmp/reproducible" REPRODUCIBLE_JSON = BASE + '/reproducible.json' @@ -143,7 +144,7 @@ if args.ignore_missing_files: tab = ' ' # Templates used for creating package pages -renderer = pystache.Renderer(); +renderer = pystache.Renderer() status_icon_link_template = renderer.load_template( TEMPLATE_PATH + '/status_icon_link') default_page_footer_template = renderer.load_template( @@ -260,11 +261,13 @@ def create_main_navigation(page_title, suite, arch, displayed_page=None): 'suite': suite, 'arch': arch, 'project_links_html': renderer.render(project_links_template), - 'experimental': False, 'suite_list': [{'s': s} for s in SUITES], 'arch_list': [{'a': a} for a in ARCHS], 'debian_url': DEBIAN_URL, } + if suite != 'experimental': + # there are not package sets in experimental + context['include_pkgset_link'] = True # this argument controls which of the main page navigation items will be # highlighted. if displayed_page: @@ -289,15 +292,18 @@ def write_html_page(title, body, destfile, suite=defaultsuite, arch=defaultarch, arch=arch, displayed_page=displayed_page, ) - + # Add the 'mainbody' div only if including a header + html += "<div class='mainbody'>" html += body if style_note: html += renderer.render(pkg_legend_template, {}) if not noendpage: html += create_default_page_footer(now) - html += '</body>\n</html>' - else: - html += '</body>\n</html>' + if not noheader: + # If the header was included, we need to end the 'mainbody' div after + # the 'mainbody' content + html += '</div>' + html += '</body>\n</html>' try: os.makedirs(destfile.rsplit('/', 1)[0], exist_ok=True) except OSError as e: @@ -414,8 +420,9 @@ def link_package(package, suite, arch, bugs={}, popcon=None, is_popular=None): return html -def link_packages(packages, suite, arch): - bugs = get_bugs() +def link_packages(packages, suite, arch, bugs=None): + if bugs is None: + bugs = get_bugs() html = '' for pkg in packages: html += link_package(pkg, suite, arch, bugs) diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 00e7add0..202b4f21 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -400,12 +400,6 @@ write_page_footer() { write_page "</body></html>" } -# A mustache equivalent of this function has been created. See TEMPLATE_PATH/pkg_list_key.mustache -write_page_meta_sign() { - write_page "<p style=\"font-size:0.9em;\">A package name displayed with a <span style=\"font-weight: bold;\">bold font</span> is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue.</br>" - write_page "A <code><span class=\"bug\">#</span></code> sign after the name of a package indicates that a bug is filed against it. Likewise, a <code><span class=\"bug-patch\">+</span></code> sign indicates there is a patch available, a <code><span class="bug-pending">P</span></code> means a pending bug while <code><span class=\"bug-done\">#</span></code> indicates a closed bug. In cases of several bugs, the symbol is repeated.</p>" -} - write_variation_table() { write_page "<p style=\"clear:both;\">" if [ "$1" = "fedora-23" ] ; then diff --git a/bin/reproducible_html_pkg_sets.py b/bin/reproducible_html_pkg_sets.py new file mode 100755 index 00000000..8072c730 --- /dev/null +++ b/bin/reproducible_html_pkg_sets.py @@ -0,0 +1,299 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +# +# Copyright © 2016 Valerie Young <spectranaut@riseup.net> +# Based on reproducible_html_pkg_sets.sh, which was +# © 2014-2016 Holger Levsen <holger@layer-acht.org> +# © 2015 Mattia Rizzolo <mattia@debian.org> +# Licensed under GPL-2 +# +# Depends: python3, reproducible_common, time, sqlite3, pystache, csv +# +# Build rb-pkg pages (the pages that describe the package status) + +from reproducible_common import * + +import csv +import time +import sqlite3 +import pystache + +# Templates used for creating package pages +renderer = pystache.Renderer() +pkgset_navigation_template = renderer.load_template( + os.path.join(TEMPLATE_PATH, 'pkgset_navigation')) +pkgset_details_template = renderer.load_template( + os.path.join(TEMPLATE_PATH, 'pkgset_details')) +basic_page_template = renderer.load_template( + os.path.join(TEMPLATE_PATH, 'basic_page')) +pkg_legend_template = renderer.load_template( + os.path.join(TEMPLATE_PATH, 'pkg_symbol_legend')) + +# we only do stats up until yesterday +YESTERDAY = (datetime.now()-timedelta(days=1)).strftime('%Y-%m-%d') + +def gather_meta_stats(suite, arch, pkgset_name): + pkgset_file = os.path.join(PKGSET_DEF_PATH, 'meta_pkgsets-' + suite, + pkgset_name + '.pkgset') + + try: + with open(pkgset_file) as f: + pkgset_list = [s.strip() for s in f.readlines()] + except FileNotFoundError: + log.warning('No meta package set information exists at ' + pkgset_file) + return {} + + if not pkgset_list: + log.warning('No packages listed for package set: ' + pkgset_name) + return {} + + package_where = "s.name in ('" + ("', '").join(pkgset_list) + "')" + root_query = """ + SELECT s.name + FROM results AS r + JOIN sources AS s ON r.package_id=s.id + WHERE s.suite='{suite}' + AND s.architecture='{arch}' + AND date(r.build_date)<='{date}' + AND {package_where} + """.format(suite=suite, arch=arch, date=YESTERDAY, + package_where=package_where) + + stats = {} + good = query_db(root_query + "AND r.status = 'reproducible' " + + "ORDER BY s.name;") + stats['good'] = [t[0] for t in good] + stats['count_good'] = len(stats['good']) + + bad = query_db(root_query + "AND r.status = 'unreproducible'" + + "ORDER BY r.build_date;") + stats['bad'] = [t[0] for t in bad] + stats['count_bad'] = len(stats['bad']) + + ugly = query_db(root_query + "AND r.status = 'FTBFS'" + + "ORDER BY r.build_date;") + stats['ugly'] = [t[0] for t in ugly] + stats['count_ugly'] = len(stats['ugly']) + + rest = query_db(root_query + "AND (r.status != 'FTBFS' AND " + + "r.status != 'unreproducible' AND " + + "r.status != 'reproducible') ORDER BY r.build_date;") + stats['rest'] = [t[0] for t in rest] + stats['count_rest'] = len(stats['rest']) + + stats['count_all'] = (stats['count_good'] + stats['count_bad'] + + stats['count_ugly'] + stats['count_rest']) + stats['count_all'] = stats['count_all'] if stats['count_all'] else 1 + stats['percent_good'] = percent(stats['count_good'], stats['count_all']) + stats['percent_bad'] = percent(stats['count_bad'], stats['count_all']) + stats['percent_ugly'] = percent(stats['count_ugly'], stats['count_all']) + stats['percent_rest'] = percent(stats['count_rest'], stats['count_all']) + return stats + + +def update_stats(suite, arch, stats, pkgset_name): + result = query_db(""" + SELECT datum, meta_pkg, suite + FROM stats_meta_pkg_state + WHERE datum = '{date}' AND suite = '{suite}' + AND architecture = '{arch}' AND meta_pkg = '{name}' + """.format(date=YESTERDAY, suite=suite, arch=arch, name=pkgset_name)) + + # if there is not a result for this day, add one + if not result: + insert = "INSERT INTO stats_meta_pkg_state VALUES ('{date}', " + \ + "'{suite}', '{arch}', '{pkgset_name}', '{count_good}', " + \ + "'{count_bad}', '{count_ugly}', '{count_rest}')" + query_db(insert.format(date=YESTERDAY, suite=suite, arch=arch, + pkgset_name=pkgset_name, count_good=stats['count_good'], + count_bad=stats['count_bad'], count_ugly=stats['count_ugly'], + count_rest=stats['count_rest'])) + log.info("Updating meta pkgset stats for %s in %s/%s on %s.", + pkgset_name, suite, arch, YESTERDAY) + else: + log.info("Stats for meta pkgset %s in %s/%s on %s exist - not updating.", + pkgset_name, suite, arch, YESTERDAY) + + +def create_pkgset_navigation(suite, arch, view=None): + context = { + 'package_sets': [], + 'suite': suite, + 'arch': arch, + } + + for index in range(1, len(META_PKGSET)+1): + pkgset_name = META_PKGSET[index] + pkgset = { + 'class': "active" if pkgset_name == view else "", + 'pkgset_name': pkgset_name, + } + + thumb_file, thumb_href = stats_thumb_file_href(suite, arch, pkgset_name) + # if the graph image doesn't exist, don't include it in the context + if os.access(thumb_file, os.R_OK): + pkgset['thumb'] = thumb_href + context['package_sets'].append(pkgset) + + return renderer.render(pkgset_navigation_template, context) + + +def create_index_page(suite, arch): + title = 'Package sets in %s/%s for Reproducible Builds' % (suite, arch) + now = datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC') + page_context = { + 'main_html': create_pkgset_navigation(suite, arch) + + create_default_page_footer(now), + 'main_navigation_html': create_main_navigation(title, suite, arch, + 'pkg_set'), + } + body = renderer.render(basic_page_template, page_context) + destfile = os.path.join(DEBIAN_BASE, suite, arch, + "index_pkg_sets.html") + write_html_page(title=title, body=body, destfile=destfile, + noheader=True, noendpage=True) + + +def gen_other_arch_context(archs, suite, pkgset_name): + context = [] + page = "pkg_set_" + pkgset_name + ".html" + for arch in archs: + context.append({ + 'arch': arch, + 'link': "/".join([DEBIAN_BASE, suite, arch, page]) + }) + return context + + +def stats_png_file_href(suite, arch, pkgset_name): + return (os.path.join(DEBIAN_BASE, suite, arch, 'stats_meta_pkg_state_' + + pkgset_name + '.png'), + "/".join(["/debian", suite, arch, 'stats_meta_pkg_state_' + + pkgset_name + '.png']) + ) + + +def stats_thumb_file_href(suite, arch, pkgset_name): + return (os.path.join(DEBIAN_BASE, suite, arch, 'stats_meta_pkg_state_' + + pkgset_name + '-thumbnail.png'), + "/".join(["/debian", suite, arch, 'stats_meta_pkg_state_' + + pkgset_name + '-thumbnail.png']) + ) + + +def create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name): + html = "" + html += create_pkgset_navigation(suite, arch, pkgset_name) + pkgset_context = ({ + 'pkgset_name': pkgset_name, + 'suite': suite, + 'arch': arch, + 'pkg_symbol_legend_html': + renderer.render(pkg_legend_template, {}), + }) + + png_file, png_href = stats_png_file_href(suite, arch, pkgset_name) + thumb_file, thumb_href = stats_thumb_file_href(suite, arch, pkgset_name) + yesterday_timestamp = (datetime.now()-timedelta(days=1)).timestamp() + + if ( not os.access(png_file, os.R_OK) or + os.stat(png_file).st_mtime < yesterday_timestamp ): + create_pkgset_graph(png_file, suite, arch, pkgset_name) + check_call(['convert', png_file, '-adaptive-resize', '160x80', + thumb_file]) + + pkgset_context['png'] = png_href + other_archs = [a for a in ARCHS if a != arch] + pkgset_context['other_archs']= \ + gen_other_arch_context(other_archs, suite, pkgset_name) + + pkgset_context['status_details'] = [] + + status_cutename_descriptions = [ + ('unreproducible', 'bad', 'failed to build reproducibly'), + ('FTBFS', 'ugly', 'failed to build from souce'), + ('rest', 'rest', + 'are either blacklisted, not for us or cannot be downloaded'), + ('reproducible', 'good', 'successfully build reproducibly'), + ] + + for (status, cutename, description) in status_cutename_descriptions: + icon_html = '' + if status == 'rest': + for s in ['not_for_us', 'blacklisted', '404']: + s, icon, spokenstatus = get_status_icon(s) + icon_html += gen_status_link_icon(s, None, icon, suite, arch) + else: + status, icon, spokenstatus = get_status_icon(status) + icon_html = gen_status_link_icon(status, None, icon, suite, arch) + + details_context = { + 'icon_html': icon_html, + 'description': description, + 'package_list_html': link_packages(stats[cutename], suite, arch, bugs), + 'status_count': stats["count_" + cutename], + 'status_percent': stats["percent_" + cutename], + } + + if (status in ('reproducible', 'unreproducible') or + stats["count_" + cutename] != 0): + pkgset_context['status_details'].append(details_context) + + html += renderer.render(pkgset_details_template, pkgset_context) + now = datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC') + html += create_default_page_footer(now) + title = 'Reproducible Builds %s package set for %s/%s for' % \ + (pkgset_name, suite, arch) + + body = renderer.render(basic_page_template, { + 'main_html': html, + 'main_navigation_html': create_main_navigation(title, suite, arch, + 'pkg_set'), + }) + page = "pkg_set_" + pkgset_name + ".html" + destfile = os.path.join(DEBIAN_BASE, suite, arch, page) + write_html_page(title=title, body=body, destfile=destfile, + noheader=True, noendpage=True) + + +def create_pkgset_graph(png_file, suite, arch, pkgset_name): + table = "stats_meta_pkg_state" + columns = ["datum", "reproducible", "unreproducible", "FTBFS", "other"] + where = "WHERE suite = '%s' AND architecture = '%s' AND meta_pkg = '%s'" % \ + (suite, arch, pkgset_name) + if arch == 'i386': + # i386 only has pkg sets since later to make nicer graphs + # (date added in commit 7f2525f7) + where += " AND datum >= '2016-05-06'" + query = "SELECT {fields} FROM {table} {where} ORDER BY datum".format( + fields=", ".join(columns), table=table, where=where) + result = query_db(query) + result_rearranged = [dict(zip(columns, row)) for row in result] + + with create_temp_file(mode='w') as f: + csv_tmp_file = f.name + csv_writer = csv.DictWriter(f, columns) + csv_writer.writeheader() + csv_writer.writerows(result_rearranged) + f.flush() + + graph_command = os.path.join(BIN_PATH, "make_graph.py") + main_label = "Reproducibility status for packages in " + suite + \ + " from " + pkgset_name + y_label = "Amount (" + pkgset_name + " packages)" + check_call([graph_command, csv_tmp_file, png_file, '4', main_label, + y_label, '1920', '960']) + + +bugs = get_bugs() +for arch in ARCHS: + for suite in SUITES: + if suite == 'experimental': + continue + create_index_page(suite, arch) + for index in META_PKGSET: + pkgset_name = META_PKGSET[index] + stats = gather_meta_stats(suite, arch, pkgset_name) + if (stats): + update_stats(suite, arch, stats, pkgset_name) + create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name) diff --git a/bin/reproducible_html_pkg_sets.sh b/bin/reproducible_html_pkg_sets.sh deleted file mode 100755 index bc875fa6..00000000 --- a/bin/reproducible_html_pkg_sets.sh +++ /dev/null @@ -1,219 +0,0 @@ -#!/bin/bash - -# Copyright 2014-2016 Holger Levsen <holger@layer-acht.org> -# © 2015 Mattia Rizzolo <mattia@mapreri.org> -# released under the GPLv=2 - -DEBUG=false -. /srv/jenkins/bin/common-functions.sh -common_init "$@" - -# common code defining db access -. /srv/jenkins/bin/reproducible_common.sh - -# -# init some variables -# -# we only do stats up until yesterday... we also could do today too but not update the db yet... -DATE=$(date -d "1 day ago" '+%Y-%m-%d') -FORCE_DATE=$(date -d "3 days ago" '+%Y-%m-%d') - -# variables related to the stats we update -FIELDS[6]="datum, reproducible, unreproducible, FTBFS, other" -COLOR[6]=4 - -# -# gather meta pkg stats -# -gather_meta_stats() { - PKGSET_PATH=/srv/reproducible-results/meta_pkgsets-$SUITE/${META_PKGSET[$1]}.pkgset - if [ -f $PKGSET_PATH ] ; then - META_LIST=$(cat $PKGSET_PATH) - if [ ! -z "$META_LIST" ] ; then - META_WHERE="" - # gather data about all packages we know about - # as a result, unknown packages in the package set - # are silently ignored - set +x - for PKG in $META_LIST ; do - if [ -z "$META_WHERE" ] ; then - META_WHERE="s.name in ('$PKG'" - else - META_WHERE="$META_WHERE, '$PKG'" - fi - done - if "$DEBUG" ; then set -x ; fi - META_WHERE="$META_WHERE)" - else - META_WHERE="name = 'meta-name-does-not-exist'" - fi - COUNT_META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;") - COUNT_META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE;") - COUNT_META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE;") - COUNT_META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT count(r.status) FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE;") - let META_ALL=COUNT_META_GOOD+COUNT_META_BAD+COUNT_META_UGLY+COUNT_META_REST || META_ALL=1 - PERCENT_META_GOOD=$(echo "scale=1 ; ($COUNT_META_GOOD*100/$META_ALL)" | bc) - PERCENT_META_BAD=$(echo "scale=1 ; ($COUNT_META_BAD*100/$META_ALL)" | bc) - PERCENT_META_UGLY=$(echo "scale=1 ; ($COUNT_META_UGLY*100/$META_ALL)" | bc) - PERCENT_META_REST=$(echo "scale=1 ; ($COUNT_META_REST*100/$META_ALL)" | bc) - # order reproducible packages by name, the rest by build_date - META_GOOD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'reproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY s.name;") - META_BAD=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'unreproducible' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") - META_UGLY=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND r.status = 'FTBFS' AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") - META_REST=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT s.name AS NAME FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='$SUITE' AND s.architecture='$ARCH' AND (r.status != 'FTBFS' AND r.status != 'unreproducible' AND r.status != 'reproducible') AND date(r.build_date)<='$DATE' AND $META_WHERE ORDER BY r.build_date;") - else - META_RESULT=false - fi -} - -# -# update meta pkg stats -# -update_meta_pkg_stats() { - for i in $(seq 1 ${#META_PKGSET[@]}) ; do - RESULT=$(sqlite3 -init ${INIT} ${PACKAGES_DB} "SELECT datum,meta_pkg,suite from ${TABLE[6]} WHERE datum = \"$DATE\" AND suite = \"$SUITE\" AND architecture = \"$ARCH\" AND meta_pkg = \"${META_PKGSET[$i]}\"") - if [ -z $RESULT ] ; then - META_RESULT=true - gather_meta_stats $i - if $META_RESULT ; then - sqlite3 -init ${INIT} ${PACKAGES_DB} "INSERT INTO ${TABLE[6]} VALUES (\"$DATE\", \"$SUITE\", \"$ARCH\", \"${META_PKGSET[$i]}\", $COUNT_META_GOOD, $COUNT_META_BAD, $COUNT_META_UGLY, $COUNT_META_REST)" - echo "Updating meta pkg set stats for ${META_PKGSET[$1]} in $SUITE/$ARCH on $DATE." - fi - echo "Touching $SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png..." - touch -d "$FORCE_DATE 00:00 UTC" $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png - fi - done -} - -# -# create pkg set navigation -# -create_pkg_sets_navigation() { - local i - local view=$1 - local CLASS="" - write_page "<ul><li>Tracked package sets in $SUITE/$ARCH: </li>" - write_page "<br />" - for i in $(seq 1 ${#META_PKGSET[@]}) ; do - case $i in - 7|10|18|26) write_page "<br />" - ;; - *) ;; - esac - if [ "$i" = "$view" ] ; then - CLASS=" class=\"active\"" - else - CLASS="" - fi - - if [ -f $DEBIAN_BASE/$SUITE/$ARCH/${TABLE[6]}_${META_PKGSET[$i]}.png ] ; then - THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" - LABEL="Reproducibility status for packages in $SUITE/$ARCH from '${META_PKGSET[$i]}'" - write_page "<li>" - write_page "<a href=\"/debian/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"$CLASS>${META_PKGSET[$i]}</a>" - write_page "<a href=\"/debian/$SUITE/$ARCH/pkg_set_${META_PKGSET[$i]}.html\"><img src=\"/debian/$SUITE/$ARCH/$THUMB\" class=\"setview\" alt=\"$LABEL\" title=\"${META_PKGSET[$i]}\" name=\"${META_PKGSET[$i]}\"></a>" - write_page "</li>" - fi - done - write_page "</ul>" -} - -# -# create pkg sets pages -# -create_pkg_sets_pages() { - # - # create index page - # - VIEW=pkg_sets - PAGE=index_${VIEW}.html - echo "$(date -u) - starting to write $PAGE page." - write_page_header $VIEW "Package sets in $SUITE/$ARCH for Reproducible builds" - create_pkg_sets_navigation index - write_page_footer - publish_page debian/$SUITE/$ARCH - # - # create individual pages for all the sets - # - local i - for i in $(seq 1 ${#META_PKGSET[@]}) ; do - PAGE="pkg_set_${META_PKGSET[$i]}.html" - echo "$(date -u) - starting to write $PAGE page." - write_page_header $VIEW "Reproducible builds ${META_PKGSET[$i]} package set in $SUITE/$ARCH" - create_pkg_sets_navigation $i - write_page "<hr />" - META_RESULT=true - gather_meta_stats $i - if $META_RESULT ; then - MAINLABEL[6]="Reproducibility status for packages in $SUITE from '${META_PKGSET[$i]}'" - YLABEL[6]="Amount (${META_PKGSET[$i]} packages)" - PNG=${TABLE[6]}_${META_PKGSET[$i]}.png - THUMB="${TABLE[6]}_${META_PKGSET[$i]}-thumbnail.png" - # redo pngs once a day - if [ ! -f $DEBIAN_BASE/$SUITE/$ARCH/$PNG ] || [ ! -z $(find $DEBIAN_BASE/$SUITE/$ARCH -maxdepth 1 -mtime +0 -name $PNG) ] ; then - create_png_from_table 6 $SUITE/$ARCH/$PNG ${META_PKGSET[$i]} - convert $DEBIAN_BASE/$SUITE/$ARCH/$PNG -adaptive-resize 160x80 $DEBIAN_BASE/$SUITE/$ARCH/$THUMB - fi - LABEL="package set '${META_PKGSET[$j]}' in $SUITE/$ARCH" - write_page "<p><a href=\"/debian/$SUITE/$ARCH/$PNG\"><img src=\"/debian/$SUITE/$ARCH/$PNG\" class=\"overview\" alt=\"$LABEL\"></a>" - write_page "<br />The package set '${META_PKGSET[$i]}' in $SUITE/$ARCH consists of:" - write_page " (this set on " - for LINKARCH in ${ARCHS} ; do - if [ "$LINKARCH" = "$ARCH" ] ; then - continue - else - write_page "<a href=\"/debian/$SUITE/$LINKARCH/pkg_set_${META_PKGSET[$i]}.html\">$LINKARCH</a> " - fi - done - write_page ") <br /> <br />" - set_icon unreproducible - write_icon - write_page "$COUNT_META_BAD ($PERCENT_META_BAD%) packages failed to build reproducibly:" - link_packages $META_BAD - write_page "<br />" - if [ $COUNT_META_UGLY -gt 0 ] ; then - set_icon FTBFS - write_icon - write_page "$COUNT_META_UGLY ($PERCENT_META_UGLY%) packages failed to build from source:" - link_packages $META_UGLY - write_page "<br />" - fi - if [ $COUNT_META_REST -gt 0 ] ; then - set_icon not_for_us - write_icon - set_icon blacklisted - write_icon - set_icon 404 - write_icon - write_page "$COUNT_META_REST ($PERCENT_META_REST%) packages are either blacklisted, not for us or cannot be downloaded:" - link_packages $META_REST - write_page "<br />" - fi - write_page " <br />" - set_icon reproducible - write_icon - write_page "$COUNT_META_GOOD packages ($PERCENT_META_GOOD%) successfully built reproducibly:" - link_packages $META_GOOD - write_page "<br />" - write_page "</p>" - write_page_meta_sign - fi - write_page_footer - publish_page debian/$SUITE/$ARCH - done -} - -# -# main -# -for ARCH in $ARCHS ; do - for SUITE in $SUITES ; do - if [ "$SUITE" = "experimental" ] ; then - # no pkg sets in experimental - continue - fi - update_meta_pkg_stats - create_pkg_sets_pages - done -done - diff --git a/bin/templates/basic_page.mustache b/bin/templates/basic_page.mustache new file mode 100644 index 00000000..99fee20f --- /dev/null +++ b/bin/templates/basic_page.mustache @@ -0,0 +1,4 @@ +{{{main_navigation_html}}} +<div class="mainbody"> +{{{main_html}}} +</div> diff --git a/bin/templates/main_navigation.mustache b/bin/templates/main_navigation.mustache index 776a1207..6dab1692 100644 --- a/bin/templates/main_navigation.mustache +++ b/bin/templates/main_navigation.mustache @@ -39,13 +39,13 @@ </a> </li> </ul></li> - {{^experimental}} + {{#include_pkgset_link}} <li {{#pkg_set}}class='active'{{/pkg_set}}> <a href="/debian/{{suite}}/{{arch}}/index_pkg_sets.html"> package sets </a> </li> - {{/experimental}} + {{/include_pkgset_link}} <li>Recently tested:<ul class="children"> <li {{#last_24h}}class='active'{{/last_24h}}> <a href="/debian/{{suite}}/{{arch}}/index_last_24h.html"> diff --git a/bin/templates/pkgset_details.mustache b/bin/templates/pkgset_details.mustache new file mode 100644 index 00000000..ac721747 --- /dev/null +++ b/bin/templates/pkgset_details.mustache @@ -0,0 +1,17 @@ +<p> +{{#png}} + <a href="{{png}}"><img src="{{png}}" class="overview" alt="package set {{pkgset_name}} in {{suite}}/{{arch}}"></a>" +{{/png}} + + <br /> + The package set '{{pkgset_name}}' in {{suite}}/{{arch}} consists of: + (see this package set on {{#other_archs}}<a href="{{link}}" title="See this packageset in unstable/{{arch}}" target="_parent">{{arch}}</a> +{{/other_archs}}) + + <br /> + {{#status_details}} + {{{icon_html}}} {{status_count}} ({{status_percent}}%) packages {{description}}: {{{package_list_html}}} + <br /> + {{/status_details}} + {{{pkg_symbol_legend_html}}} +</op> diff --git a/bin/templates/pkgset_navigation.mustache b/bin/templates/pkgset_navigation.mustache new file mode 100644 index 00000000..40d19c18 --- /dev/null +++ b/bin/templates/pkgset_navigation.mustache @@ -0,0 +1,14 @@ +<ul> + <li>Tracked package sets in {{suite}}/{{arch}}: </li> +<br/> +{{#package_sets}} + <li class="{{class}}"> + <a href="/debian/{{suite}}/{{arch}}/pkg_set_{{pkgset_name}}.html">{{pkgset_name}}</a> + {{#thumb}} + <a href="/debian/{{suite}}/{{arch}}/pkg_set_{{pkgset_name}}.html"> + <img src="{{thumb}}" class="setview" alt="Reproducibility status for packages un {{suite}}/{{arch}} from '{{pkg_name}}'" title="{{pkgset_name}}" name="{{pkgset_name}}"> + </a> + {{/thumb}} + </li> +{{/package_sets}} +</ul> diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index 355f027b..4ea6a0d6 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -426,6 +426,7 @@ - 'html_pkg_sets': my_description: 'Generate HTML results (package sets with graphs) for reproducible builds.' my_timed: '0 */4 * * *' + my_shellext: ".py" - 'html_indexes': my_description: 'Generate HTML results (indexes) for reproducible builds.' my_timed: '*/15 * * * *' |