#!/usr/bin/python3 # -*- coding: utf-8 -*- # # Copyright © 2016 Valerie Young # Based on reproducible_html_pkg_sets.sh, which was # © 2014-2016 Holger Levsen # © 2015 Mattia Rizzolo # Licensed under GPL-2 # # Depends: python3, reproducible_common, time, sqlite3, pystache, csv # # Build rb-pkg pages (the pages that describe the package status) from reproducible_common import * import csv import time import sqlite3 import pystache # Templates used for creating package pages renderer = pystache.Renderer() pkgset_navigation_template = renderer.load_template( os.path.join(TEMPLATE_PATH, 'pkgset_navigation')) pkgset_details_template = renderer.load_template( os.path.join(TEMPLATE_PATH, 'pkgset_details')) basic_page_template = renderer.load_template( os.path.join(TEMPLATE_PATH, 'basic_page')) pkg_legend_template = renderer.load_template( os.path.join(TEMPLATE_PATH, 'pkg_symbol_legend')) # we only do stats up until yesterday YESTERDAY = (datetime.now()-timedelta(days=1)).strftime('%Y-%m-%d') def gather_meta_stats(suite, arch, pkgset_name): pkgset_file = os.path.join(PKGSET_DEF_PATH, 'meta_pkgsets-' + suite, pkgset_name + '.pkgset') try: with open(pkgset_file) as f: pkgset_list = [s.strip() for s in f.readlines()] except FileNotFoundError: log.warning('No meta package set information exists at ' + pkgset_file) return {} if not pkgset_list: log.warning('No packages listed for package set: ' + pkgset_name) return {} package_where = "s.name in ('" + ("', '").join(pkgset_list) + "')" root_query = """ SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite='{suite}' AND s.architecture='{arch}' AND date(r.build_date)<='{date}' AND {package_where} """.format(suite=suite, arch=arch, date=YESTERDAY, package_where=package_where) stats = {} good = query_db(root_query + "AND r.status = 'reproducible' " + "ORDER BY s.name;") stats['good'] = [t[0] for t in good] stats['count_good'] = len(stats['good']) bad = query_db(root_query + "AND r.status = 'unreproducible'" + "ORDER BY r.build_date;") stats['bad'] = [t[0] for t in bad] stats['count_bad'] = len(stats['bad']) ugly = query_db(root_query + "AND r.status = 'FTBFS'" + "ORDER BY r.build_date;") stats['ugly'] = [t[0] for t in ugly] stats['count_ugly'] = len(stats['ugly']) rest = query_db(root_query + "AND (r.status != 'FTBFS' AND " + "r.status != 'unreproducible' AND " + "r.status != 'reproducible') ORDER BY r.build_date;") stats['rest'] = [t[0] for t in rest] stats['count_rest'] = len(stats['rest']) stats['count_all'] = (stats['count_good'] + stats['count_bad'] + stats['count_ugly'] + stats['count_rest']) stats['count_all'] = stats['count_all'] if stats['count_all'] else 1 stats['percent_good'] = percent(stats['count_good'], stats['count_all']) stats['percent_bad'] = percent(stats['count_bad'], stats['count_all']) stats['percent_ugly'] = percent(stats['count_ugly'], stats['count_all']) stats['percent_rest'] = percent(stats['count_rest'], stats['count_all']) return stats def update_stats(suite, arch, stats, pkgset_name): result = query_db(""" SELECT datum, meta_pkg, suite FROM stats_meta_pkg_state WHERE datum = '{date}' AND suite = '{suite}' AND architecture = '{arch}' AND meta_pkg = '{name}' """.format(date=YESTERDAY, suite=suite, arch=arch, name=pkgset_name)) # if there is not a result for this day, add one if not result: insert = "INSERT INTO stats_meta_pkg_state VALUES ('{date}', " + \ "'{suite}', '{arch}', '{pkgset_name}', '{count_good}', " + \ "'{count_bad}', '{count_ugly}', '{count_rest}')" query_db(insert.format(date=YESTERDAY, suite=suite, arch=arch, pkgset_name=pkgset_name, count_good=stats['count_good'], count_bad=stats['count_bad'], count_ugly=stats['count_ugly'], count_rest=stats['count_rest'])) log.info("Updating meta pkgset stats for %s in %s/%s on %s.", pkgset_name, suite, arch, YESTERDAY) else: log.info("Stats for meta pkgset %s in %s/%s on %s exist - not updating.", pkgset_name, suite, arch, YESTERDAY) def create_pkgset_navigation(suite, arch, view=None): context = { 'package_sets': [], 'suite': suite, 'arch': arch, } for index in range(1, len(META_PKGSET)+1): pkgset_name = META_PKGSET[index] pkgset = { 'class': "active" if pkgset_name == view else "", 'pkgset_name': pkgset_name, } thumb_file, thumb_href = stats_thumb_file_href(suite, arch, pkgset_name) # if the graph image doesn't exist, don't include it in the context if os.access(thumb_file, os.R_OK): pkgset['thumb'] = thumb_href context['package_sets'].append(pkgset) return renderer.render(pkgset_navigation_template, context) def create_index_page(suite, arch): title = 'Package sets in %s/%s for Reproducible Builds' % (suite, arch) now = datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC') page_context = { 'main_html': create_pkgset_navigation(suite, arch) + create_default_page_footer(now), 'main_navigation_html': create_main_navigation(title, suite, arch, 'pkg_set'), } body = renderer.render(basic_page_template, page_context) destfile = os.path.join(DEBIAN_BASE, suite, arch, "index_pkg_sets.html") log.info("Creating pkgset index page for %s/%s.", suite, arch) write_html_page(title=title, body=body, destfile=destfile, noheader=True, noendpage=True) def gen_other_arch_context(archs, suite, pkgset_name): context = [] page = "pkg_set_" + pkgset_name + ".html" for arch in archs: context.append({ 'arch': arch, 'link': "/".join([DEBIAN_BASE, suite, arch, page]) }) return context def stats_png_file_href(suite, arch, pkgset_name): return (os.path.join(DEBIAN_BASE, suite, arch, 'stats_meta_pkg_state_' + pkgset_name + '.png'), "/".join(["/debian", suite, arch, 'stats_meta_pkg_state_' + pkgset_name + '.png']) ) def stats_thumb_file_href(suite, arch, pkgset_name): return (os.path.join(DEBIAN_BASE, suite, arch, 'stats_meta_pkg_state_' + pkgset_name + '-thumbnail.png'), "/".join(["/debian", suite, arch, 'stats_meta_pkg_state_' + pkgset_name + '-thumbnail.png']) ) def create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name): html = "" html += create_pkgset_navigation(suite, arch, pkgset_name) pkgset_context = ({ 'pkgset_name': pkgset_name, 'suite': suite, 'arch': arch, 'pkg_symbol_legend_html': renderer.render(pkg_legend_template, {}), }) png_file, png_href = stats_png_file_href(suite, arch, pkgset_name) thumb_file, thumb_href = stats_thumb_file_href(suite, arch, pkgset_name) yesterday_timestamp = (datetime.now()-timedelta(days=1)).timestamp() if ( not os.access(png_file, os.R_OK) or os.stat(png_file).st_mtime < yesterday_timestamp ): create_pkgset_graph(png_file, suite, arch, pkgset_name) check_call(['convert', png_file, '-adaptive-resize', '160x80', thumb_file]) pkgset_context['png'] = png_href other_archs = [a for a in ARCHS if a != arch] pkgset_context['other_archs']= \ gen_other_arch_context(other_archs, suite, pkgset_name) pkgset_context['status_details'] = [] status_cutename_descriptions = [ ('unreproducible', 'bad', 'failed to build reproducibly'), ('FTBFS', 'ugly', 'failed to build from souce'), ('rest', 'rest', 'are either blacklisted, not for us or cannot be downloaded'), ('reproducible', 'good', 'successfully build reproducibly'), ] for (status, cutename, description) in status_cutename_descriptions: icon_html = '' if status == 'rest': for s in ['not_for_us', 'blacklisted', '404']: s, icon, spokenstatus = get_status_icon(s) icon_html += gen_status_link_icon(s, None, icon, suite, arch) else: status, icon, spokenstatus = get_status_icon(status) icon_html = gen_status_link_icon(status, None, icon, suite, arch) details_context = { 'icon_html': icon_html, 'description': description, 'package_list_html': link_packages(stats[cutename], suite, arch, bugs), 'status_count': stats["count_" + cutename], 'status_percent': stats["percent_" + cutename], } if (status in ('reproducible', 'unreproducible') or stats["count_" + cutename] != 0): pkgset_context['status_details'].append(details_context) html += renderer.render(pkgset_details_template, pkgset_context) now = datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC') html += create_default_page_footer(now) title = 'Reproducible Builds %s package set for %s/%s for' % \ (pkgset_name, suite, arch) body = renderer.render(basic_page_template, { 'main_html': html, 'main_navigation_html': create_main_navigation(title, suite, arch, 'pkg_set'), }) page = "pkg_set_" + pkgset_name + ".html" destfile = os.path.join(DEBIAN_BASE, suite, arch, page) log.info("Creating meta pkgset page for %s in %s/%s.", pkgset_name, suite, arch) write_html_page(title=title, body=body, destfile=destfile, noheader=True, noendpage=True) def create_pkgset_graph(png_file, suite, arch, pkgset_name): table = "stats_meta_pkg_state" columns = ["datum", "reproducible", "unreproducible", "FTBFS", "other"] where = "WHERE suite = '%s' AND architecture = '%s' AND meta_pkg = '%s'" % \ (suite, arch, pkgset_name) if arch == 'i386': # i386 only has pkg sets since later to make nicer graphs # (date added in commit 7f2525f7) where += " AND datum >= '2016-05-06'" query = "SELECT {fields} FROM {table} {where} ORDER BY datum".format( fields=", ".join(columns), table=table, where=where) result = query_db(query) result_rearranged = [dict(zip(columns, row)) for row in result] with create_temp_file(mode='w') as f: csv_tmp_file = f.name csv_writer = csv.DictWriter(f, columns) csv_writer.writeheader() csv_writer.writerows(result_rearranged) f.flush() graph_command = os.path.join(BIN_PATH, "make_graph.py") main_label = "Reproducibility status for packages in " + suite + \ " from " + pkgset_name y_label = "Amount (" + pkgset_name + " packages)" log.info("Creating graph for meta pkgset %s in %s/%s.", pkgset_name, suite, arch) check_call([graph_command, csv_tmp_file, png_file, '4', main_label, y_label, '1920', '960']) bugs = get_bugs() for arch in ARCHS: for suite in SUITES: if suite == 'experimental': continue create_index_page(suite, arch) for index in META_PKGSET: pkgset_name = META_PKGSET[index] stats = gather_meta_stats(suite, arch, pkgset_name) if (stats): update_stats(suite, arch, stats, pkgset_name) create_pkgset_page_and_graphs(suite, arch, stats, pkgset_name)