$page_title

#!/usr/bin/python3 # -*- coding: utf-8 -*- # # Copyright © 2015 Mattia Rizzolo # Based on the reproducible_common.sh by © 2014 Holger Levsen # Licensed under GPL-2 # # Depends: python3 python3-psycopg2 # # This is included by all reproducible_*.py scripts, it contains common functions import os import re import sys import sqlite3 import logging import argparse import datetime import psycopg2 from traceback import print_exception from string import Template DEBUG = False QUIET = False # tested suites SUITES = ['sid'] BIN_PATH = '/srv/jenkins/bin' BASE = '/var/lib/jenkins/userContent' REPRODUCIBLE_JSON = BASE + '/reproducible.json' REPRODUCIBLE_DB = '/var/lib/jenkins/reproducible.db' DBD_URI = '/dbd' NOTES_URI = '/notes' ISSUES_URI = '/issues' RB_PKG_URI = '/rb-pkg' RBUILD_URI = '/rbuild' BUILDINFO_URI = '/buildinfo' DBD_PATH = BASE + DBD_URI NOTES_PATH = BASE + NOTES_URI ISSUES_PATH = BASE + ISSUES_URI RB_PKG_PATH = BASE + RB_PKG_URI RBUILD_PATH = BASE + RBUILD_URI BUILDINFO_PATH = BASE + BUILDINFO_URI REPRODUCIBLE_URL = 'https://reproducible.debian.net' JENKINS_URL = 'https://jenkins.debian.net' parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group() group.add_argument("-d", "--debug", action="store_true") group.add_argument("-q", "--quiet", action="store_true") args = parser.parse_args() log_level = logging.INFO if args.debug or DEBUG: log_level = logging.DEBUG if args.quiet or QUIET: log_level = logging.ERROR log = logging.getLogger(__name__) log.setLevel(log_level) sh = logging.StreamHandler() sh.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) log.addHandler(sh) log.debug("BIN_PATH:\t" + BIN_PATH) log.debug("BASE:\t\t" + BASE) log.debug("DBD_URI:\t\t" + DBD_URI) log.debug("DBD_PATH:\t" + DBD_PATH) log.debug("NOTES_URI:\t" + NOTES_URI) log.debug("ISSUES_URI:\t" + ISSUES_URI) log.debug("NOTES_PATH:\t" + NOTES_PATH) log.debug("ISSUES_PATH:\t" + ISSUES_PATH) log.debug("RB_PKG_URI:\t" + RB_PKG_URI) log.debug("RB_PKG_PATH:\t" + RB_PKG_PATH) log.debug("RBUILD_URI:\t" + RBUILD_URI) log.debug("RBUILD_PATH:\t" + RBUILD_PATH) log.debug("BUILDINFO_URI:\t" + BUILDINFO_URI) log.debug("BUILDINFO_PATH:\t" + BUILDINFO_PATH) log.debug("REPRODUCIBLE_DB:\t" + REPRODUCIBLE_DB) log.debug("REPRODUCIBLE_JSON:\t" + REPRODUCIBLE_JSON) log.debug("JENKINS_URL:\t\t" + JENKINS_URL) log.debug("REPRODUCIBLE_URL:\t" + REPRODUCIBLE_URL) tab = ' ' html_header = Template(""" $page_title """) html_footer = Template("""

There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $date. Copyright 2014-2015 Holger Levsen and others, GPL-2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library.

""" % (JENKINS_URL)) html_head_page = Template((tab*2).join("""

$page_title

$count_total packages have been attempted to be build so far, that's $percent_total% of $amount source packages in Debian sid currently. Out of these, $count_good packages ($percent_good%) could be built reproducible!

""".splitlines(True))) html_foot_page_style_note = Template((tab*2).join("""

A package name displayed with a bold font is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue.
A # sign after the name of a package indicates that a bug is filed against it. Likewise, a + means that there is bug with a patch attached. In case of more than one bug, the symbol is repeated.

""".splitlines(True))) url2html = re.compile(r'((mailto\:|((ht|f)tps?)\://|file\:///){1}\S+)') def print_critical_message(msg): print('\n\n\n') try: for line in msg.splitlines(): log.critical(line) except AttributeError: log.critical(msg) print('\n\n\n') def write_html_page(title, body, destfile, noheader=False, style_note=False, noendpage=False): now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC') html = '' html += html_header.substitute(page_title=title) if not noheader: html += html_head_page.substitute( page_title=title, count_total=count_total, amount=amount, percent_total=percent_total, count_good=count_good, percent_good=percent_good) html += body if style_note: html += html_foot_page_style_note.substitute() if not noendpage: html += html_footer.substitute(date=now) else: html += '\n' os.makedirs(destfile.rsplit('/', 1)[0], exist_ok=True) with open(destfile, 'w') as fd: fd.write(html) def start_db_connection(): return sqlite3.connect(REPRODUCIBLE_DB) def query_db(query): cursor = conn_db.cursor() try: cursor.execute(query) except: print_critical_message('Error execting this query:\n' + query) raise conn_db.commit() return cursor.fetchall() def start_udd_connection(): username = "public-udd-mirror" password = "public-udd-mirror" host = "public-udd-mirror.xvm.mit.edu" port = 5432 db = "udd" try: log.debug("Starting connection to the UDD database") conn = psycopg2.connect("dbname=" + db + " user=" + username + " host=" + host + " password=" + password) except: log.error('Erorr connecting to the UDD database replica.' + 'The full error is:') exc_type, exc_value, exc_traceback = sys.exc_info() print_exception(exc_type, exc_value, exc_traceback) log.error('Failing nicely anyway, all queries will return an empty ' + 'response.') return None conn.set_client_encoding('utf8') return conn def query_udd(query): if not conn_udd: log.error('There has been an error connecting to the UDD database. ' + 'Please look for a previous error for more information.') log.error('Failing nicely anyway, returning an empty response.') return [] cursor = conn_udd.cursor() try: cursor.execute(query) except: log.error('The UDD server encountered a issue while executing the ' + 'query. The full error is:') exc_type, exc_value, exc_traceback = sys.exc_info() print_exception(exc_type, exc_value, exc_traceback) log.error('Failing nicely anyway, returning an empty response.') return [] return cursor.fetchall() def is_virtual_package(package): rows = query_udd("""SELECT source FROM sources WHERE source='%s'""" % package) if len(rows) > 0: return False return True def are_virtual_packages(packages): pkgs = "source='" + "' OR source='".join(packages) + "'" query = 'SELECT source FROM sources WHERE %s' % pkgs rows = query_udd(query) result = {x: False for x in packages if (x,) in rows} result.update({x: True for x in packages if (x,) not in rows}) return result def bug_has_patch(bug): query = """SELECT id FROM bugs_tags WHERE id=%s AND tag='patch'""" % bug if len(query_udd(query)) > 0: return True return False def bugs_have_patches(bugs): ''' This returns a list of tuples where every tuple has a bug with patch ''' bugs = 'id=' + ' OR id='.join(bugs) query = """SELECT id FROM bugs_tags WHERE (%s) AND tag='patch'""" % bugs return query_udd(query) def package_has_notes(package): # not a really serious check, it'd be better to check the yaml file path = NOTES_PATH + '/' + package + '_note.html' if os.access(path, os.R_OK): return True else: return False def join_status_icon(status, package=None, version=None): table = {'reproducible' : 'weather-clear.png', 'FTBFS': 'weather-storm.png', 'FTBR' : 'weather-showers-scattered.png', '404': 'weather-severe-alert.png', 'not for us': 'weather-few-clouds-night.png', 'not_for_us': 'weather-few-clouds-night.png', 'blacklisted': 'error.png'} if status == 'unreproducible': if not package: log.error('Could not determinate the real state of package None. ' + 'Returning a generic "FTBR"') status = 'FTBR' else: status = 'FTBR' log.debug('Linking status ⇔ icon. package: ' + str(package) + ' @ ' + str(version) + ' status: ' + status) try: return (status, table[status]) except KeyError: log.error('Status of package ' + package + ' (' + status + ') not recognized') return (status, '') def strip_epoch(version): """ Stip the epoch out of the version string. Some file (e.g. buildlogs, debs) do not have epoch in their filenames. This recognize a epoch if there is a colon in the second or third character of the version. """ try: if version[1] == ':' or version[2] == ':': return version.split(':', 1)[1] else: return version except IndexError: return version def pkg_has_buildinfo(package, version=False): """ if there is no version specified it will use the version listed in reproducible.db """ if not version: query = 'SELECT version FROM source_packages WHERE name="%s"' % package version = str(query_db(query)[0][0]) buildinfo = BUILDINFO_PATH + '/' + package + '_' + \ strip_epoch(version) + '_amd64.buildinfo' if os.access(buildinfo, os.R_OK): return True else: return False def get_bugs(): """ This function returns a dict: { "package_name": { bug1: {patch: True, done: False}, bug2: {patch: False, done: False}, } } """ query = """ SELECT bugs.id, bugs.source, bugs.done FROM bugs JOIN bugs_tags on bugs.id = bugs_tags.id JOIN bugs_usertags on bugs_tags.id = bugs_usertags.id WHERE bugs_usertags.email = 'reproducible-builds@lists.alioth.debian.org' AND bugs.id NOT IN ( SELECT id FROM bugs_usertags WHERE email = 'reproducible-builds@lists.alioth.debian.org' AND ( bugs_usertags.tag = 'toolchain' OR bugs_usertags.tag = 'infrastructure') ) """ # returns a list of tuples [(id, source, done)] rows = query_udd(query) log.info("finding out which usertagged bugs have been closed or at least have patches") packages = {} bugs = [str(x[0]) for x in rows] bugs_patches = bugs_have_patches(bugs) pkgs = [str(x[1]) for x in rows] pkgs_real = are_virtual_packages(pkgs) for bug in rows: if bug[1] not in packages: packages[bug[1]] = {} # bug[0] = bug_id, bug[1] = source_name, bug[2] = who_when_done if pkgs_real[str(bug[1])]: continue # package is virtual, I don't care about virtual pkgs packages[bug[1]][bug[0]] = {'done': False, 'patch': False} if bug[2]: # if the bug is done packages[bug[1]][bug[0]]['done'] = True try: if (bug[0],) in bugs_patches: packages[bug[1]][bug[0]]['patch'] = True except KeyError: log.error('item: ' + str(bug)) return packages def get_trailing_icon(package, bugs): html = '' if package in bugs: for bug in bugs[package]: html += '#' elif bugs[package][bug]['patch']: html += 'bug-patch" title="#' + str(bug) + ', with patch">+' else: html += '" title="#' + str(bug) + '">#' return html # init the databases connections conn_db = start_db_connection() # the local sqlite3 reproducible db conn_udd = start_udd_connection() # do the db querying try: amount = int(query_db('SELECT count(name) FROM sources')[0][0]) count_total = int(query_db('SELECT COUNT(name) FROM source_packages')[0][0]) count_good = int(query_db('SELECT COUNT(name) FROM source_packages ' + \ 'WHERE status="reproducible"')[0][0]) except sqlite3.OperationalError: log.critical('Error performing basic queries. You have to setup the ' + \ 'database to successfully continue after this point.') log.critical('Continuing anyway, hoping for the best!') amount = 0 count_total = 0 count_good = 0 try: percent_total = round(((count_total/amount)*100), 1) percent_good = round(((count_good/count_total)*100), 1) except ZeroDivisionError: log.error('Looks like there are either no tested package or no packages' + \ ' available at all. Maybe it\'s a new database?') percent_total = 0.0 percent_good = 0.0 log.info('Total packages in Sid:\t\t' + str(amount)) log.info('Total tested packages:\t\t' + str(count_total)) log.info('Total reproducible packages:\t' + str(count_good)) log.info('That means that out of the ' + str(percent_total) + '% of ' + 'the Sid tested packages the ' + str(percent_good) + '% are ' + 'reproducible!')