#!/usr/bin/python3 # -*- coding: utf-8 -*- # # Copyright © 2015 Mattia Rizzolo # Based on the reproducible_common.sh by © 2014 Holger Levsen # Licensed under GPL-2 # # Depends: python3 python3-psycopg2 # # This is included by all reproducible_*.py scripts, it contains common functions import os import re import sys import json import errno import atexit import sqlite3 import logging import argparse import psycopg2 import html as HTML from string import Template from subprocess import call from traceback import print_exception from datetime import datetime, timedelta DEBUG = False QUIET = False # tested suites SUITES = ['testing', 'unstable', 'experimental'] # tested architectures ARCHS = ['amd64'] # defaults defaultsuite = 'unstable' defaultarch = 'amd64' BIN_PATH = '/srv/jenkins/bin' BASE = '/var/lib/jenkins/userContent/reproducible' REPRODUCIBLE_JSON = BASE + '/reproducible.json' REPRODUCIBLE_TRACKER_JSON = BASE + '/reproducible-tracker.json' REPRODUCIBLE_DB = '/var/lib/jenkins/reproducible.db' DBD_URI = '/dbd' DBDTXT_URI = '/dbdtxt' LOGS_URI = '/logs' DIFFS_URI = '/logdiffs' NOTES_URI = '/notes' ISSUES_URI = '/issues' RB_PKG_URI = '/rb-pkg' RBUILD_URI = '/rbuild' BUILDINFO_URI = '/buildinfo' DBD_PATH = BASE + DBD_URI DBDTXT_PATH = BASE + DBDTXT_URI LOGS_PATH = BASE + LOGS_URI DIFFS_PATH = BASE + DIFFS_URI NOTES_PATH = BASE + NOTES_URI ISSUES_PATH = BASE + ISSUES_URI RB_PKG_PATH = BASE + RB_PKG_URI RBUILD_PATH = BASE + RBUILD_URI BUILDINFO_PATH = BASE + BUILDINFO_URI REPRODUCIBLE_URL = 'https://reproducible.debian.net' JENKINS_URL = 'https://jenkins.debian.net' parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group() group.add_argument("-d", "--debug", action="store_true") group.add_argument("-q", "--quiet", action="store_true") parser.add_argument("--ignore-missing-files", action="store_true", help="useful for local testing, where you don't have all the build logs, etc..") args, unknown_args = parser.parse_known_args() log_level = logging.INFO if args.debug or DEBUG: DEBUG = True log_level = logging.DEBUG if args.quiet or QUIET: log_level = logging.ERROR log = logging.getLogger(__name__) log.setLevel(log_level) sh = logging.StreamHandler() sh.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) log.addHandler(sh) started_at = datetime.now() log.info('Starting at %s', started_at) log.debug("BIN_PATH:\t" + BIN_PATH) log.debug("BASE:\t\t" + BASE) log.debug("DBD_URI:\t\t" + DBD_URI) log.debug("DBD_PATH:\t" + DBD_PATH) log.debug("DBDTXT_URI:\t" + DBDTXT_URI) log.debug("DBDTXT_PATH:\t" + DBDTXT_PATH) log.debug("LOGS_URI:\t" + LOGS_URI) log.debug("LOGS_PATH:\t" + LOGS_PATH) log.debug("DIFFS_URI:\t" + DIFFS_URI) log.debug("DIFFS_PATH:\t" + DIFFS_PATH) log.debug("NOTES_URI:\t" + NOTES_URI) log.debug("ISSUES_URI:\t" + ISSUES_URI) log.debug("NOTES_PATH:\t" + NOTES_PATH) log.debug("ISSUES_PATH:\t" + ISSUES_PATH) log.debug("RB_PKG_URI:\t" + RB_PKG_URI) log.debug("RB_PKG_PATH:\t" + RB_PKG_PATH) log.debug("RBUILD_URI:\t" + RBUILD_URI) log.debug("RBUILD_PATH:\t" + RBUILD_PATH) log.debug("BUILDINFO_URI:\t" + BUILDINFO_URI) log.debug("BUILDINFO_PATH:\t" + BUILDINFO_PATH) log.debug("REPRODUCIBLE_DB:\t" + REPRODUCIBLE_DB) log.debug("REPRODUCIBLE_JSON:\t" + REPRODUCIBLE_JSON) log.debug("JENKINS_URL:\t\t" + JENKINS_URL) log.debug("REPRODUCIBLE_URL:\t" + REPRODUCIBLE_URL) if args.ignore_missing_files: log.warning("Missing files will be ignored!") tab = ' ' html_header = Template(""" $page_title """) html_footer = Template("""

There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $date. Copyright 2014-2015 Holger Levsen and others, GPL-2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library.

""" % (JENKINS_URL)) html_head_page = Template((tab*2).join("""

$page_title

""".splitlines(True))) html_foot_page_style_note = Template((tab*2).join("""

A package name displayed with a bold font is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue.
A # sign after the name of a package indicates that a bug is filed against it. Likewise, a + sign indicates there is a patch available, a P means a pending bug while # indicates a closed bug. In cases of several bugs, the symbol is repeated.

""".splitlines(True))) url2html = re.compile(r'((mailto\:|((ht|f)tps?)\://|file\:///){1}\S+)') # filter used on the index_FTBFS pages and for the reproducible.json filtered_issues = ( 'bad_handling_of_extra_warnings', 'ftbfs_wdatetime', 'ftbfs_wdatetime_due_to_swig', 'ftbfs_pbuilder_malformed_dsc', 'ftbfs_in_jenkins_setup', 'ftbfs_build_depends_not_available_on_amd64', 'ftbfs_due_to_root_username', 'ftbfs_due_to_virtual_dependencies') filter_query = '' for issue in filtered_issues: if filter_query == '': filter_query = 'n.issues LIKE "%' + issue + '%"' filter_html = '' + issue + '' else: filter_query += ' OR n.issues LIKE "%' + issue + '%"' filter_html += ' or ' + issue + '' @atexit.register def print_time(): log.info('Finished at %s, took: %s', datetime.now(), datetime.now()-started_at) def print_critical_message(msg): print('\n\n\n') try: for line in msg.splitlines(): log.critical(line) except AttributeError: log.critical(msg) print('\n\n\n') class bcolors: BOLD = '\033[1m' UNDERLINE = '\033[4m' RED = '\033[91m' GOOD = '\033[92m' WARN = '\033[93m' + UNDERLINE FAIL = RED + BOLD + UNDERLINE ENDC = '\033[0m' def _gen_links(suite, arch): links = [ ('last_24h', '
  • packages tested in the last 24h
    • '), ('last_48h', '
  • packages tested in the last 48h
    • '), ('all_abc', '
  • all tested packages (sorted alphabetically)
    • '), ('notify', '
  • ⚑
    • '), ('dd-list', '
  • maintainers of unreproducible packages
    • '), ('pkg_sets', '
  • package sets stats
    • ') ] html = '' for link in links: if link[0] == 'pkg_sets' and suite == 'experimental': html += link[1].format(suite=defaultsuite, arch=arch) + '\n' continue html += link[1].format(suite=suite, arch=arch) + '\n' for i in SUITES: # suite links html += '
  • suite: ' + i + '
    • ' return html def write_html_page(title, body, destfile, suite=defaultsuite, arch=defaultarch, noheader=False, style_note=False, noendpage=False): now = datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC') html = '' html += html_header.substitute(page_title=title) if not noheader: links = _gen_links(suite, arch) html += html_head_page.substitute( page_title=title, suite=suite, arch=arch, links=links) html += body if style_note: html += html_foot_page_style_note.substitute() if not noendpage: html += html_footer.substitute(date=now) else: html += '\n' try: os.makedirs(destfile.rsplit('/', 1)[0], exist_ok=True) except OSError as e: if e.errno != errno.EEXIST: # that's 'File exists' error (errno 17) raise with open(destfile, 'w', encoding='UTF-8') as fd: fd.write(html) def start_db_connection(): return sqlite3.connect(REPRODUCIBLE_DB, timeout=60) def query_db(query): cursor = conn_db.cursor() try: cursor.execute(query) except: print_critical_message('Error execting this query:\n' + query) raise conn_db.commit() return cursor.fetchall() def start_udd_connection(): username = "public-udd-mirror" password = "public-udd-mirror" host = "public-udd-mirror.xvm.mit.edu" port = 5432 db = "udd" try: log.debug("Starting connection to the UDD database") conn = psycopg2.connect("dbname=" + db + " user=" + username + " host=" + host + " password=" + password) except: log.error('Erorr connecting to the UDD database replica.' + 'The full error is:') exc_type, exc_value, exc_traceback = sys.exc_info() print_exception(exc_type, exc_value, exc_traceback) log.error('Failing nicely anyway, all queries will return an empty ' + 'response.') return None conn.set_client_encoding('utf8') return conn def query_udd(query): if not conn_udd: log.error('There has been an error connecting to the UDD database. ' + 'Please look for a previous error for more information.') log.error('Failing nicely anyway, returning an empty response.') return [] cursor = conn_udd.cursor() try: cursor.execute(query) except: log.error('The UDD server encountered a issue while executing the ' + 'query. The full error is:') exc_type, exc_value, exc_traceback = sys.exc_info() print_exception(exc_type, exc_value, exc_traceback) log.error('Failing nicely anyway, returning an empty response.') return [] return cursor.fetchall() def package_has_notes(package): # not a really serious check, it'd be better to check the yaml file path = NOTES_PATH + '/' + package + '_note.html' if os.access(path, os.R_OK): return True else: return False def link_package(package, suite, arch, bugs={}): url = RB_PKG_URI + '/' + suite + '/' + arch + '/' + package + '.html' query = 'SELECT n.issues, n.bugs, n.comments ' + \ 'FROM notes AS n JOIN sources AS s ON s.id=n.package_id ' + \ 'WHERE s.name="{pkg}" AND s.suite="{suite}" ' + \ 'AND s.architecture="{arch}"' try: notes = query_db(query.format(pkg=package, suite=suite, arch=arch))[0] except IndexError: # no notes for this package html = '' + package + '' else: title = '' for issue in json.loads(notes[0]): title += issue + '\n' for bug in json.loads(notes[1]): title += '#' + str(bug) + '\n' if notes[2]: title += notes[2] title = HTML.escape(title.strip()) html = '' + package + '' finally: html += get_trailing_icon(package, bugs) + '\n' return html def link_packages(packages, suite, arch): bugs = get_bugs() html = '' for pkg in packages: html += link_package(pkg, suite, arch, bugs) return html def join_status_icon(status, package=None, version=None): table = {'reproducible' : 'weather-clear.png', 'FTBFS': 'weather-storm.png', 'FTBR' : 'weather-showers-scattered.png', '404': 'weather-severe-alert.png', 'depwait': 'weather-snow.png', 'not for us': 'weather-few-clouds-night.png', 'not_for_us': 'weather-few-clouds-night.png', 'untested': 'weather-clear-night.png', 'blacklisted': 'error.png'} if status == 'unreproducible': status = 'FTBR' elif status == 'not for us': status = 'not_for_us' log.debug('Linking status ⇔ icon. package: ' + str(package) + ' @ ' + str(version) + ' status: ' + status) try: return (status, table[status]) except KeyError: log.error('Status of package ' + package + ' (' + status + ') not recognized') return (status, '') def strip_epoch(version): """ Stip the epoch out of the version string. Some file (e.g. buildlogs, debs) do not have epoch in their filenames. """ try: return version.split(':', 1)[1] except IndexError: return version def pkg_has_buildinfo(package, version=False, suite=defaultsuite, arch=defaultarch): """ if there is no version specified it will use the version listed in reproducible.db """ if not version: query = 'SELECT r.version ' + \ 'FROM results AS r JOIN sources AS s ON r.package_id=s.id ' + \ 'WHERE s.name="{}" AND s.suite="{}" AND s.architecture="{}"' query = query.format(package, suite, arch) version = str(query_db(query)[0][0]) buildinfo = BUILDINFO_PATH + '/' + suite + '/' + arch + '/' + package + \ '_' + strip_epoch(version) + '_amd64.buildinfo' if os.access(buildinfo, os.R_OK): return True else: return False def pkg_has_rbuild(package, version=False, suite=defaultsuite, arch=defaultarch): if not version: query = 'SELECT r.version ' + \ 'FROM results AS r JOIN sources AS s ON r.package_id=s.id ' + \ 'WHERE s.name="{}" AND s.suite="{}" AND s.architecture="{}"' query = query.format(package, suite, arch) version = str(query_db(query)[0][0]) rbuild = RBUILD_PATH + '/' + suite + '/' + arch + '/' + package + '_' + \ strip_epoch(version) + '.rbuild.log' if os.access(rbuild, os.R_OK): return (rbuild, os.stat(rbuild).st_size) elif os.access(rbuild+'.gz', os.R_OK): return (rbuild+'.gz', os.stat(rbuild+'.gz').st_size) else: return () def get_bugs(): """ This function returns a dict: { "package_name": { bug1: {patch: True, done: False}, bug2: {patch: False, done: False}, } } """ query = """ SELECT bugs.id, bugs.source, bugs.done, ARRAY_AGG(tags.tag) FROM bugs JOIN bugs_tags ON bugs.id = bugs_tags.id JOIN bugs_usertags ON bugs_tags.id = bugs_usertags.id JOIN sources ON bugs.source=sources.source LEFT JOIN ( SELECT id, tag FROM bugs_tags WHERE tag='patch' OR tag='pending' ) AS tags ON bugs.id = tags.id WHERE bugs_usertags.email = 'reproducible-builds@lists.alioth.debian.org' AND bugs.id NOT IN ( SELECT id FROM bugs_usertags WHERE email = 'reproducible-builds@lists.alioth.debian.org' AND ( bugs_usertags.tag = 'toolchain' OR bugs_usertags.tag = 'infrastructure') ) GROUP BY bugs.id, bugs.source, bugs.done """ # returns a list of tuples [(id, source, done)] global conn_udd if not conn_udd: conn_udd = start_udd_connection() global bugs if bugs: return bugs rows = query_udd(query) log.info("finding out which usertagged bugs have been closed or at least have patches") packages = {} for bug in rows: if bug[1] not in packages: packages[bug[1]] = {} # bug[0] = bug_id, bug[1] = source_name, bug[2] = who_when_done, # bug[3] = tag (patch or pending) packages[bug[1]][bug[0]] = { 'done': False, 'patch': False, 'pending': False } if bug[2]: # if the bug is done packages[bug[1]][bug[0]]['done'] = True if 'patch' in bug[3]: # the bug is patched packages[bug[1]][bug[0]]['patch'] = True if 'pending' in bug[3]: # the bug is pending packages[bug[1]][bug[0]]['pending'] = True log.debug(packages) return packages def get_trailing_icon(package, bugs): html = '' if package in bugs: for bug in bugs[package]: html += '#' elif bugs[package][bug]['pending']: html += 'bug-pending" title="#' + str(bug) + ', pending">P' elif bugs[package][bug]['patch']: html += 'bug-patch" title="#' + str(bug) + ', with patch">+' else: html += 'bug" title="#' + str(bug) + '">#' return html def get_trailing_bug_icon(bug, bugs, package=None): html = '' if not package: for pkg in bugs.keys(): if get_trailing_bug_icon(bug, bugs, pkg): return get_trailing_bug_icon(bug, bugs, pkg) else: try: if bug in bugs[package].keys(): html += '#' elif bugs[package][bug]['pending']: html += 'bug-pending" title="#' + str(bug) + ', pending">P' elif bugs[package][bug]['patch']: html += 'bug-patch" title="#' + str(bug) + ', with patch">+' else: html += 'bug">' html += '' except KeyError: pass return html def irc_msg(msg): kgb = ['kgb-client', '--conf', '/srv/jenkins/kgb/debian-reproducible.conf', '--relay-msg'] kgb.extend(str(msg).strip().split()) call(kgb) # init the databases connections conn_db = start_db_connection() # the local sqlite3 reproducible db # get_bugs() is the only user of this, let it initialize the connection itself, # during it's first call to speed up things when unneeded # also "share" the bugs, to avoid collecting them multiple times per run conn_udd = None bugs = None