$page_title

#!/usr/bin/python3 # -*- coding: utf-8 -*- # # Copyright © 2015 Mattia Rizzolo # Licensed under GPL-2+ # # Depends: python3 # # This is included by all reproducible_*.py scripts, it contains common functions import os import re import sys import sqlite3 import logging import argparse import datetime from string import Template DEBUG = False QUIET = False BIN_PATH = '/srv/jenkins/bin' BASE = '/var/lib/jenkins/userContent' REPRODUCIBLE_JSON = BASE + '/reproducible.json' REPRODUCIBLE_DB = '/var/lib/jenkins/reproducible.db' DBD_URI = '/dbd' NOTES_URI = '/notes' ISSUES_URI = '/issues' RB_PKG_URI = '/rb-pkg' RBUILD_URI = '/rbuild' BUILDINFO_URI = '/buildinfo' DBD_PATH = BASE + DBD_URI NOTES_PATH = BASE + NOTES_URI ISSUES_PATH = BASE + ISSUES_URI RB_PKG_PATH = BASE + RB_PKG_URI RBUILD_PATH = BASE + RBUILD_URI BUILDINFO_PATH = BASE + BUILDINFO_URI REPRODUCIBLE_URL = 'https://reproducible.debian.net' JENKINS_URL = 'https://jenkins.debian.net' parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group() group.add_argument("-d", "--debug", action="store_true") group.add_argument("-q", "--quiet", action="store_true") args = parser.parse_args() log_level = logging.INFO if args.debug or DEBUG: log_level = logging.DEBUG if args.quiet or QUIET: log_level = logging.ERROR log = logging.getLogger(__name__) log.setLevel(log_level) sh = logging.StreamHandler() sh.setFormatter(logging.Formatter('%(levelname)s: %(message)s')) log.addHandler(sh) log.debug("BIN_PATH:\t" + BIN_PATH) log.debug("BASE:\t\t" + BASE) log.debug("DBD_URI:\t\t" + DBD_URI) log.debug("DBD_PATH:\t" + DBD_PATH) log.debug("NOTES_URI:\t" + NOTES_URI) log.debug("ISSUES_URI:\t" + ISSUES_URI) log.debug("NOTES_PATH:\t" + NOTES_PATH) log.debug("ISSUES_PATH:\t" + ISSUES_PATH) log.debug("RB_PKG_URI:\t" + RB_PKG_URI) log.debug("RB_PKG_PATH:\t" + RB_PKG_PATH) log.debug("RBUILD_URI:\t" + RBUILD_URI) log.debug("RBUILD_PATH:\t" + RBUILD_PATH) log.debug("BUILDINFO_URI:\t" + BUILDINFO_URI) log.debug("BUILDINFO_PATH:\t" + BUILDINFO_PATH) log.debug("REPRODUCIBLE_DB:\t" + REPRODUCIBLE_DB) log.debug("REPRODUCIBLE_JSON:\t" + REPRODUCIBLE_JSON) log.debug("JENKINS_URL:\t\t" + JENKINS_URL) log.debug("REPRODUCIBLE_URL:\t" + REPRODUCIBLE_URL) tab = ' ' html_header = Template(""" $page_title """) html_footer = Template("""

There is more information about jenkins.debian.net and about reproducible builds of Debian available elsewhere. Last update: $date. Copyright 2014-2015 Holger Levsen, GPL-2 licensed. The weather icons are public domain and have been taken from the Tango Icon Library.

""" % (JENKINS_URL)) html_head_page = Template((tab*2).join("""

$page_title

$count_total packages have been attempted to be build so far, that's $percent_total% of $amount source packages in Debian sid currently. Out of these, $count_good packages ($percent_good%) could be built reproducible!

""".splitlines(True))) html_foot_page_style_note = Template((tab*2).join("""

A package name displayed with a bold font is an indication that this package has a note. Visited packages are linked in green, those which have not been visited are linked in blue. """.splitlines(True))) html_foot_page_buildinfo_note = Template((tab*2).join("""
A β sign after a package which is unreproducible indicates that a .buildinfo file was generated. And that means the basics for building packages reproducibly are covered.

""".splitlines(True))) url2html = re.compile(r'((mailto\:|((ht|f)tps?)\://|file\:///){1}\S+)') def write_html_page(title, body, destfile, noheader=False, style_note=False, buildinfo_note=False, noendpage=False): now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC') html = '' html += html_header.substitute(page_title=title) if not noheader: html += html_head_page.substitute( page_title=title, count_total=count_total, amount=amount, percent_total=percent_total, count_good=count_good, percent_good=percent_good) html += body if style_note: html += html_foot_page_style_note.substitute() if buildinfo_note: html += html_foot_page_buildinfo_note.substitute() else: html += (tab*2) + '

' if not noendpage: html += html_footer.substitute(date=now) else: html += '\n' os.makedirs(destfile.rsplit('/', 1)[0], exist_ok=True) with open(destfile, 'w') as fd: fd.write(html) def init_conn(): return sqlite3.connect(REPRODUCIBLE_DB) def query_db(query): cursor = conn.cursor() cursor.execute(query) return cursor.fetchall() def join_status_icon(status, package=None, version=None): table = {'reproducible' : 'weather-clear.png', 'FTBFS': 'weather-storm.png', 'FTBR' : 'weather-showers.png', 'FTBR_with_buildinfo': 'weather-showers-scattered.png', '404': 'weather-severe-alert.png', 'not for us': 'weather-few-clouds-night.png', 'not_for_us': 'weather-few-clouds-night.png', 'blacklisted': 'error.png'} if status == 'unreproducible': if not package: log.error('Could not determinate the real state of package None. ' + 'Returning a generic "FTBR"') status = 'FTBR' elif pkg_has_buildinfo(package, version): status = 'FTBR_with_buildinfo' else: status = 'FTBR' log.debug('Linking status ⇔ icon. package: ' + str(package) + ' @ ' + str(version) + ' status: ' + status) try: return (status, table[status]) except KeyError: log.error('Status of package ' + package + ' (' + status + ') not recognized') return (status, '') def strip_epoch(version): """ Stip the epoch out of the version string. Some file (e.g. buildlogs, debs) do not have epoch in their filenames. This recognize a epoch if there is a colon in the second or third character of the version. """ try: if version[1] == ':' or version[2] == ':': return version.split(':', 1)[1] else: return version except IndexError: return version def pkg_has_buildinfo(package, version=False): """ if there is no version specified it will use the version listed in reproducible.db """ if not version: query = 'SELECT version FROM source_packages WHERE name="%s"' % package version = str(query_db(query)[0][0]) buildinfo = BUILDINFO_PATH + '/' + package + '_' + \ strip_epoch(version) + '_amd64.buildinfo' if os.access(buildinfo, os.R_OK): return True else: return False # do the db querying conn = init_conn() amount = int(query_db('SELECT count(name) FROM sources')[0][0]) count_total = int(query_db('SELECT COUNT(name) FROM source_packages')[0][0]) count_good = int(query_db( 'SELECT COUNT(name) FROM source_packages WHERE status="reproducible"')[0][0]) percent_total = round(((count_total/amount)*100), 1) percent_good = round(((count_good/count_total)*100), 1) log.info('Total packages in Sid:\t\t' + str(amount)) log.info('Total tested packages:\t\t' + str(count_total)) log.info('Total reproducible packages:\t' + str(count_good)) log.info('That means that out of the ' + str(percent_total) + '% of ' + 'the Sid tested packages the ' + str(percent_good) + '% are ' + 'reproducible!')