diff options
| author | Mattia Rizzolo <mattia@mapreri.org> | 2015-01-08 00:21:29 +0100 |
|---|---|---|
| committer | Holger Levsen <holger@layer-acht.org> | 2015-01-08 16:42:15 +0100 |
| commit | 3e0dac83b499e1e6b9124659a28bf4324d42110e (patch) | |
| tree | ae01bbf6fb8f5d8c142d2a1516b8a25abe43cb38 | |
| parent | 9f50f78088a928e2ddeed63e10e5029f1bd81af9 (diff) | |
| download | jenkins.debian.net-3e0dac83b499e1e6b9124659a28bf4324d42110e.tar.xz | |
reproducible: add reproducible_{common,html_notes}.py. "equivalent" of their homonyms .sh
calling reproducible_html_notes.py in the same directory where packages.yml and
issues.yml (from the git.debian.org:/git/reproducible/notes.git repo) is enough
to fast build all notes html pages for every pacakge listed in the packages.yml
file, a page for every issue listed in the issues.yml and a index_issues.html
| -rwxr-xr-x | bin/reproducible_common.py | 191 | ||||
| -rwxr-xr-x | bin/reproducible_html_notes.py | 229 |
2 files changed, 420 insertions, 0 deletions
diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py new file mode 100755 index 00000000..0df8cdaf --- /dev/null +++ b/bin/reproducible_common.py @@ -0,0 +1,191 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# clean-notes: sort and clean the notes stored in notes.git +# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org> +# Licensed under GPL-2+ +# +# Depends: python3 + +import os +import re +import sys +import sqlite3 +import logging +import argparse +import datetime +from string import Template + +DEBUG = False +QUIET = False + +BIN_PATH = '/srv/jenkins/bin' +BASE = '/var/lib/jenkins' + +REPRODUCIBLE_DB = BASE + '/userContent/reproducible.db' +REPRODUCIBLE_JSON = BASE + '/userContent/reproducible.json' + +NOTES_URI = '/userContent/notes' +ISSUES_URI = '/userContent/issues' +RB_PKG_URI = '/usrtContent/rb-pkg' +NOTES_PATH = BASE + NOTES_URI +ISSUES_PATH = BASE + ISSUES_URI +RB_PKG_PATH = BASE + RB_PKG_URI + +REPRODUCIBLE_URL = 'https://reproducible.debian.net' +JENKINS_URL = 'https://jenkins.debian.net' + +parser = argparse.ArgumentParser() +group = parser.add_mutually_exclusive_group() +group.add_argument("-d", "--debug", action="store_true") +group.add_argument("-q", "--quiet", action="store_true") +args = parser.parse_args() +log_level = logging.INFO +if args.debug or DEBUG: + log_level = logging.DEBUG +if args.quiet or QUIET: + log_level = logging.ERROR +logging.basicConfig(level=log_level) + +logging.debug("BIN_PATH:\t" + BIN_PATH) +logging.debug("BASE:\t" + BASE) +logging.debug("NOTES_URI:\t" + NOTES_URI) +logging.debug("ISSUES_URI:\t" + ISSUES_URI) +logging.debug("NOTES_PATH:\t" + NOTES_PATH) +logging.debug("ISSUES_PATH:\t" + ISSUES_PATH) +logging.debug("RB_PKG_URI:\t" + RB_PKG_URI) +logging.debug("RB_PKG_PATH:\t" + RB_PKG_PATH) +logging.debug("REPRODUCIBLE_DB:\t" + REPRODUCIBLE_DB) +logging.debug("REPRODUCIBLE_JSON:\t" + REPRODUCIBLE_JSON) +logging.debug("JENKINS_URL:\t\t" + JENKINS_URL) +logging.debug("REPRODUCIBLE_URL:\t" + REPRODUCIBLE_URL) + +html_header = Template("""<!DOCTYPE html> +<html>\n<head> +<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> +<link href="/userContent/static/style.css" type="text/css" rel="stylesheet" /> +<title>$page_title</title>\n</head> +<body> +""") +html_footer = Template("""<hr /><p style="font-size:0.9em;">There is more +information <a href="%s/userContent/about.html">about jenkins.debian.net</a> +and about <a href="https://wiki.debian.org/ReproducibleBuilds"> reproducible +builds of Debian</a> available elsewhere. Last update: $date. +Copyright 2014-%s <a href="mailto:holger@layer-acht.org">Holger Levsen</a>, +GPL-2 licensed. The weather icons are public domain and have been taken from +the <a href=http://tango.freedesktop.org/Tango_Icon_Library target=_blank>Tango +Icon Library</a>.</p> +</body></html>""" % (JENKINS_URL, datetime.datetime.now().strftime('%Y'))) + +html_head_page = Template("""<header> +<h2>$page_title</h2> +<p>$count_total packages have been attempted to be build so far, that's +$percent_total% of $amount source packages in Debian sid +currently. Out of these, $count_good packagea ($percent_good%) +<a href="https://wiki.debian.org/ReproducibleBuilds">could be built +reproducible!</a></p> +<ul> + <li>Have a look at:</li> + <li> + <a href="/userContent/index_reproducible.html" target="_parent"> + <img src="/userContent/static/weather-clear.png" + alt="reproducible icon" /></a> + </li> + <li> + <a href="/userContent/index_FTBR_with_buildinfo.html" target="_parent"> + <img src="/userContent/static/weather-showers-scattered.png" + alt="FTBR_with_buildinfo icon" /></a> + </li> + <li> + <a href="/userContent/index_FTBR.html" target="_parent"> + <img src="/userContent/static/weather-showers.png" alt="FTBR icon" /> + </a> + </li> + <li> + <a href="/userContent/index_FTBFS.html" target="_parent"> + <img src="/userContent/static/weather-storm.png" alt="FTBFS icon" /> + </a> + </li> + <li> + <a href="/userContent/index_404.html" target="_parent"> + <img src="/userContent/static/weather-severe-alert.png" + alt="404 icon" /></a> + </li> + <li> + <a href="/userContent/index_not_for_us.html" target="_parent"> + <img src="/userContent/static/weather-few-clouds-night.png" + alt="not_for_us icon" /></a> + </li> + <li> + <a href="/userContent/index_blacklisted.html" target="_parent"> + <img src="/userContent/static/error.png" alt="blacklisted icon" /> + </a> + </li> + <li><a href="/userContent/index_issues.html">issues</a></li + <li><li><a href="/userContent/index_notes.html">packages with notes</a></li></li> + <li><a href="/userContent/index_scheduled.html">currently scheduled</a></li> + <li><a href="/userContent/index_last_24h.html">packages tested in the last + 24h</a></li> + <li><a href="/userContent/index_last_48h.html">packages tested in the last + 48h</a></li> + <li><a href="/userContent/index_all_abc.html">all tested packages (sorted + alphabetically)</a></li> + <li><a href="/userContent/index_dd-list.html">maintainers of unreproducible + packages</a></li> + <li><a href="/userContent/index_stats.html">stats</a></li> + <li><a href="/userContent/index_pkg_sets.html">package sets stats</a></li> +</ul></header>""") + +html_foot_page = Template(""" +<p style="font-size:0.9em;">A package name displayed with a bold font is an +indication that this package has a note. Visited packages are linked in green, +those which have not been visited are linked in blue.</p>""") + + +url2html = re.compile(r'((mailto\:|((ht|f)tps?)\://|file\:///){1}\S+)') + + +def write_html_page(title, body, destfile, noheader=False, nofooter=False): + now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC') + html = '' + html += html_header.substitute(page_title=title) + if not noheader: + html += html_head_page.substitute( + page_title=title, + count_total=count_total, + amount=amount, + percent_total=percent_total, + count_good=count_good, + percent_good=percent_good) + html += body + if not nofooter: + html += html_foot_page.substitute() + html += html_footer.substitute(date=now) + os.makedirs(destfile.rsplit('/', 1)[0], exist_ok=True) + with open(destfile, 'w') as fd: + fd.write(html) + +def init_conn(): + return sqlite3.connect(REPRODUCIBLE_DB) + +def query_db(query): + cursor = conn.cursor() + cursor.execute(query) + return cursor.fetchall() + +# do the db querying +conn = init_conn() +amount = int(query_db('SELECT count(name) FROM sources')[0][0]) +count_total = int(query_db('SELECT COUNT(name) FROM source_packages')[0][0]) +count_good = int(query_db( + 'SELECT COUNT(name) FROM source_packages WHERE status="reproducible"')[0][0]) +percent_total = round(((count_total/amount)*100), 2) +percent_good = round(((count_good/count_total)*100), 2) +logging.info('Total packages in Sid:\t' + str(amount)) +logging.info('Total tested packages:\t' + str(count_total)) +logging.info('Total reproducible packages:\t' + str(count_good)) +logging.info('That means that out of the ' + str(percent_total) + '% of ' + + 'the Sid tested packages the ' + str(percent_good) + '% are ' + + 'reproducible!') + + diff --git a/bin/reproducible_html_notes.py b/bin/reproducible_html_notes.py new file mode 100755 index 00000000..a89e02da --- /dev/null +++ b/bin/reproducible_html_notes.py @@ -0,0 +1,229 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# clean-notes: sort and clean the notes stored in notes.git +# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org> +# Licensed under GPL-2+ +# +# Depends: python3 python3-yaml + +import yaml +from reproducible_common import * + +NOTES = 'packages.yml' +ISSUES = 'issues.yml' + +note_html = Template("""<table class="body"> +<tr> + <td>Version annotated:</td> + <td>$version</td> +</tr> +$infos +<tr> + <td colspan="2"> </td> +</tr> +<tr> + <td colspan="2" style="text-align:right; font-size:0.9em;"> + <p>Notes are stored in <a href="https://anonscm.debian.org/cgit/reproducible/notes.git">notes.git</a>.</p> + </td> +</tr> +</table>""") +note_issues_html = Template("<tr><td>Identified issues:</td><td>$issues</td></tr>") +note_bugs_html = Template("<tr><td>Bugs noted:</td><td>$bugs</td></tr>") +note_comments_html = Template("<tr><td>Comments:</td><td>$comments</td></tr>") + +note_issue_html_url = Template("""<tr><td>URL</td> + <td><a href="$url" target="_blank">$url</a></td></tr>""") +note_issue_html_desc = Template("""<tr><td>Description</td> + <td>$description</td></tr>""") +note_issue_html = Template("""<table class="body"> +<tr> + <td>Identifier:</td> + <td><a href="%s/${issue}_issue.html" target="_parent">$issue</a> +</tr> +$issue_info +</table> +""" % ISSUES_URI) + +issue_html_url = Template("""<tr><td>URL:</td><td><a href="$url">$url</a> + </td></tr>""") +issue_html = Template("""<table class="body"> +<tr> + <td>Identifier:</td> + <th>$issue</th> +</tr> +$urls +<tr> + <td>Description:</td> + <td>$description</td> +</tr> +<tr> + <td>Packages known to be affected by this issue:</td> + <td>$affected_pkgs</td> +</tr> +<tr><td colspan="2"> </td></tr> +<tr><td colspan="2" style="text-align:right; font-size:0.9em;"> +<p>Notes are stored in <a href="https://anonscm.debian.org/cgit/reproducible/notes.git">notes.git</a>.</p> +</td></tr></table>""") + + +def load_notes(): + """ + format: + { 'package_name': {'version': '0.0', 'comments'<etc>}, 'package_name':{} } + """ + with open(NOTES) as fd: + notes = yaml.load(fd) + logging.debug("notes loaded. There are " + str(len(notes)) + + " package listed") + return notes + + +def load_issues(): + """ + format: + { 'issue_name': {'description': 'blabla', 'url': 'blabla'} } + """ + with open(ISSUES) as fd: + issues = yaml.load(fd) + logging.debug("issues loaded. There are " + str(len(issues)) + + " issues listed") + return issues + + +def fill_issue_in_note(issue): + details = issues[issue] + html = '' + if 'url' in details: + html += note_issue_html_url.substitute(url=details['url']) + if 'description' in details: + desc = details['description'].replace('\n', '<br />') + html += note_issue_html_desc.substitute(description=desc) + else: + logging.warning("The issue " + issue + " misses a description") + return note_issue_html.substitute(issue=issue, issue_info=html) + + +def gen_html_note(note): + """ + Given a note as input (as a dict: + {"package_name": {"version": "0.0.0", "comments": "blablabla", + "bugs": [111, 222], "issues": ["issue1", "issue2"]}} + ) it returns the html body + """ + infos = '' + # check for issues: + if 'issues' in note: + tmp = '' + for issue in note['issues']: + tmp += fill_issue_in_note(issue) + try: + issues_count[issue].append(note['package']) + except KeyError: + issues_count[issue] = [note['package']] + infos += note_issues_html.substitute(issues=tmp) + # check for bugs: + if 'bugs' in note: + bugurls = '' + for bug in note['bugs']: + bugurls += '<a href="https://bugs.debian.org/' + str(bug) + \ + '">' + str(bug) + '</a><br />' + infos += note_bugs_html.substitute(bugs=bugurls) + # check for comments: + if 'comments' in note: + comment = note['comments'] + comment = url2html.sub(r'<a href="\1">\1</a>', comment) + comment = comment.replace('\n', '<br />') + infos += note_comments_html.substitute(comments=comment) + try: + return note_html.substitute(version=str(note['version']), infos=infos) + except KeyError: + logging.warning('You should really include a version in the ' + + str(note['package']) + ' note') + return note_html.substitute(version='N/A', infos=infos) + +def gen_html_issue(issue): + """ + Given a issue as input (as a dict: + {"issue_identifier": {"description": "blablabla", "url": "blabla"}} + ) it returns the html body + """ + # check for url: + if 'url' in issues[issue]: + url = issue_html_url.substitute(url=issues[issue]['url']) + else: + url = '' + # add affected packages: + affected = '' + for pkg in sorted(issues_count[issue]): + affected += '<a href="%s/%s.html" class="noted">%s</a>\n' % ( + RB_PKG_URI, pkg, pkg) + # check for description: + try: + desc = issues[issue]['description'] + except KeyError: + logging.warning('You should really include a description in the ' + + issue + ' issue') + desc = 'N/A' + desc = url2html.sub(r'<a href="\1">\1</a>', desc) + desc = desc.replace('\n', '<br />') + return issue_html.substitute(issue=issue, urls=url, description=desc, + affected_pkgs=affected) + +def iterate_over_notes(notes): + num_notes = str(len(notes)) + i = 0 + for package in sorted(notes): + logging.debug('iterating over notes... ' + str(i) + '/' + num_notes) + note = notes[package] + note['package'] = package + logging.debug('\t' + str(note)) + html = gen_html_note(note) + + title = 'Notes for ' + package + ' - reproducible builds result' + destfile = NOTES_PATH + '/' + package + '_note.html' + write_html_page(title=title, body=html, destfile=destfile, + noheader=True, nofooter=True) + + desturl = REPRODUCIBLE_URL + NOTES_URI + '/' + package + '_note.html' + #logging.info("you can now see your notes at " + desturl) + i = i + 1 + +def iterate_over_issues(issues): + num_issues = str(len(issues)) + i = 0 + for issue in sorted(issues): + logging.debug('iterating over issues... ' + str(i) + '/' + num_issues) + logging.debug('\t' + str(issue)) + html = gen_html_issue(issue) + + title = 'Notes about issue ' + issue + destfile = ISSUES_PATH + '/' + issue + '_issue.html' + write_html_page(title=title, body=html, destfile=destfile) + + desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + issue + '_issue.html' + #logging.info("you can now see the issue at " +desturl) + i = i + 1 + +def index_issues(issues): + html = '<table class="body">' + html += '<tr><th>Identified issues</th></tr>' + for issue in sorted(issues): + html += '<tr><td><a href="' + ISSUES_URI + '/' + issue + \ + '_issue.html">' + issue + '</a></td></tr>' + html += '</table>' + html += '<p>Notes are stored in <a href="https://anonscm.debian.org/cgit/reproducible/notes.git">notes.git</a>.</p>' + title = 'Overview of known issues related to reproducible builds' + destfile = BASE + '/userContent/index_issues.html' + desturl = REPRODUCIBLE_URL + '/userContent/index_issues.html' + write_html_page(title=title, body=html, destfile=destfile, nofooter=True) + logging.info('Issues index now available at ' + desturl) + +if __name__ == '__main__': + issues_count = {} + notes = load_notes() + issues = load_issues() + iterate_over_notes(notes) + iterate_over_issues(issues) + index_issues(issues) + |