From 3e0dac83b499e1e6b9124659a28bf4324d42110e Mon Sep 17 00:00:00 2001 From: Mattia Rizzolo Date: Thu, 8 Jan 2015 00:21:29 +0100 Subject: reproducible: add reproducible_{common,html_notes}.py. "equivalent" of their homonyms .sh calling reproducible_html_notes.py in the same directory where packages.yml and issues.yml (from the git.debian.org:/git/reproducible/notes.git repo) is enough to fast build all notes html pages for every pacakge listed in the packages.yml file, a page for every issue listed in the issues.yml and a index_issues.html --- bin/reproducible_common.py | 191 ++++++++++++++++++++++++++++++++++ bin/reproducible_html_notes.py | 229 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 420 insertions(+) create mode 100755 bin/reproducible_common.py create mode 100755 bin/reproducible_html_notes.py diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py new file mode 100755 index 00000000..0df8cdaf --- /dev/null +++ b/bin/reproducible_common.py @@ -0,0 +1,191 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# clean-notes: sort and clean the notes stored in notes.git +# Copyright © 2015 Mattia Rizzolo +# Licensed under GPL-2+ +# +# Depends: python3 + +import os +import re +import sys +import sqlite3 +import logging +import argparse +import datetime +from string import Template + +DEBUG = False +QUIET = False + +BIN_PATH = '/srv/jenkins/bin' +BASE = '/var/lib/jenkins' + +REPRODUCIBLE_DB = BASE + '/userContent/reproducible.db' +REPRODUCIBLE_JSON = BASE + '/userContent/reproducible.json' + +NOTES_URI = '/userContent/notes' +ISSUES_URI = '/userContent/issues' +RB_PKG_URI = '/usrtContent/rb-pkg' +NOTES_PATH = BASE + NOTES_URI +ISSUES_PATH = BASE + ISSUES_URI +RB_PKG_PATH = BASE + RB_PKG_URI + +REPRODUCIBLE_URL = 'https://reproducible.debian.net' +JENKINS_URL = 'https://jenkins.debian.net' + +parser = argparse.ArgumentParser() +group = parser.add_mutually_exclusive_group() +group.add_argument("-d", "--debug", action="store_true") +group.add_argument("-q", "--quiet", action="store_true") +args = parser.parse_args() +log_level = logging.INFO +if args.debug or DEBUG: + log_level = logging.DEBUG +if args.quiet or QUIET: + log_level = logging.ERROR +logging.basicConfig(level=log_level) + +logging.debug("BIN_PATH:\t" + BIN_PATH) +logging.debug("BASE:\t" + BASE) +logging.debug("NOTES_URI:\t" + NOTES_URI) +logging.debug("ISSUES_URI:\t" + ISSUES_URI) +logging.debug("NOTES_PATH:\t" + NOTES_PATH) +logging.debug("ISSUES_PATH:\t" + ISSUES_PATH) +logging.debug("RB_PKG_URI:\t" + RB_PKG_URI) +logging.debug("RB_PKG_PATH:\t" + RB_PKG_PATH) +logging.debug("REPRODUCIBLE_DB:\t" + REPRODUCIBLE_DB) +logging.debug("REPRODUCIBLE_JSON:\t" + REPRODUCIBLE_JSON) +logging.debug("JENKINS_URL:\t\t" + JENKINS_URL) +logging.debug("REPRODUCIBLE_URL:\t" + REPRODUCIBLE_URL) + +html_header = Template(""" +\n + + +$page_title\n + +""") +html_footer = Template("""

There is more +information about jenkins.debian.net +and about reproducible +builds of Debian available elsewhere. Last update: $date. +Copyright 2014-%s Holger Levsen, +GPL-2 licensed. The weather icons are public domain and have been taken from +the Tango +Icon Library.

+""" % (JENKINS_URL, datetime.datetime.now().strftime('%Y'))) + +html_head_page = Template("""
+

$page_title

+

$count_total packages have been attempted to be build so far, that's +$percent_total% of $amount source packages in Debian sid +currently. Out of these, $count_good packagea ($percent_good%) +could be built +reproducible!

+
""") + +html_foot_page = Template(""" +

A package name displayed with a bold font is an +indication that this package has a note. Visited packages are linked in green, +those which have not been visited are linked in blue.

""") + + +url2html = re.compile(r'((mailto\:|((ht|f)tps?)\://|file\:///){1}\S+)') + + +def write_html_page(title, body, destfile, noheader=False, nofooter=False): + now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC') + html = '' + html += html_header.substitute(page_title=title) + if not noheader: + html += html_head_page.substitute( + page_title=title, + count_total=count_total, + amount=amount, + percent_total=percent_total, + count_good=count_good, + percent_good=percent_good) + html += body + if not nofooter: + html += html_foot_page.substitute() + html += html_footer.substitute(date=now) + os.makedirs(destfile.rsplit('/', 1)[0], exist_ok=True) + with open(destfile, 'w') as fd: + fd.write(html) + +def init_conn(): + return sqlite3.connect(REPRODUCIBLE_DB) + +def query_db(query): + cursor = conn.cursor() + cursor.execute(query) + return cursor.fetchall() + +# do the db querying +conn = init_conn() +amount = int(query_db('SELECT count(name) FROM sources')[0][0]) +count_total = int(query_db('SELECT COUNT(name) FROM source_packages')[0][0]) +count_good = int(query_db( + 'SELECT COUNT(name) FROM source_packages WHERE status="reproducible"')[0][0]) +percent_total = round(((count_total/amount)*100), 2) +percent_good = round(((count_good/count_total)*100), 2) +logging.info('Total packages in Sid:\t' + str(amount)) +logging.info('Total tested packages:\t' + str(count_total)) +logging.info('Total reproducible packages:\t' + str(count_good)) +logging.info('That means that out of the ' + str(percent_total) + '% of ' + + 'the Sid tested packages the ' + str(percent_good) + '% are ' + + 'reproducible!') + + diff --git a/bin/reproducible_html_notes.py b/bin/reproducible_html_notes.py new file mode 100755 index 00000000..a89e02da --- /dev/null +++ b/bin/reproducible_html_notes.py @@ -0,0 +1,229 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +# clean-notes: sort and clean the notes stored in notes.git +# Copyright © 2015 Mattia Rizzolo +# Licensed under GPL-2+ +# +# Depends: python3 python3-yaml + +import yaml +from reproducible_common import * + +NOTES = 'packages.yml' +ISSUES = 'issues.yml' + +note_html = Template(""" + + + + +$infos + + + + + + +
Version annotated:$version
 
+

Notes are stored in notes.git.

+
""") +note_issues_html = Template("Identified issues:$issues") +note_bugs_html = Template("Bugs noted:$bugs") +note_comments_html = Template("Comments:$comments") + +note_issue_html_url = Template("""URL + $url""") +note_issue_html_desc = Template("""Description + $description""") +note_issue_html = Template(""" + + + +$issue_info +
Identifier:$issue +
+""" % ISSUES_URI) + +issue_html_url = Template("""URL:$url + """) +issue_html = Template(""" + + + + +$urls + + + + + + + + + +
Identifier:$issue
Description:$description
Packages known to be affected by this issue:$affected_pkgs
 
+

Notes are stored in notes.git.

+
""") + + +def load_notes(): + """ + format: + { 'package_name': {'version': '0.0', 'comments'}, 'package_name':{} } + """ + with open(NOTES) as fd: + notes = yaml.load(fd) + logging.debug("notes loaded. There are " + str(len(notes)) + + " package listed") + return notes + + +def load_issues(): + """ + format: + { 'issue_name': {'description': 'blabla', 'url': 'blabla'} } + """ + with open(ISSUES) as fd: + issues = yaml.load(fd) + logging.debug("issues loaded. There are " + str(len(issues)) + + " issues listed") + return issues + + +def fill_issue_in_note(issue): + details = issues[issue] + html = '' + if 'url' in details: + html += note_issue_html_url.substitute(url=details['url']) + if 'description' in details: + desc = details['description'].replace('\n', '
') + html += note_issue_html_desc.substitute(description=desc) + else: + logging.warning("The issue " + issue + " misses a description") + return note_issue_html.substitute(issue=issue, issue_info=html) + + +def gen_html_note(note): + """ + Given a note as input (as a dict: + {"package_name": {"version": "0.0.0", "comments": "blablabla", + "bugs": [111, 222], "issues": ["issue1", "issue2"]}} + ) it returns the html body + """ + infos = '' + # check for issues: + if 'issues' in note: + tmp = '' + for issue in note['issues']: + tmp += fill_issue_in_note(issue) + try: + issues_count[issue].append(note['package']) + except KeyError: + issues_count[issue] = [note['package']] + infos += note_issues_html.substitute(issues=tmp) + # check for bugs: + if 'bugs' in note: + bugurls = '' + for bug in note['bugs']: + bugurls += '' + str(bug) + '
' + infos += note_bugs_html.substitute(bugs=bugurls) + # check for comments: + if 'comments' in note: + comment = note['comments'] + comment = url2html.sub(r'\1', comment) + comment = comment.replace('\n', '
') + infos += note_comments_html.substitute(comments=comment) + try: + return note_html.substitute(version=str(note['version']), infos=infos) + except KeyError: + logging.warning('You should really include a version in the ' + + str(note['package']) + ' note') + return note_html.substitute(version='N/A', infos=infos) + +def gen_html_issue(issue): + """ + Given a issue as input (as a dict: + {"issue_identifier": {"description": "blablabla", "url": "blabla"}} + ) it returns the html body + """ + # check for url: + if 'url' in issues[issue]: + url = issue_html_url.substitute(url=issues[issue]['url']) + else: + url = '' + # add affected packages: + affected = '' + for pkg in sorted(issues_count[issue]): + affected += '%s\n' % ( + RB_PKG_URI, pkg, pkg) + # check for description: + try: + desc = issues[issue]['description'] + except KeyError: + logging.warning('You should really include a description in the ' + + issue + ' issue') + desc = 'N/A' + desc = url2html.sub(r'\1', desc) + desc = desc.replace('\n', '
') + return issue_html.substitute(issue=issue, urls=url, description=desc, + affected_pkgs=affected) + +def iterate_over_notes(notes): + num_notes = str(len(notes)) + i = 0 + for package in sorted(notes): + logging.debug('iterating over notes... ' + str(i) + '/' + num_notes) + note = notes[package] + note['package'] = package + logging.debug('\t' + str(note)) + html = gen_html_note(note) + + title = 'Notes for ' + package + ' - reproducible builds result' + destfile = NOTES_PATH + '/' + package + '_note.html' + write_html_page(title=title, body=html, destfile=destfile, + noheader=True, nofooter=True) + + desturl = REPRODUCIBLE_URL + NOTES_URI + '/' + package + '_note.html' + #logging.info("you can now see your notes at " + desturl) + i = i + 1 + +def iterate_over_issues(issues): + num_issues = str(len(issues)) + i = 0 + for issue in sorted(issues): + logging.debug('iterating over issues... ' + str(i) + '/' + num_issues) + logging.debug('\t' + str(issue)) + html = gen_html_issue(issue) + + title = 'Notes about issue ' + issue + destfile = ISSUES_PATH + '/' + issue + '_issue.html' + write_html_page(title=title, body=html, destfile=destfile) + + desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + issue + '_issue.html' + #logging.info("you can now see the issue at " +desturl) + i = i + 1 + +def index_issues(issues): + html = '' + html += '' + for issue in sorted(issues): + html += '' + html += '
Identified issues
' + issue + '
' + html += '

Notes are stored in notes.git.

' + title = 'Overview of known issues related to reproducible builds' + destfile = BASE + '/userContent/index_issues.html' + desturl = REPRODUCIBLE_URL + '/userContent/index_issues.html' + write_html_page(title=title, body=html, destfile=destfile, nofooter=True) + logging.info('Issues index now available at ' + desturl) + +if __name__ == '__main__': + issues_count = {} + notes = load_notes() + issues = load_issues() + iterate_over_notes(notes) + iterate_over_issues(issues) + index_issues(issues) + -- cgit v1.2.3-70-g09d2