summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMattia Rizzolo <mattia@mapreri.org>2015-01-08 00:21:29 +0100
committerHolger Levsen <holger@layer-acht.org>2015-01-08 16:42:15 +0100
commit3e0dac83b499e1e6b9124659a28bf4324d42110e (patch)
treeae01bbf6fb8f5d8c142d2a1516b8a25abe43cb38
parent9f50f78088a928e2ddeed63e10e5029f1bd81af9 (diff)
downloadjenkins.debian.net-3e0dac83b499e1e6b9124659a28bf4324d42110e.tar.xz
reproducible: add reproducible_{common,html_notes}.py. "equivalent" of their homonyms .sh
calling reproducible_html_notes.py in the same directory where packages.yml and issues.yml (from the git.debian.org:/git/reproducible/notes.git repo) is enough to fast build all notes html pages for every pacakge listed in the packages.yml file, a page for every issue listed in the issues.yml and a index_issues.html
-rwxr-xr-xbin/reproducible_common.py191
-rwxr-xr-xbin/reproducible_html_notes.py229
2 files changed, 420 insertions, 0 deletions
diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py
new file mode 100755
index 00000000..0df8cdaf
--- /dev/null
+++ b/bin/reproducible_common.py
@@ -0,0 +1,191 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+# clean-notes: sort and clean the notes stored in notes.git
+# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# Licensed under GPL-2+
+#
+# Depends: python3
+
+import os
+import re
+import sys
+import sqlite3
+import logging
+import argparse
+import datetime
+from string import Template
+
+DEBUG = False
+QUIET = False
+
+BIN_PATH = '/srv/jenkins/bin'
+BASE = '/var/lib/jenkins'
+
+REPRODUCIBLE_DB = BASE + '/userContent/reproducible.db'
+REPRODUCIBLE_JSON = BASE + '/userContent/reproducible.json'
+
+NOTES_URI = '/userContent/notes'
+ISSUES_URI = '/userContent/issues'
+RB_PKG_URI = '/usrtContent/rb-pkg'
+NOTES_PATH = BASE + NOTES_URI
+ISSUES_PATH = BASE + ISSUES_URI
+RB_PKG_PATH = BASE + RB_PKG_URI
+
+REPRODUCIBLE_URL = 'https://reproducible.debian.net'
+JENKINS_URL = 'https://jenkins.debian.net'
+
+parser = argparse.ArgumentParser()
+group = parser.add_mutually_exclusive_group()
+group.add_argument("-d", "--debug", action="store_true")
+group.add_argument("-q", "--quiet", action="store_true")
+args = parser.parse_args()
+log_level = logging.INFO
+if args.debug or DEBUG:
+ log_level = logging.DEBUG
+if args.quiet or QUIET:
+ log_level = logging.ERROR
+logging.basicConfig(level=log_level)
+
+logging.debug("BIN_PATH:\t" + BIN_PATH)
+logging.debug("BASE:\t" + BASE)
+logging.debug("NOTES_URI:\t" + NOTES_URI)
+logging.debug("ISSUES_URI:\t" + ISSUES_URI)
+logging.debug("NOTES_PATH:\t" + NOTES_PATH)
+logging.debug("ISSUES_PATH:\t" + ISSUES_PATH)
+logging.debug("RB_PKG_URI:\t" + RB_PKG_URI)
+logging.debug("RB_PKG_PATH:\t" + RB_PKG_PATH)
+logging.debug("REPRODUCIBLE_DB:\t" + REPRODUCIBLE_DB)
+logging.debug("REPRODUCIBLE_JSON:\t" + REPRODUCIBLE_JSON)
+logging.debug("JENKINS_URL:\t\t" + JENKINS_URL)
+logging.debug("REPRODUCIBLE_URL:\t" + REPRODUCIBLE_URL)
+
+html_header = Template("""<!DOCTYPE html>
+<html>\n<head>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<link href="/userContent/static/style.css" type="text/css" rel="stylesheet" />
+<title>$page_title</title>\n</head>
+<body>
+""")
+html_footer = Template("""<hr /><p style="font-size:0.9em;">There is more
+information <a href="%s/userContent/about.html">about jenkins.debian.net</a>
+and about <a href="https://wiki.debian.org/ReproducibleBuilds"> reproducible
+builds of Debian</a> available elsewhere. Last update: $date.
+Copyright 2014-%s <a href="mailto:holger@layer-acht.org">Holger Levsen</a>,
+GPL-2 licensed. The weather icons are public domain and have been taken from
+the <a href=http://tango.freedesktop.org/Tango_Icon_Library target=_blank>Tango
+Icon Library</a>.</p>
+</body></html>""" % (JENKINS_URL, datetime.datetime.now().strftime('%Y')))
+
+html_head_page = Template("""<header>
+<h2>$page_title</h2>
+<p>$count_total packages have been attempted to be build so far, that's
+$percent_total% of $amount source packages in Debian sid
+currently. Out of these, $count_good packagea ($percent_good%)
+<a href="https://wiki.debian.org/ReproducibleBuilds">could be built
+reproducible!</a></p>
+<ul>
+ <li>Have a look at:</li>
+ <li>
+ <a href="/userContent/index_reproducible.html" target="_parent">
+ <img src="/userContent/static/weather-clear.png"
+ alt="reproducible icon" /></a>
+ </li>
+ <li>
+ <a href="/userContent/index_FTBR_with_buildinfo.html" target="_parent">
+ <img src="/userContent/static/weather-showers-scattered.png"
+ alt="FTBR_with_buildinfo icon" /></a>
+ </li>
+ <li>
+ <a href="/userContent/index_FTBR.html" target="_parent">
+ <img src="/userContent/static/weather-showers.png" alt="FTBR icon" />
+ </a>
+ </li>
+ <li>
+ <a href="/userContent/index_FTBFS.html" target="_parent">
+ <img src="/userContent/static/weather-storm.png" alt="FTBFS icon" />
+ </a>
+ </li>
+ <li>
+ <a href="/userContent/index_404.html" target="_parent">
+ <img src="/userContent/static/weather-severe-alert.png"
+ alt="404 icon" /></a>
+ </li>
+ <li>
+ <a href="/userContent/index_not_for_us.html" target="_parent">
+ <img src="/userContent/static/weather-few-clouds-night.png"
+ alt="not_for_us icon" /></a>
+ </li>
+ <li>
+ <a href="/userContent/index_blacklisted.html" target="_parent">
+ <img src="/userContent/static/error.png" alt="blacklisted icon" />
+ </a>
+ </li>
+ <li><a href="/userContent/index_issues.html">issues</a></li
+ <li><li><a href="/userContent/index_notes.html">packages with notes</a></li></li>
+ <li><a href="/userContent/index_scheduled.html">currently scheduled</a></li>
+ <li><a href="/userContent/index_last_24h.html">packages tested in the last
+ 24h</a></li>
+ <li><a href="/userContent/index_last_48h.html">packages tested in the last
+ 48h</a></li>
+ <li><a href="/userContent/index_all_abc.html">all tested packages (sorted
+ alphabetically)</a></li>
+ <li><a href="/userContent/index_dd-list.html">maintainers of unreproducible
+ packages</a></li>
+ <li><a href="/userContent/index_stats.html">stats</a></li>
+ <li><a href="/userContent/index_pkg_sets.html">package sets stats</a></li>
+</ul></header>""")
+
+html_foot_page = Template("""
+<p style="font-size:0.9em;">A package name displayed with a bold font is an
+indication that this package has a note. Visited packages are linked in green,
+those which have not been visited are linked in blue.</p>""")
+
+
+url2html = re.compile(r'((mailto\:|((ht|f)tps?)\://|file\:///){1}\S+)')
+
+
+def write_html_page(title, body, destfile, noheader=False, nofooter=False):
+ now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M UTC')
+ html = ''
+ html += html_header.substitute(page_title=title)
+ if not noheader:
+ html += html_head_page.substitute(
+ page_title=title,
+ count_total=count_total,
+ amount=amount,
+ percent_total=percent_total,
+ count_good=count_good,
+ percent_good=percent_good)
+ html += body
+ if not nofooter:
+ html += html_foot_page.substitute()
+ html += html_footer.substitute(date=now)
+ os.makedirs(destfile.rsplit('/', 1)[0], exist_ok=True)
+ with open(destfile, 'w') as fd:
+ fd.write(html)
+
+def init_conn():
+ return sqlite3.connect(REPRODUCIBLE_DB)
+
+def query_db(query):
+ cursor = conn.cursor()
+ cursor.execute(query)
+ return cursor.fetchall()
+
+# do the db querying
+conn = init_conn()
+amount = int(query_db('SELECT count(name) FROM sources')[0][0])
+count_total = int(query_db('SELECT COUNT(name) FROM source_packages')[0][0])
+count_good = int(query_db(
+ 'SELECT COUNT(name) FROM source_packages WHERE status="reproducible"')[0][0])
+percent_total = round(((count_total/amount)*100), 2)
+percent_good = round(((count_good/count_total)*100), 2)
+logging.info('Total packages in Sid:\t' + str(amount))
+logging.info('Total tested packages:\t' + str(count_total))
+logging.info('Total reproducible packages:\t' + str(count_good))
+logging.info('That means that out of the ' + str(percent_total) + '% of ' +
+ 'the Sid tested packages the ' + str(percent_good) + '% are ' +
+ 'reproducible!')
+
+
diff --git a/bin/reproducible_html_notes.py b/bin/reproducible_html_notes.py
new file mode 100755
index 00000000..a89e02da
--- /dev/null
+++ b/bin/reproducible_html_notes.py
@@ -0,0 +1,229 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+# clean-notes: sort and clean the notes stored in notes.git
+# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# Licensed under GPL-2+
+#
+# Depends: python3 python3-yaml
+
+import yaml
+from reproducible_common import *
+
+NOTES = 'packages.yml'
+ISSUES = 'issues.yml'
+
+note_html = Template("""<table class="body">
+<tr>
+ <td>Version annotated:</td>
+ <td>$version</td>
+</tr>
+$infos
+<tr>
+ <td colspan="2">&nbsp;</td>
+</tr>
+<tr>
+ <td colspan="2" style="text-align:right; font-size:0.9em;">
+ <p>Notes are stored in <a href="https://anonscm.debian.org/cgit/reproducible/notes.git">notes.git</a>.</p>
+ </td>
+</tr>
+</table>""")
+note_issues_html = Template("<tr><td>Identified issues:</td><td>$issues</td></tr>")
+note_bugs_html = Template("<tr><td>Bugs noted:</td><td>$bugs</td></tr>")
+note_comments_html = Template("<tr><td>Comments:</td><td>$comments</td></tr>")
+
+note_issue_html_url = Template("""<tr><td>URL</td>
+ <td><a href="$url" target="_blank">$url</a></td></tr>""")
+note_issue_html_desc = Template("""<tr><td>Description</td>
+ <td>$description</td></tr>""")
+note_issue_html = Template("""<table class="body">
+<tr>
+ <td>Identifier:</td>
+ <td><a href="%s/${issue}_issue.html" target="_parent">$issue</a>
+</tr>
+$issue_info
+</table>
+""" % ISSUES_URI)
+
+issue_html_url = Template("""<tr><td>URL:</td><td><a href="$url">$url</a>
+ </td></tr>""")
+issue_html = Template("""<table class="body">
+<tr>
+ <td>Identifier:</td>
+ <th>$issue</th>
+</tr>
+$urls
+<tr>
+ <td>Description:</td>
+ <td>$description</td>
+</tr>
+<tr>
+ <td>Packages known to be affected by this issue:</td>
+ <td>$affected_pkgs</td>
+</tr>
+<tr><td colspan="2">&nbsp;</td></tr>
+<tr><td colspan="2" style="text-align:right; font-size:0.9em;">
+<p>Notes are stored in <a href="https://anonscm.debian.org/cgit/reproducible/notes.git">notes.git</a>.</p>
+</td></tr></table>""")
+
+
+def load_notes():
+ """
+ format:
+ { 'package_name': {'version': '0.0', 'comments'<etc>}, 'package_name':{} }
+ """
+ with open(NOTES) as fd:
+ notes = yaml.load(fd)
+ logging.debug("notes loaded. There are " + str(len(notes)) +
+ " package listed")
+ return notes
+
+
+def load_issues():
+ """
+ format:
+ { 'issue_name': {'description': 'blabla', 'url': 'blabla'} }
+ """
+ with open(ISSUES) as fd:
+ issues = yaml.load(fd)
+ logging.debug("issues loaded. There are " + str(len(issues)) +
+ " issues listed")
+ return issues
+
+
+def fill_issue_in_note(issue):
+ details = issues[issue]
+ html = ''
+ if 'url' in details:
+ html += note_issue_html_url.substitute(url=details['url'])
+ if 'description' in details:
+ desc = details['description'].replace('\n', '<br />')
+ html += note_issue_html_desc.substitute(description=desc)
+ else:
+ logging.warning("The issue " + issue + " misses a description")
+ return note_issue_html.substitute(issue=issue, issue_info=html)
+
+
+def gen_html_note(note):
+ """
+ Given a note as input (as a dict:
+ {"package_name": {"version": "0.0.0", "comments": "blablabla",
+ "bugs": [111, 222], "issues": ["issue1", "issue2"]}}
+ ) it returns the html body
+ """
+ infos = ''
+ # check for issues:
+ if 'issues' in note:
+ tmp = ''
+ for issue in note['issues']:
+ tmp += fill_issue_in_note(issue)
+ try:
+ issues_count[issue].append(note['package'])
+ except KeyError:
+ issues_count[issue] = [note['package']]
+ infos += note_issues_html.substitute(issues=tmp)
+ # check for bugs:
+ if 'bugs' in note:
+ bugurls = ''
+ for bug in note['bugs']:
+ bugurls += '<a href="https://bugs.debian.org/' + str(bug) + \
+ '">' + str(bug) + '</a><br />'
+ infos += note_bugs_html.substitute(bugs=bugurls)
+ # check for comments:
+ if 'comments' in note:
+ comment = note['comments']
+ comment = url2html.sub(r'<a href="\1">\1</a>', comment)
+ comment = comment.replace('\n', '<br />')
+ infos += note_comments_html.substitute(comments=comment)
+ try:
+ return note_html.substitute(version=str(note['version']), infos=infos)
+ except KeyError:
+ logging.warning('You should really include a version in the ' +
+ str(note['package']) + ' note')
+ return note_html.substitute(version='N/A', infos=infos)
+
+def gen_html_issue(issue):
+ """
+ Given a issue as input (as a dict:
+ {"issue_identifier": {"description": "blablabla", "url": "blabla"}}
+ ) it returns the html body
+ """
+ # check for url:
+ if 'url' in issues[issue]:
+ url = issue_html_url.substitute(url=issues[issue]['url'])
+ else:
+ url = ''
+ # add affected packages:
+ affected = ''
+ for pkg in sorted(issues_count[issue]):
+ affected += '<a href="%s/%s.html" class="noted">%s</a>\n' % (
+ RB_PKG_URI, pkg, pkg)
+ # check for description:
+ try:
+ desc = issues[issue]['description']
+ except KeyError:
+ logging.warning('You should really include a description in the ' +
+ issue + ' issue')
+ desc = 'N/A'
+ desc = url2html.sub(r'<a href="\1">\1</a>', desc)
+ desc = desc.replace('\n', '<br />')
+ return issue_html.substitute(issue=issue, urls=url, description=desc,
+ affected_pkgs=affected)
+
+def iterate_over_notes(notes):
+ num_notes = str(len(notes))
+ i = 0
+ for package in sorted(notes):
+ logging.debug('iterating over notes... ' + str(i) + '/' + num_notes)
+ note = notes[package]
+ note['package'] = package
+ logging.debug('\t' + str(note))
+ html = gen_html_note(note)
+
+ title = 'Notes for ' + package + ' - reproducible builds result'
+ destfile = NOTES_PATH + '/' + package + '_note.html'
+ write_html_page(title=title, body=html, destfile=destfile,
+ noheader=True, nofooter=True)
+
+ desturl = REPRODUCIBLE_URL + NOTES_URI + '/' + package + '_note.html'
+ #logging.info("you can now see your notes at " + desturl)
+ i = i + 1
+
+def iterate_over_issues(issues):
+ num_issues = str(len(issues))
+ i = 0
+ for issue in sorted(issues):
+ logging.debug('iterating over issues... ' + str(i) + '/' + num_issues)
+ logging.debug('\t' + str(issue))
+ html = gen_html_issue(issue)
+
+ title = 'Notes about issue ' + issue
+ destfile = ISSUES_PATH + '/' + issue + '_issue.html'
+ write_html_page(title=title, body=html, destfile=destfile)
+
+ desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + issue + '_issue.html'
+ #logging.info("you can now see the issue at " +desturl)
+ i = i + 1
+
+def index_issues(issues):
+ html = '<table class="body">'
+ html += '<tr><th>Identified issues</th></tr>'
+ for issue in sorted(issues):
+ html += '<tr><td><a href="' + ISSUES_URI + '/' + issue + \
+ '_issue.html">' + issue + '</a></td></tr>'
+ html += '</table>'
+ html += '<p>Notes are stored in <a href="https://anonscm.debian.org/cgit/reproducible/notes.git">notes.git</a>.</p>'
+ title = 'Overview of known issues related to reproducible builds'
+ destfile = BASE + '/userContent/index_issues.html'
+ desturl = REPRODUCIBLE_URL + '/userContent/index_issues.html'
+ write_html_page(title=title, body=html, destfile=destfile, nofooter=True)
+ logging.info('Issues index now available at ' + desturl)
+
+if __name__ == '__main__':
+ issues_count = {}
+ notes = load_notes()
+ issues = load_issues()
+ iterate_over_notes(notes)
+ iterate_over_issues(issues)
+ index_issues(issues)
+