#!/usr/bin/python3 # -*- coding: utf-8 -*- # # Copyright © 2015 Mattia Rizzolo # Copyright © 2015 Holger Levsen # Based on reproducible_html_notes.sh © 2014 Holger Levsen # Licensed under GPL-2 # # Depends: python3 python3-yaml # # Build html pages based on the content of the notes.git repository import copy import yaml from reproducible_common import * from reproducible_html_packages import gen_packages_html NOTES = 'packages.yml' ISSUES = 'issues.yml' note_html = Template((tab*2).join(""" $infos
Version annotated: $version
 

Notes are stored in notes.git.

""".splitlines(True))) note_issues_html = Template((tab*3).join(""" Identified issues: $issues """.splitlines(True))) note_bugs_html = Template((tab*4).join(""" Bugs noted: $bugs """.splitlines(True))) note_comments_html = Template((tab*3).join(""" Comments: $comments """.splitlines(True))) note_issue_html_url = Template((tab*6).join(""" URL $url """.splitlines(True))) note_issue_html_desc = Template((tab*6).join(""" Description $description """.splitlines(True))) note_issue_html = Template((tab*5).join((""" $issue_info
Identifier: $issue
""" % ISSUES_URI).splitlines(True))) issue_html_url = Template((tab*4).join(""" URL: $url """.splitlines(True))) issue_html = Template((tab*3).join(""" $urls
Identifier: $issue
Description: $description
Packages known to be affected by this issue: $affected_pkgs
 

Notes are stored in notes.git.

""".splitlines(True))) def load_notes(): """ format: { 'package_name': {'version': '0.0', 'comments'}, 'package_name':{} } """ with open(NOTES) as fd: possible_notes = yaml.load(fd) log.debug("notes loaded. There are " + str(len(possible_notes)) + " package listed") notes = copy.copy(possible_notes) for package in possible_notes: # check if every package listed on the notes try: # actually have been tested query = 'SELECT s.name ' + \ 'FROM results AS r JOIN sources AS s ON r.package_id=s.id ' + \ 'WHERE s.name="{pkg}" AND r.status != ""' query = query.format(pkg=package) result = query_db(query)[0] except IndexError: log.warning("This query produces no results: " + query) log.warning("This means there is no tested package with the name " + package + ".") del notes[package] log.debug("notes checked. There are " + str(len(notes)) + " package listed") return notes def load_issues(): """ format: { 'issue_name': {'description': 'blabla', 'url': 'blabla'} } """ with open(ISSUES) as fd: issues = yaml.load(fd) log.debug("issues loaded. There are " + str(len(issues)) + " issues listed") return issues def fill_issue_in_note(issue): details = issues[issue] html = '' if 'url' in details: html += note_issue_html_url.substitute(url=details['url']) if 'description' in details: desc = details['description'].replace('\n', '
') html += note_issue_html_desc.substitute(description=desc) else: log.warning("The issue " + issue + " misses a description") return note_issue_html.substitute(issue=issue, issue_info=html) def gen_html_note(package, note): """ Given a note as input (as a dict: {"package_name": {"version": "0.0.0", "comments": "blablabla", "bugs": [111, 222], "issues": ["issue1", "issue2"]}} ) it returns the html body """ infos = '' # check for issues: if 'issues' in note: tmp = '' for issue in note['issues']: tmp += fill_issue_in_note(issue) try: issues_count[issue].append(note['package']) except KeyError: issues_count[issue] = [note['package']] infos += note_issues_html.substitute(issues=tmp) # check for bugs: if 'bugs' in note: bugurls = '' for bug in note['bugs']: bugurls += '' + str(bug) + '' + \ get_trailing_bug_icon(bug, bugs, package) + '
' infos += note_bugs_html.substitute(bugs=bugurls) # check for comments: if 'comments' in note: comment = note['comments'] comment = url2html.sub(r'\1', comment) comment = comment.replace('\n', '
') infos += note_comments_html.substitute(comments=comment) try: return note_html.substitute(version=str(note['version']), infos=infos) except KeyError: log.warning('You should really include a version in the ' + str(note['package']) + ' note') return note_html.substitute(version='N/A', infos=infos) def gen_html_issue(issue): """ Given a issue as input (as a dict: {"issue_identifier": {"description": "blablabla", "url": "blabla"}} ) it returns the html body """ # check for url: if 'url' in issues[issue]: url = issue_html_url.substitute(url=issues[issue]['url']) else: url = '' # add affected packages: affected = '' try: suite = 'unstable' arch = 'amd64' for status in ['unreproducible', 'FTBFS', 'not for us', 'blacklisted', 'reproducible']: pkgs = [x[0] for x in all_pkgs if x[1] == status and x[2] == suite and x[3] == arch and x[0] in issues_count[issue]] if not pkgs: continue affected += tab*4 + '

\n' affected += tab*5 + '\n' affected += tab*5 + str(len(pkgs)) + ' ' + status + ' packages in ' + suite + '/' + arch +':\n' affected += tab*5 + '\n' for pkg in pkgs: affected += tab*6 + '' + pkg affected += '' + get_trailing_icon(pkg, bugs) + '\n' affected += tab*5 + '\n' affected += tab*4 + '

\n' except KeyError: # The note is not listed in any package, that is affected = 'None' # check for description: try: desc = issues[issue]['description'] except KeyError: log.warning('You should really include a description in the ' + issue + ' issue') desc = 'N/A' desc = url2html.sub(r'\1', desc) desc = desc.replace('\n', '
') return issue_html.substitute(issue=issue, urls=url, description=desc, affected_pkgs=affected) def purge_old_notes(notes): removed_pages = [] to_rebuild = [] presents = sorted(os.listdir(NOTES_PATH)) for page in presents: pkg = page.rsplit('_', 1)[0] log.debug('Checking if ' + page + ' (from ' + pkg + ') is still needed') if pkg not in notes: log.info('There are no notes for ' + pkg + '. Removing old page.') os.remove(NOTES_PATH + '/' + page) removed_pages.append(pkg) for pkg in removed_pages: for suite in SUITES: try: query = 'SELECT s.name ' + \ 'FROM results AS r JOIN sources AS s ON r.package_id=s.id ' + \ 'WHERE s.name="{pkg}" AND r.status != "" AND s.suite="{suite}"' query = query.format(pkg=pkg, suite=suite) to_rebuild.append(query_db(query)[0][0]) except IndexError: # the package is not tested. this can happen if pass # a package got removed from the archive if to_rebuild: gen_packages_html(to_rebuild) def iterate_over_notes(notes): num_notes = str(len(notes)) i = 0 for package in sorted(notes): log.debug('iterating over notes... ' + str(i) + '/' + num_notes) note = notes[package] note['package'] = package log.debug('\t' + str(note)) html = gen_html_note(package, note) title = 'Notes for ' + package + ' - reproducible builds result' destfile = NOTES_PATH + '/' + package + '_note.html' write_html_page(title=title, body=html, destfile=destfile, noheader=True) desturl = REPRODUCIBLE_URL + NOTES_URI + '/' + package + '_note.html' log.info("Note created: " + desturl) i = i + 1 def iterate_over_issues(issues): num_issues = str(len(issues)) i = 0 for issue in sorted(issues): log.debug('iterating over issues... ' + str(i) + '/' + num_issues) log.debug('\t' + str(issue)) html = gen_html_issue(issue) title = 'Notes about issue ' + issue destfile = ISSUES_PATH + '/' + issue + '_issue.html' write_html_page(title=title, body=html, destfile=destfile, style_note=True) desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + issue + '_issue.html' log.info("Issue created: " + desturl) i = i + 1 def sort_issues(issue): try: return (-len(issues_count[issue]), issue) except KeyError: # there are no packages affected by this issue return (0, issue) def index_issues(issues): templ = "\n\n" + tab + "\n" + tab*2 + "\n" + tab*2 + "\n" + tab + "\n" html = (tab*2).join(templ.splitlines(True)) for issue in sorted(issues, key=sort_issues): html += tab*3 + '\n' html += tab*4 + '\n' html += tab*4 + '\n' html += tab*3 + '\n' html += tab*2 + '
\n" \ + tab*3 + "Identified issues\n" + tab*2 + "\n" \ + tab*3 + "Affected packages\n" + tab*2 + "
' + issue + '\n' try: html += tab*5 + '' + str(len(issues_count[issue])) + ':\n' html += tab*5 + ', '.join(issues_count[issue]) + '\n' except KeyError: # there are no packages affected by this issue html += tab*5 + '0\n' html += tab*4 + '
\n' html += tab*2 + '

For a total of ' + \ str(len([x for x in notes if notes[x].get('issues')])) + \ ' packages categorized in ' + str(len(issues)) + \ ' issues.

' html += tab*2 + '

Notes are stored in notes.git.

' title = 'Known issues related to reproducible builds' destfile = BASE + '/index_issues.html' desturl = REPRODUCIBLE_URL + '/index_issues.html' write_html_page(title=title, body=html, destfile=destfile) log.info('Issues index now available at ' + desturl) def index_notes(notes, bugs): log.debug('Building the index_notes page...') suite = 'unstable' arch = 'amd64' with_notes = [x for x in all_pkgs if x[2] == suite and x[3] == arch and x[0] in notes] html = '\n

There are ' + str(len(notes)) + ' packages with notes.

\n' for status in ['unreproducible', 'FTBFS', 'not for us', 'blacklisted', 'reproducible']: pkgs = [x[0] for x in with_notes if x[1] == status] if not pkgs: continue html += '

\n' html += tab + '\n' html += tab + str(len(pkgs)) + ' ' + status + ' packages in ' + suite + '/' + arch + ':\n' html += tab + '\n' for pkg in sorted(pkgs): url = RB_PKG_URI + '/' + suite + '/' + arch + '/' + pkg + '.html' html += tab*2 + '' + pkg html += '' + get_trailing_icon(pkg, bugs) + '\n' html += tab + '\n' html += '

\n' html += '

Notes are stored in notes.git.

' html = (tab*2).join(html.splitlines(True)) title = 'Packages with notes' destfile = BASE + '/index_notes.html' desturl = REPRODUCIBLE_URL + '/index_notes.html' write_html_page(title=title, body=html, destfile=destfile, style_note=True) log.info('Notes index now available at ' + desturl) def index_no_notes_section(notes, bugs, packages, suite, arch): html = '' for status in ['unreproducible', 'FTBFS']: pkgs = [x for x in packages if x[3] == status] if not pkgs: continue html += '

\n' html += tab + '\n' html += tab + str(len(pkgs)) + ' ' + status + ' packages in ' + suite + '/' + arch + ':\n' html += tab + '\n' for pkg in pkgs: # 0: name, 1: suite, 2: arch, 3: status url = RB_PKG_URI + '/' + pkg[1] + '/' + pkg[2] + '/' + pkg[0] + '.html' html += tab*2 + '' + pkg[0] html += '' + get_trailing_icon(pkg[0], bugs) + '\n' html += tab + '\n' html += '

\n' return html def index_no_notes(notes, bugs): log.debug('Building the index_no_notes page...') all_bad_pkgs = query_db('SELECT s.name, s.suite, s.architecture, r.status ' + 'FROM results AS r JOIN sources AS s ON r.package_id=s.id ' + 'WHERE r.status = "unreproducible" OR r.status = "FTBFS"' + 'ORDER BY r.build_date DESC') without_notes = [x for x in all_bad_pkgs if x[0] not in notes] html = '\n

There are ' + str(len(without_notes)) + ' faulty ' \ + 'packages without notes, in all suites. These are the packages ' \ + 'with failures that still need to be investigated.

\n' for suite in SUITES: for arch in ARCHES: pkgs = [x for x in without_notes if x[1] == suite and x[2] == arch] html += index_no_notes_section(notes, bugs, pkgs, suite, arch) html += '

Notes are stored in notes.git.

' html = (tab*2).join(html.splitlines(True)) title = 'Packages without notes' destfile = BASE + '/index_no_notes.html' desturl = REPRODUCIBLE_URL + '/index_no_notes.html' write_html_page(title=title, body=html, destfile=destfile, style_note=True) log.info('Packages without notes index now available at ' + desturl) if __name__ == '__main__': all_pkgs = query_db('SELECT s.name, r.status, s.suite, s.architecture ' + 'FROM results AS r JOIN sources AS s ON r.package_id=s.id ' + 'ORDER BY s.name') issues_count = {} bugs = get_bugs() notes = load_notes() issues = load_issues() iterate_over_notes(notes) iterate_over_issues(issues) index_issues(issues) index_notes(notes, bugs) index_no_notes(notes, bugs) purge_old_notes(notes) gen_packages_html(notes) # regenerate all rb-pkg/ pages