#!/usr/bin/python3 # -*- coding: utf-8 -*- # # Copyright © 2015 Mattia Rizzolo # Based on reproducible_html_notes.sh © 2014 Holger Levsen # Licensed under GPL-2 # # Depends: python3 python3-yaml # # Build html pages based on the content of the notes.git repository import yaml from reproducible_common import * from reproducible_html_packages import process_packages NOTES = 'packages.yml' ISSUES = 'issues.yml' note_html = Template((tab*2).join(""" $infos
Version annotated: $version
 

Notes are stored in notes.git.

""".splitlines(True))) note_issues_html = Template((tab*3).join(""" Identified issues: $issues """.splitlines(True))) note_bugs_html = Template((tab*4).join(""" Bugs noted: $bugs """.splitlines(True))) note_comments_html = Template((tab*3).join(""" Comments: $comments """.splitlines(True))) note_issue_html_url = Template((tab*6).join(""" URL $url """.splitlines(True))) note_issue_html_desc = Template((tab*6).join(""" Description $description """.splitlines(True))) note_issue_html = Template((tab*5).join((""" $issue_info
Identifier: $issue
""" % ISSUES_URI).splitlines(True))) issue_html_url = Template((tab*4).join(""" URL: $url """.splitlines(True))) issue_html = Template((tab*3).join(""" $urls
Identifier: $issue
Description: $description
Packages known to be affected by this issue: $affected_pkgs
 

Notes are stored in notes.git.

""".splitlines(True))) def load_notes(): """ format: { 'package_name': {'version': '0.0', 'comments'}, 'package_name':{} } """ with open(NOTES) as fd: notes = yaml.load(fd) log.debug("notes loaded. There are " + str(len(notes)) + " package listed") for package in notes: # check if every pacakge listed on the notes try: # actually have been tested query = 'SELECT name ' + \ 'FROM source_packages ' + \ 'WHERE name="%s"' % package result = query_db(query)[0] except IndexError: print_critical_message('This query produces no results: ' + query + '\nThis means there is no tested package with the name ' + package + '.') raise return notes def load_issues(): """ format: { 'issue_name': {'description': 'blabla', 'url': 'blabla'} } """ with open(ISSUES) as fd: issues = yaml.load(fd) log.debug("issues loaded. There are " + str(len(issues)) + " issues listed") return issues def fill_issue_in_note(issue): details = issues[issue] html = '' if 'url' in details: html += note_issue_html_url.substitute(url=details['url']) if 'description' in details: desc = details['description'].replace('\n', '
') html += note_issue_html_desc.substitute(description=desc) else: log.warning("The issue " + issue + " misses a description") return note_issue_html.substitute(issue=issue, issue_info=html) def gen_html_note(note): """ Given a note as input (as a dict: {"package_name": {"version": "0.0.0", "comments": "blablabla", "bugs": [111, 222], "issues": ["issue1", "issue2"]}} ) it returns the html body """ infos = '' # check for issues: if 'issues' in note: tmp = '' for issue in note['issues']: tmp += fill_issue_in_note(issue) try: issues_count[issue].append(note['package']) except KeyError: issues_count[issue] = [note['package']] infos += note_issues_html.substitute(issues=tmp) # check for bugs: if 'bugs' in note: bugurls = '' for bug in note['bugs']: bugurls += '' + str(bug) + '
' infos += note_bugs_html.substitute(bugs=bugurls) # check for comments: if 'comments' in note: comment = note['comments'] comment = url2html.sub(r'\1', comment) comment = comment.replace('\n', '
') infos += note_comments_html.substitute(comments=comment) try: return note_html.substitute(version=str(note['version']), infos=infos) except KeyError: log.warning('You should really include a version in the ' + str(note['package']) + ' note') return note_html.substitute(version='N/A', infos=infos) def gen_html_issue(issue): """ Given a issue as input (as a dict: {"issue_identifier": {"description": "blablabla", "url": "blabla"}} ) it returns the html body """ # check for url: if 'url' in issues[issue]: url = issue_html_url.substitute(url=issues[issue]['url']) else: url = '' # add affected packages: affected = '' try: for pkg in sorted(issues_count[issue]): affected += tab*6 + '%s\n' \ % (RB_PKG_URI, pkg, pkg) except KeyError: # The note is not listed in any package, that is affected = 'None' # check for description: try: desc = issues[issue]['description'] except KeyError: log.warning('You should really include a description in the ' + issue + ' issue') desc = 'N/A' desc = url2html.sub(r'\1', desc) desc = desc.replace('\n', '
') return issue_html.substitute(issue=issue, urls=url, description=desc, affected_pkgs=affected) def purge_old_notes(notes): removed_pages = [] presents = sorted(os.listdir(NOTES_PATH)) for page in presents: pkg = page.rsplit('_', 1)[0] log.debug('Checking if ' + page + ' (from ' + pkg + ') is still needed') if pkg not in notes: log.info('There are no notes for ' + pkg + '. Removing old page.') os.remove(NOTES_PATH + '/' + page) removed_pages.append(pkg) if removed_pages: process_packages(removed_pages) def iterate_over_notes(notes): num_notes = str(len(notes)) i = 0 for package in sorted(notes): log.debug('iterating over notes... ' + str(i) + '/' + num_notes) note = notes[package] note['package'] = package log.debug('\t' + str(note)) html = gen_html_note(note) title = 'Notes for ' + package + ' - reproducible builds result' destfile = NOTES_PATH + '/' + package + '_note.html' write_html_page(title=title, body=html, destfile=destfile, noheader=True) desturl = REPRODUCIBLE_URL + NOTES_URI + '/' + package + '_note.html' log.info("you can now see your notes at " + desturl) i = i + 1 def iterate_over_issues(issues): num_issues = str(len(issues)) i = 0 for issue in sorted(issues): log.debug('iterating over issues... ' + str(i) + '/' + num_issues) log.debug('\t' + str(issue)) html = gen_html_issue(issue) title = 'Notes about issue ' + issue destfile = ISSUES_PATH + '/' + issue + '_issue.html' write_html_page(title=title, body=html, destfile=destfile, style_note=True) desturl = REPRODUCIBLE_URL + ISSUES_URI + '/' + issue + '_issue.html' log.info("you can now see the issue at " + desturl) i = i + 1 def sort_issues(issue): try: return (-len(issues_count[issue]), issue) except KeyError: # there are no packages affected by this issue return (0, issue) def index_issues(issues): templ = "\n\n" + tab + "\n" + tab*2 + "\n" + tab*2 + "\n" + tab + "\n" html = (tab*2).join(templ.splitlines(True)) for issue in sorted(issues, key=sort_issues): html += tab*3 + '\n' html += tab*4 + '\n' html += tab*4 + '\n' html += tab*3 + '\n' html += tab*2 + '
\n" \ + tab*3 + "Identified issues\n" + tab*2 + "\n" \ + tab*3 + "Affected packages\n" + tab*2 + "
' + issue + '\n' try: html += tab*5 + '' + str(len(issues_count[issue])) + ':\n' html += tab*5 + ', '.join(issues_count[issue]) + '\n' except KeyError: # there are no packages affected by this issue html += tab*5 + '0\n' html += tab*4 + '
\n' html += tab*2 + '

For a total of ' + \ str(len([x for x in notes if notes[x].get('issues')])) + \ ' packages categorized in ' + str(len(issues)) + \ ' issues.

' html += tab*2 + '

Notes are stored in notes.git.

' title = 'Overview of known issues related to reproducible builds' destfile = BASE + '/index_issues.html' desturl = REPRODUCIBLE_URL + '/index_issues.html' write_html_page(title=title, body=html, destfile=destfile) log.info('Issues index now available at ' + desturl) def get_trailing_icon(package, bugs): html = '' if package in bugs: for bug in bugs[package]: html += '#' elif bugs[package][bug]['patch']: html += 'bug-patch" title="#' + str(bug) + ', with patch">+' else: html += '" title="#' + str(bug) + '">+' return html def index_notes(notes, bugs): log.debug('Building the index_notes page...') html = '\n

There are ' + str(len(notes)) + ' packages with notes.

\n' html += '

\n' + tab + '\n' html = (tab*2).join(html.splitlines(True)) for pkg in sorted(notes): url = RB_PKG_URI + '/' + pkg + '.html' html += tab*4 + '' + pkg + '' html += get_trailing_icon(pkg, bugs) + '\n' html += tab*3 + '\n' html += tab*2 + '

\n' html += tab*2 + '

Notes are stored in notes.git.

' title = 'Overview of packages with notes' destfile = BASE + '/index_notes.html' desturl = REPRODUCIBLE_URL + '/index_notes.html' write_html_page(title=title, body=html, destfile=destfile, style_note=True) log.info('Notes index now available at ' + desturl) def index_no_notes(notes, bugs): log.debug('Building the index_no_notes page...') all_pkgs = query_db('SELECT name FROM source_packages ' + 'WHERE status = "unreproducible" OR status = "FTBFS"' + 'ORDER BY build_date DESC') without_notes = [x[0] for x in all_pkgs if x[0] not in notes] html = '\n

There are ' + str(len(without_notes)) + ' unreproducible ' \ + 'packages without notes. These are the packages with failures ' \ + 'that still need to be investigated:

\n' html += '

\n' + tab + '\n' html = (tab*2).join(html.splitlines(True)) for pkg in without_notes: url = RB_PKG_URI + '/' + pkg + '.html' html += tab*4 + '' + pkg + '' html += get_trailing_icon(pkg, bugs) + '\n' html += tab*3 + '\n' html += tab*2 + '

\n' html += tab*2 + '

Notes are stored in notes.git.

' title = 'Overview of packages without notes' destfile = BASE + '/index_no_notes.html' desturl = REPRODUCIBLE_URL + '/index_no_notes.html' write_html_page(title=title, body=html, destfile=destfile, style_note=True) log.info('Packages without notes index now available at ' + desturl) if __name__ == '__main__': issues_count = {} bugs = get_bugs() notes = load_notes() issues = load_issues() iterate_over_notes(notes) iterate_over_issues(issues) index_issues(issues) index_notes(notes, bugs) index_no_notes(notes, bugs) purge_old_notes(notes) process_packages(notes) # regenerate all rb-pkg/ pages