diff options
| -rwxr-xr-x | bin/reproducible_html_indexes.py | 144 |
1 files changed, 86 insertions, 58 deletions
diff --git a/bin/reproducible_html_indexes.py b/bin/reproducible_html_indexes.py index d02b4fe5..8277f09c 100755 --- a/bin/reproducible_html_indexes.py +++ b/bin/reproducible_html_indexes.py @@ -15,8 +15,7 @@ from reproducible_common import * Reference doc for the folowing lists: * queries is just a list of queries. They are referred further below. - + every query return the following tuple: - (package_name, suite, architecture) + + every query must return only a list of package names * pages is just a list of pages. It is actually a dictionary, where every element is a page. Every page has: + `title`: The page title @@ -34,30 +33,34 @@ Reference doc for the folowing lists: * $count_total being the number of all tested packages * $count being the len() of the query indicated by `query2` - `query2`: useful only if `timely` is True. +* global_pages is another list of pages. They follows the same structure of + "normal" pages, but with a difference: every section is building for every + (suite, arch) and the page itself is placed outside of any suite/arch + directory. Technically speaking, a package can be empty (we all love nonsense) but every section must have at least a `query` defining what to file in. """ queries = { - 'scheduled': 'SELECT sources.name, sources.suite, sources.architecture FROM schedule JOIN sources ON schedule.package_id=sources.id ORDER BY schedule.date_scheduled', - 'reproducible_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY r.build_date DESC', - 'reproducible_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-24 hours") ORDER BY r.build_date DESC', - 'reproducible_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-48 hours") ORDER BY r.build_date DESC', - 'reproducible_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY name', - 'FTBR_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY build_date DESC', - 'FTBR_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC', - 'FTBR_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC', - 'FTBR_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY name', - 'FTBFS_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY build_date DESC', - 'FTBFS_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC', - 'FTBFS_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC', - 'FTBFS_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY name', - '404_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY build_date DESC', - '404_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY name', - 'not_for_us_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY build_date DESC', - 'not_for_us_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY name', - 'blacklisted_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "blacklisted" ORDER BY name' + 'scheduled': 'SELECT s.name FROM schedule AS p JOIN sources AS s ON p.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" ORDER BY p.date_scheduled', + 'reproducible_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="reproducible" ORDER BY r.build_date DESC', + 'reproducible_last24h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="reproducible" AND r.build_date > datetime("now", "-24 hours") ORDER BY r.build_date DESC', + 'reproducible_last48h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="reproducible" AND r.build_date > datetime("now", "-48 hours") ORDER BY r.build_date DESC', + 'reproducible_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="reproducible" ORDER BY name', + 'FTBR_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "unreproducible" ORDER BY build_date DESC', + 'FTBR_last24h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "unreproducible" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC', + 'FTBR_last48h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "unreproducible" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC', + 'FTBR_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "unreproducible" ORDER BY name', + 'FTBFS_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "FTBFS" ORDER BY build_date DESC', + 'FTBFS_last24h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "FTBFS" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC', + 'FTBFS_last48h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "FTBFS" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC', + 'FTBFS_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "FTBFS" ORDER BY name', + '404_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "404" ORDER BY build_date DESC', + '404_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "404" ORDER BY name', + 'not_for_us_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "not for us" ORDER BY build_date DESC', + 'not_for_us_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "not for us" ORDER BY name', + 'blacklisted_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "blacklisted" ORDER BY name' } pages = { @@ -68,7 +71,7 @@ pages = { 'icon_status': 'reproducible', 'icon_link': '/index_reproducible.html', 'query': 'reproducible_all', - 'text': Template('$tot ($percent%) packages which built reproducibly:') + 'text': Template('$tot ($percent%) packages which built reproducibly in $suite/$arch:') } ] }, @@ -78,7 +81,7 @@ pages = { { 'icon_status': 'FTBR', 'query': 'FTBR_all', - 'text': Template('$tot ($percent%) packages which failed to build reproducibly:') + 'text': Template('$tot ($percent%) packages which failed to build reproducibly in $suite/$arch:') } ] }, @@ -88,7 +91,7 @@ pages = { { 'icon_status': 'FTBFS', 'query': 'FTBFS_all', - 'text': Template('$tot ($percent%) packages where the sources failed to download:') + 'text': Template('$tot ($percent%) packages where the sources failed to download in $suite/$arch:') } ] }, @@ -98,7 +101,7 @@ pages = { { 'icon_status': '404', 'query': '404_all', - 'text': Template('$tot ($percent%) packages which failed to build from source:') + 'text': Template('$tot ($percent%) packages which failed to build from source in $suite/$arch:') } ] }, @@ -108,7 +111,7 @@ pages = { { 'icon_status': 'not_for_us', 'query': 'not_for_us_all', - 'text': Template('$tot ($percent%) packages which should not be build on "amd64":') + 'text': Template('$tot ($percent%) packages which should not be build in $suite/$arch:') } ] }, @@ -118,7 +121,7 @@ pages = { { 'icon_status': 'blacklisted', 'query': 'blacklisted_all', - 'text': Template('$tot ($percent%) packages which have been blacklisted:') + 'text': Template('$tot ($percent%) packages which have been blacklisted in $suite/$arch:') } ] }, @@ -127,7 +130,7 @@ pages = { 'body': [ { 'query': 'scheduled', - 'text': Template('$tot packages are currently scheduled for testing:') + 'text': Template('$tot packages are currently scheduled for testing in $suite/$arch:') } ] }, @@ -138,37 +141,37 @@ pages = { 'icon_status': 'FTBR', 'icon_link': '/index_unreproducible.html', 'query': 'FTBR_all_abc', - 'text': Template('$tot packages ($percent%) failed to built reproducibly in total:') + 'text': Template('$tot packages ($percent%) failed to built reproducibly in total in $suite/$arch:') }, { 'icon_status': 'FTBFS', 'icon_link': '/index_FTBFS.html', 'query': 'FTBFS_all_abc', - 'text': Template('$tot packages ($percent%) failed to built from source in total:') + 'text': Template('$tot packages ($percent%) failed to built from source in total $suite/$arch:') }, { 'icon_status': 'not_for_us', 'icon_link': '/index_not_for_us.html', 'query': 'not_for_us_all_abc', - 'text': Template('$tot ($percent%) packages which are neither Architecture: "any", "all", "amd64", "linux-any", "linux-amd64" nor "any-amd64":') + 'text': Template('$tot ($percent%) packages which should not be build in $suite/$arch:') }, { 'icon_status': '404', 'icon_link': '/index_404.html', 'query': '404_all_abc', - 'text': Template('$tot ($percent%) source packages could not be downloaded:') + 'text': Template('$tot ($percent%) source packages could not be downloaded in $suite/$arch:') }, { 'icon_status': 'blacklisted', 'icon_link': '/index_blacklisted.html', 'query': 'blacklisted_all', - 'text': Template('$tot ($percent%) packages are blacklisted and will not be tested here:') + 'text': Template('$tot ($percent%) packages are blacklisted and will not be tested in $suite/$arch:') }, { 'icon_status': 'reproducible', 'icon_link': '/index_reproducible.html', 'query': 'reproducible_all_abc', - 'text': Template('$tot ($percent%) packages successfully built reproducibly:') + 'text': Template('$tot ($percent%) packages successfully built reproducibly in $suite/$arch:') }, ] }, @@ -181,7 +184,7 @@ pages = { 'query': 'FTBR_last24h', 'query2': 'FTBR_all', 'text': Template('$count packages ($percent% of ${count_total}) ' + \ - 'failed to built reproducibly in total, $tot of them in the last 24h:'), + 'failed to built reproducibly in total, $tot of them in the last 24h in $suite/$arch:'), 'timely': True }, { @@ -190,7 +193,7 @@ pages = { 'query': 'FTBFS_last24h', 'query2': 'FTBFS_all', 'text': Template('$count packages ($percent% of ${count_total}) ' + \ - 'failed to built from source in total, $tot of them in the last 24h:'), + 'failed to built from source in total, $tot of them in the last 24h in $suite/$arch:'), 'timely': True }, { @@ -199,7 +202,7 @@ pages = { 'query': 'reproducible_last24h', 'query2': 'reproducible_all', 'text': Template('$count packages ($percent% of ${count_total}) ' + \ - 'successfully built reproducibly in total, $tot of them in the last 24h:'), + 'successfully built reproducibly in total, $tot of them in the last 24h in $suite/$arch:'), 'timely': True }, ] @@ -213,7 +216,7 @@ pages = { 'query': 'FTBR_last48h', 'query2': 'FTBR_all', 'text': Template('$count packages ($percent% of ${count_total}) ' + \ - 'failed to built reproducibly in total, $tot of them in the last 48h:'), + 'failed to built reproducibly in total, $tot of them in the last 48h in $suite/$arch:'), 'timely': True }, { @@ -222,7 +225,7 @@ pages = { 'query': 'FTBFS_last48h', 'query2': 'FTBFS_all', 'text': Template('$count packages ($percent% of ${count_total}) ' + \ - 'failed to built from source in total, $tot of them in the last 48h:'), + 'failed to built from source in total, $tot of them in the last 48h in $suite/$arch:'), 'timely': True }, { @@ -231,15 +234,27 @@ pages = { 'query': 'reproducible_last48h', 'query2': 'reproducible_all', 'text': Template('$count packages ($percent% of ${count_total}) ' + \ - 'successfully built reproducibly in total, $tot of them in the last 48h:'), + 'successfully built reproducibly in total, $tot of them in the last 48h in $suite/$arch:'), 'timely': True }, ] } } +global_pages = { + 'scheduled': { + 'title': 'Overview of packages currently scheduled for testing for build reproducibility', + 'body': [ + { + 'query': 'scheduled', + 'text': Template('$tot packages are currently scheduled for testing in $suite/$arch:') + } + ] + } +} -def build_leading_text_section(section, rows): + +def build_leading_text_section(section, rows, suite, arch): html = '<p>\n' + tab total = len(rows) try: @@ -261,36 +276,33 @@ def build_leading_text_section(section, rows): html += '</a>' html += '\n' + tab if section.get('text') and section.get('timely') and section['timely']: - count = len(query_db(queries[section['query2']].format(suite=suite))) + count = len(query_db(queries[section['query2']].format(suite=suite, arch=arch))) percent = round(((count/count_total)*100), 1) html += section['text'].substitute(tot=total, percent=percent, count_total=count_total, - count=count) + count=count, suite=suite, arch=arch) elif section.get('text'): - html += section['text'].substitute(tot=total, percent=percent) + html += section['text'].substitute(tot=total, percent=percent, + suite=suite, arch=arch) else: log.warning('There is no text for this section') html += '\n</p>\n' return html -def build_page_section(section): +def build_page_section(section, suite, arch): try: - rows = query_db(queries[section['query']].format(suite=suite)) - # remember: this is a list of tuples! so while looping the package - # name will be pkg[0] and not simply pkg. + rows = query_db(queries[section['query']].format(suite=suite, arch=arch)) except: print_critical_message('A query failed: ' + queries[section['query']]) raise html = '' if not rows: # there are no package in this set return html # do not output anything on the page. - html += build_leading_text_section(section, rows) + html += build_leading_text_section(section, rows, suite, arch) html += '<p>\n' + tab + '<code>\n' for row in rows: pkg = row[0] - #suite = row[1] # FIXME currently we build indexes pages only for sid - arch = row[2] url = RB_PKG_URI + '/' + suite + '/' + arch + '/' + pkg + '.html' html += tab*2 + '<a href="' + url + '" class="' if package_has_notes(pkg): @@ -305,25 +317,41 @@ def build_page_section(section): return html -def build_page(page): - log.info('Building the ' + page + ' index page...') +def build_page(page, suite=None, arch=None): + if not suite: # global page + log.info('Building the ' + page + ' global index page...') + else: + log.info('Building the ' + page + ' index page for ' + suite + '/' + + arch + '...') html = '' for section in pages[page]['body']: - html += build_page_section(section) + if not suite: # global page + for lsuite in SUITES: + for larch in ARCHES: + html += build_page_section(section, lsuite, larch) + else: + html += build_page_section(section, suite, arch) try: title = pages[page]['title'] except KeyError: title = page - destfilename = page - destfile = BASE + '/index_' + destfilename + '.html' - desturl = REPRODUCIBLE_URL + '/index_' + destfilename + '.html' + if not suite: # global page + destfile = BASE + '/index_' + page + '.html' + desturl = REPRODUCIBLE_URL + '/index_' + page + '.html' + else: + destfile = BASE + '/' + suite + '/' + arch + '/index_' + page + '.html' + desturl = REPRODUCIBLE_URL + '/' + suite + '/' + arch + '/index_' + \ + page + '.html' write_html_page(title=title, body=html, destfile=destfile, style_note=True) log.info('"' + title + '" now available at ' + desturl) bugs = get_bugs() -suite = 'sid' if __name__ == '__main__': - for page in pages.keys(): + for suite in SUITES: + for arch in ARCHES: + for page in pages.keys(): + build_page(page, suite, arch) + for page in global_pages.keys(): build_page(page) |