summaryrefslogtreecommitdiffstats
path: root/bin
diff options
context:
space:
mode:
authorMattia Rizzolo <mattia@mapreri.org>2015-03-02 17:33:09 +0100
committerHolger Levsen <holger@layer-acht.org>2015-03-02 19:29:32 +0100
commita6a4843f039c541330ea86a61515f63cd2a3aef3 (patch)
tree7fca4f0bfd4653316431ea8535cceed2b7856244 /bin
parentd4667a399108b360a63d9d4953039410dee553a5 (diff)
downloadjenkins.debian.net-a6a4843f039c541330ea86a61515f63cd2a3aef3.tar.xz
reproducible: html_indexes: build indexes for every suite/arch, and place them in a sensible directory. plus, support a new kind of index global_pages, which are superset of the suite-relative ones
Diffstat (limited to 'bin')
-rwxr-xr-xbin/reproducible_html_indexes.py144
1 files changed, 86 insertions, 58 deletions
diff --git a/bin/reproducible_html_indexes.py b/bin/reproducible_html_indexes.py
index d02b4fe5..8277f09c 100755
--- a/bin/reproducible_html_indexes.py
+++ b/bin/reproducible_html_indexes.py
@@ -15,8 +15,7 @@ from reproducible_common import *
Reference doc for the folowing lists:
* queries is just a list of queries. They are referred further below.
- + every query return the following tuple:
- (package_name, suite, architecture)
+ + every query must return only a list of package names
* pages is just a list of pages. It is actually a dictionary, where every
element is a page. Every page has:
+ `title`: The page title
@@ -34,30 +33,34 @@ Reference doc for the folowing lists:
* $count_total being the number of all tested packages
* $count being the len() of the query indicated by `query2`
- `query2`: useful only if `timely` is True.
+* global_pages is another list of pages. They follows the same structure of
+ "normal" pages, but with a difference: every section is building for every
+ (suite, arch) and the page itself is placed outside of any suite/arch
+ directory.
Technically speaking, a package can be empty (we all love nonsense) but every
section must have at least a `query` defining what to file in.
"""
queries = {
- 'scheduled': 'SELECT sources.name, sources.suite, sources.architecture FROM schedule JOIN sources ON schedule.package_id=sources.id ORDER BY schedule.date_scheduled',
- 'reproducible_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY r.build_date DESC',
- 'reproducible_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-24 hours") ORDER BY r.build_date DESC',
- 'reproducible_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-48 hours") ORDER BY r.build_date DESC',
- 'reproducible_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY name',
- 'FTBR_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY build_date DESC',
- 'FTBR_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
- 'FTBR_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
- 'FTBR_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY name',
- 'FTBFS_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY build_date DESC',
- 'FTBFS_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
- 'FTBFS_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
- 'FTBFS_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY name',
- '404_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY build_date DESC',
- '404_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY name',
- 'not_for_us_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY build_date DESC',
- 'not_for_us_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY name',
- 'blacklisted_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "blacklisted" ORDER BY name'
+ 'scheduled': 'SELECT s.name FROM schedule AS p JOIN sources AS s ON p.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" ORDER BY p.date_scheduled',
+ 'reproducible_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="reproducible" ORDER BY r.build_date DESC',
+ 'reproducible_last24h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="reproducible" AND r.build_date > datetime("now", "-24 hours") ORDER BY r.build_date DESC',
+ 'reproducible_last48h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="reproducible" AND r.build_date > datetime("now", "-48 hours") ORDER BY r.build_date DESC',
+ 'reproducible_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="reproducible" ORDER BY name',
+ 'FTBR_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "unreproducible" ORDER BY build_date DESC',
+ 'FTBR_last24h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "unreproducible" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
+ 'FTBR_last48h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "unreproducible" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
+ 'FTBR_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "unreproducible" ORDER BY name',
+ 'FTBFS_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "FTBFS" ORDER BY build_date DESC',
+ 'FTBFS_last24h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "FTBFS" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
+ 'FTBFS_last48h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "FTBFS" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
+ 'FTBFS_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "FTBFS" ORDER BY name',
+ '404_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "404" ORDER BY build_date DESC',
+ '404_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "404" ORDER BY name',
+ 'not_for_us_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "not for us" ORDER BY build_date DESC',
+ 'not_for_us_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "not for us" ORDER BY name',
+ 'blacklisted_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "blacklisted" ORDER BY name'
}
pages = {
@@ -68,7 +71,7 @@ pages = {
'icon_status': 'reproducible',
'icon_link': '/index_reproducible.html',
'query': 'reproducible_all',
- 'text': Template('$tot ($percent%) packages which built reproducibly:')
+ 'text': Template('$tot ($percent%) packages which built reproducibly in $suite/$arch:')
}
]
},
@@ -78,7 +81,7 @@ pages = {
{
'icon_status': 'FTBR',
'query': 'FTBR_all',
- 'text': Template('$tot ($percent%) packages which failed to build reproducibly:')
+ 'text': Template('$tot ($percent%) packages which failed to build reproducibly in $suite/$arch:')
}
]
},
@@ -88,7 +91,7 @@ pages = {
{
'icon_status': 'FTBFS',
'query': 'FTBFS_all',
- 'text': Template('$tot ($percent%) packages where the sources failed to download:')
+ 'text': Template('$tot ($percent%) packages where the sources failed to download in $suite/$arch:')
}
]
},
@@ -98,7 +101,7 @@ pages = {
{
'icon_status': '404',
'query': '404_all',
- 'text': Template('$tot ($percent%) packages which failed to build from source:')
+ 'text': Template('$tot ($percent%) packages which failed to build from source in $suite/$arch:')
}
]
},
@@ -108,7 +111,7 @@ pages = {
{
'icon_status': 'not_for_us',
'query': 'not_for_us_all',
- 'text': Template('$tot ($percent%) packages which should not be build on "amd64":')
+ 'text': Template('$tot ($percent%) packages which should not be build in $suite/$arch:')
}
]
},
@@ -118,7 +121,7 @@ pages = {
{
'icon_status': 'blacklisted',
'query': 'blacklisted_all',
- 'text': Template('$tot ($percent%) packages which have been blacklisted:')
+ 'text': Template('$tot ($percent%) packages which have been blacklisted in $suite/$arch:')
}
]
},
@@ -127,7 +130,7 @@ pages = {
'body': [
{
'query': 'scheduled',
- 'text': Template('$tot packages are currently scheduled for testing:')
+ 'text': Template('$tot packages are currently scheduled for testing in $suite/$arch:')
}
]
},
@@ -138,37 +141,37 @@ pages = {
'icon_status': 'FTBR',
'icon_link': '/index_unreproducible.html',
'query': 'FTBR_all_abc',
- 'text': Template('$tot packages ($percent%) failed to built reproducibly in total:')
+ 'text': Template('$tot packages ($percent%) failed to built reproducibly in total in $suite/$arch:')
},
{
'icon_status': 'FTBFS',
'icon_link': '/index_FTBFS.html',
'query': 'FTBFS_all_abc',
- 'text': Template('$tot packages ($percent%) failed to built from source in total:')
+ 'text': Template('$tot packages ($percent%) failed to built from source in total $suite/$arch:')
},
{
'icon_status': 'not_for_us',
'icon_link': '/index_not_for_us.html',
'query': 'not_for_us_all_abc',
- 'text': Template('$tot ($percent%) packages which are neither Architecture: "any", "all", "amd64", "linux-any", "linux-amd64" nor "any-amd64":')
+ 'text': Template('$tot ($percent%) packages which should not be build in $suite/$arch:')
},
{
'icon_status': '404',
'icon_link': '/index_404.html',
'query': '404_all_abc',
- 'text': Template('$tot ($percent%) source packages could not be downloaded:')
+ 'text': Template('$tot ($percent%) source packages could not be downloaded in $suite/$arch:')
},
{
'icon_status': 'blacklisted',
'icon_link': '/index_blacklisted.html',
'query': 'blacklisted_all',
- 'text': Template('$tot ($percent%) packages are blacklisted and will not be tested here:')
+ 'text': Template('$tot ($percent%) packages are blacklisted and will not be tested in $suite/$arch:')
},
{
'icon_status': 'reproducible',
'icon_link': '/index_reproducible.html',
'query': 'reproducible_all_abc',
- 'text': Template('$tot ($percent%) packages successfully built reproducibly:')
+ 'text': Template('$tot ($percent%) packages successfully built reproducibly in $suite/$arch:')
},
]
},
@@ -181,7 +184,7 @@ pages = {
'query': 'FTBR_last24h',
'query2': 'FTBR_all',
'text': Template('$count packages ($percent% of ${count_total}) ' + \
- 'failed to built reproducibly in total, $tot of them in the last 24h:'),
+ 'failed to built reproducibly in total, $tot of them in the last 24h in $suite/$arch:'),
'timely': True
},
{
@@ -190,7 +193,7 @@ pages = {
'query': 'FTBFS_last24h',
'query2': 'FTBFS_all',
'text': Template('$count packages ($percent% of ${count_total}) ' + \
- 'failed to built from source in total, $tot of them in the last 24h:'),
+ 'failed to built from source in total, $tot of them in the last 24h in $suite/$arch:'),
'timely': True
},
{
@@ -199,7 +202,7 @@ pages = {
'query': 'reproducible_last24h',
'query2': 'reproducible_all',
'text': Template('$count packages ($percent% of ${count_total}) ' + \
- 'successfully built reproducibly in total, $tot of them in the last 24h:'),
+ 'successfully built reproducibly in total, $tot of them in the last 24h in $suite/$arch:'),
'timely': True
},
]
@@ -213,7 +216,7 @@ pages = {
'query': 'FTBR_last48h',
'query2': 'FTBR_all',
'text': Template('$count packages ($percent% of ${count_total}) ' + \
- 'failed to built reproducibly in total, $tot of them in the last 48h:'),
+ 'failed to built reproducibly in total, $tot of them in the last 48h in $suite/$arch:'),
'timely': True
},
{
@@ -222,7 +225,7 @@ pages = {
'query': 'FTBFS_last48h',
'query2': 'FTBFS_all',
'text': Template('$count packages ($percent% of ${count_total}) ' + \
- 'failed to built from source in total, $tot of them in the last 48h:'),
+ 'failed to built from source in total, $tot of them in the last 48h in $suite/$arch:'),
'timely': True
},
{
@@ -231,15 +234,27 @@ pages = {
'query': 'reproducible_last48h',
'query2': 'reproducible_all',
'text': Template('$count packages ($percent% of ${count_total}) ' + \
- 'successfully built reproducibly in total, $tot of them in the last 48h:'),
+ 'successfully built reproducibly in total, $tot of them in the last 48h in $suite/$arch:'),
'timely': True
},
]
}
}
+global_pages = {
+ 'scheduled': {
+ 'title': 'Overview of packages currently scheduled for testing for build reproducibility',
+ 'body': [
+ {
+ 'query': 'scheduled',
+ 'text': Template('$tot packages are currently scheduled for testing in $suite/$arch:')
+ }
+ ]
+ }
+}
-def build_leading_text_section(section, rows):
+
+def build_leading_text_section(section, rows, suite, arch):
html = '<p>\n' + tab
total = len(rows)
try:
@@ -261,36 +276,33 @@ def build_leading_text_section(section, rows):
html += '</a>'
html += '\n' + tab
if section.get('text') and section.get('timely') and section['timely']:
- count = len(query_db(queries[section['query2']].format(suite=suite)))
+ count = len(query_db(queries[section['query2']].format(suite=suite, arch=arch)))
percent = round(((count/count_total)*100), 1)
html += section['text'].substitute(tot=total, percent=percent,
count_total=count_total,
- count=count)
+ count=count, suite=suite, arch=arch)
elif section.get('text'):
- html += section['text'].substitute(tot=total, percent=percent)
+ html += section['text'].substitute(tot=total, percent=percent,
+ suite=suite, arch=arch)
else:
log.warning('There is no text for this section')
html += '\n</p>\n'
return html
-def build_page_section(section):
+def build_page_section(section, suite, arch):
try:
- rows = query_db(queries[section['query']].format(suite=suite))
- # remember: this is a list of tuples! so while looping the package
- # name will be pkg[0] and not simply pkg.
+ rows = query_db(queries[section['query']].format(suite=suite, arch=arch))
except:
print_critical_message('A query failed: ' + queries[section['query']])
raise
html = ''
if not rows: # there are no package in this set
return html # do not output anything on the page.
- html += build_leading_text_section(section, rows)
+ html += build_leading_text_section(section, rows, suite, arch)
html += '<p>\n' + tab + '<code>\n'
for row in rows:
pkg = row[0]
- #suite = row[1] # FIXME currently we build indexes pages only for sid
- arch = row[2]
url = RB_PKG_URI + '/' + suite + '/' + arch + '/' + pkg + '.html'
html += tab*2 + '<a href="' + url + '" class="'
if package_has_notes(pkg):
@@ -305,25 +317,41 @@ def build_page_section(section):
return html
-def build_page(page):
- log.info('Building the ' + page + ' index page...')
+def build_page(page, suite=None, arch=None):
+ if not suite: # global page
+ log.info('Building the ' + page + ' global index page...')
+ else:
+ log.info('Building the ' + page + ' index page for ' + suite + '/' +
+ arch + '...')
html = ''
for section in pages[page]['body']:
- html += build_page_section(section)
+ if not suite: # global page
+ for lsuite in SUITES:
+ for larch in ARCHES:
+ html += build_page_section(section, lsuite, larch)
+ else:
+ html += build_page_section(section, suite, arch)
try:
title = pages[page]['title']
except KeyError:
title = page
- destfilename = page
- destfile = BASE + '/index_' + destfilename + '.html'
- desturl = REPRODUCIBLE_URL + '/index_' + destfilename + '.html'
+ if not suite: # global page
+ destfile = BASE + '/index_' + page + '.html'
+ desturl = REPRODUCIBLE_URL + '/index_' + page + '.html'
+ else:
+ destfile = BASE + '/' + suite + '/' + arch + '/index_' + page + '.html'
+ desturl = REPRODUCIBLE_URL + '/' + suite + '/' + arch + '/index_' + \
+ page + '.html'
write_html_page(title=title, body=html, destfile=destfile, style_note=True)
log.info('"' + title + '" now available at ' + desturl)
bugs = get_bugs()
-suite = 'sid'
if __name__ == '__main__':
- for page in pages.keys():
+ for suite in SUITES:
+ for arch in ARCHES:
+ for page in pages.keys():
+ build_page(page, suite, arch)
+ for page in global_pages.keys():
build_page(page)