reproducible: html_indexes: build indexes for every suite/arch, and place them in a sensible directory. plus, support a new kind of index global_pages, which are superset of the suite-relative ones

1 files changed, 86 insertions, 58 deletions

diff --git a/bin/reproducible_html_indexes.py b/bin/reproducible_html_indexes.py
index d02b4fe5..8277f09c 100755
--- a/bin/reproducible_html_indexes.py
+++ b/bin/reproducible_html_indexes.py
@@ -15,8 +15,7 @@ from reproducible_common import *
 Reference doc for the folowing lists:
 
 * queries is just a list of queries. They are referred further below.
-  + every query return the following tuple:
-    (package_name, suite, architecture)
+  + every query must return only a list of package names
 * pages is just a list of pages. It is actually a dictionary, where every
   element is a page. Every page has:
   + `title`: The page title
@@ -34,30 +33,34 @@ Reference doc for the folowing lists:
       * $count_total being the number of all tested packages
       * $count being the len() of the query indicated by `query2`
     - `query2`: useful only if `timely` is True.
+* global_pages is another list of pages. They follows the same structure of
+  "normal" pages, but with a difference: every section is building for every
+  (suite, arch) and the page itself is placed outside of any suite/arch
+  directory.
 
 Technically speaking, a package can be empty (we all love nonsense) but every
 section must have at least a `query` defining what to file in.
 """
 
 queries = {
-    'scheduled': 'SELECT sources.name, sources.suite, sources.architecture FROM schedule JOIN sources ON schedule.package_id=sources.id ORDER BY schedule.date_scheduled',
-    'reproducible_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY r.build_date DESC',
-    'reproducible_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-24 hours") ORDER BY r.build_date DESC',
-    'reproducible_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-48 hours") ORDER BY r.build_date DESC',
-    'reproducible_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY name',
-    'FTBR_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY build_date DESC',
-    'FTBR_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
-    'FTBR_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
-    'FTBR_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY name',
-    'FTBFS_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY build_date DESC',
-    'FTBFS_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
-    'FTBFS_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
-    'FTBFS_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY name',
-    '404_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY build_date DESC',
-    '404_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY name',
-    'not_for_us_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY build_date DESC',
-    'not_for_us_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY name',
-    'blacklisted_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "blacklisted" ORDER BY name'
+    'scheduled': 'SELECT s.name FROM schedule AS p JOIN sources AS s ON p.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" ORDER BY p.date_scheduled',
+    'reproducible_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="reproducible" ORDER BY r.build_date DESC',
+    'reproducible_last24h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="reproducible" AND r.build_date > datetime("now", "-24 hours") ORDER BY r.build_date DESC',
+    'reproducible_last48h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="reproducible" AND r.build_date > datetime("now", "-48 hours") ORDER BY r.build_date DESC',
+    'reproducible_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="reproducible" ORDER BY name',
+    'FTBR_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "unreproducible" ORDER BY build_date DESC',
+    'FTBR_last24h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "unreproducible" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
+    'FTBR_last48h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "unreproducible" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
+    'FTBR_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "unreproducible" ORDER BY name',
+    'FTBFS_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "FTBFS" ORDER BY build_date DESC',
+    'FTBFS_last24h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "FTBFS" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
+    'FTBFS_last48h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "FTBFS" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
+    'FTBFS_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "FTBFS" ORDER BY name',
+    '404_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "404" ORDER BY build_date DESC',
+    '404_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "404" ORDER BY name',
+    'not_for_us_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "not for us" ORDER BY build_date DESC',
+    'not_for_us_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "not for us" ORDER BY name',
+    'blacklisted_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "blacklisted" ORDER BY name'
 }
 
 pages = {
@@ -68,7 +71,7 @@ pages = {
                 'icon_status': 'reproducible',
                 'icon_link': '/index_reproducible.html',
                 'query': 'reproducible_all',
-                'text': Template('$tot ($percent%) packages which built reproducibly:')
+                'text': Template('$tot ($percent%) packages which built reproducibly in $suite/$arch:')
             }
         ]
     },
@@ -78,7 +81,7 @@ pages = {
             {
                 'icon_status': 'FTBR',
                 'query': 'FTBR_all',
-                'text': Template('$tot ($percent%) packages which failed to build reproducibly:')
+                'text': Template('$tot ($percent%) packages which failed to build reproducibly in $suite/$arch:')
             }
         ]
     },
@@ -88,7 +91,7 @@ pages = {
             {
                 'icon_status': 'FTBFS',
                 'query': 'FTBFS_all',
-                'text': Template('$tot ($percent%) packages where the sources failed to download:')
+                'text': Template('$tot ($percent%) packages where the sources failed to download in $suite/$arch:')
             }
         ]
     },
@@ -98,7 +101,7 @@ pages = {
             {
                 'icon_status': '404',
                 'query': '404_all',
-                'text': Template('$tot ($percent%) packages which failed to build from source:')
+                'text': Template('$tot ($percent%) packages which failed to build from source in $suite/$arch:')
             }
         ]
     },
@@ -108,7 +111,7 @@ pages = {
             {
                 'icon_status': 'not_for_us',
                 'query': 'not_for_us_all',
-                'text': Template('$tot ($percent%) packages which should not be build on "amd64":')
+                'text': Template('$tot ($percent%) packages which should not be build in $suite/$arch:')
             }
         ]
     },
@@ -118,7 +121,7 @@ pages = {
             {
                 'icon_status': 'blacklisted',
                 'query': 'blacklisted_all',
-                'text': Template('$tot ($percent%) packages which have been blacklisted:')
+                'text': Template('$tot ($percent%) packages which have been blacklisted in $suite/$arch:')
             }
         ]
     },
@@ -127,7 +130,7 @@ pages = {
         'body': [
             {
                 'query': 'scheduled',
-                'text': Template('$tot packages are currently scheduled for testing:')
+                'text': Template('$tot packages are currently scheduled for testing in $suite/$arch:')
             }
         ]
     },
@@ -138,37 +141,37 @@ pages = {
                 'icon_status': 'FTBR',
                 'icon_link': '/index_unreproducible.html',
                 'query': 'FTBR_all_abc',
-                'text': Template('$tot packages ($percent%) failed to built reproducibly in total:')
+                'text': Template('$tot packages ($percent%) failed to built reproducibly in total in $suite/$arch:')
             },
             {
                 'icon_status': 'FTBFS',
                 'icon_link': '/index_FTBFS.html',
                 'query': 'FTBFS_all_abc',
-                'text': Template('$tot packages ($percent%) failed to built from source in total:')
+                'text': Template('$tot packages ($percent%) failed to built from source in total $suite/$arch:')
             },
             {
                 'icon_status': 'not_for_us',
                 'icon_link': '/index_not_for_us.html',
                 'query': 'not_for_us_all_abc',
-                'text': Template('$tot ($percent%) packages which are neither Architecture: "any", "all", "amd64", "linux-any", "linux-amd64" nor "any-amd64":')
+                'text': Template('$tot ($percent%) packages which should not be build in $suite/$arch:')
             },
             {
                 'icon_status': '404',
                 'icon_link': '/index_404.html',
                 'query': '404_all_abc',
-                'text': Template('$tot ($percent%) source packages could not be downloaded:')
+                'text': Template('$tot ($percent%) source packages could not be downloaded in $suite/$arch:')
             },
             {
                 'icon_status': 'blacklisted',
                 'icon_link': '/index_blacklisted.html',
                 'query': 'blacklisted_all',
-                'text': Template('$tot ($percent%) packages are blacklisted and will not be tested here:')
+                'text': Template('$tot ($percent%) packages are blacklisted and will not be tested in $suite/$arch:')
             },
             {
                 'icon_status': 'reproducible',
                 'icon_link': '/index_reproducible.html',
                 'query': 'reproducible_all_abc',
-                'text': Template('$tot ($percent%) packages successfully built reproducibly:')
+                'text': Template('$tot ($percent%) packages successfully built reproducibly in $suite/$arch:')
             },
         ]
     },
@@ -181,7 +184,7 @@ pages = {
                 'query': 'FTBR_last24h',
                 'query2': 'FTBR_all',
                 'text': Template('$count packages ($percent% of ${count_total}) ' + \
-                                 'failed to built reproducibly in total, $tot of them in the last 24h:'),
+                                 'failed to built reproducibly in total, $tot of them in the last 24h in $suite/$arch:'),
                 'timely': True
             },
             {
@@ -190,7 +193,7 @@ pages = {
                 'query': 'FTBFS_last24h',
                 'query2': 'FTBFS_all',
                 'text': Template('$count packages ($percent% of ${count_total}) ' + \
-                                 'failed to built from source in total, $tot of them  in the last 24h:'),
+                                 'failed to built from source in total, $tot of them  in the last 24h in $suite/$arch:'),
                 'timely': True
             },
             {
@@ -199,7 +202,7 @@ pages = {
                 'query': 'reproducible_last24h',
                 'query2': 'reproducible_all',
                 'text': Template('$count packages ($percent% of ${count_total}) ' + \
-                                 'successfully built reproducibly in total, $tot of them in the last 24h:'),
+                                 'successfully built reproducibly in total, $tot of them in the last 24h in $suite/$arch:'),
                 'timely': True
             },
         ]
@@ -213,7 +216,7 @@ pages = {
                 'query': 'FTBR_last48h',
                 'query2': 'FTBR_all',
                 'text': Template('$count packages ($percent% of ${count_total}) ' + \
-                                 'failed to built reproducibly in total, $tot of them in the last 48h:'),
+                                 'failed to built reproducibly in total, $tot of them in the last 48h in $suite/$arch:'),
                 'timely': True
             },
             {
@@ -222,7 +225,7 @@ pages = {
                 'query': 'FTBFS_last48h',
                 'query2': 'FTBFS_all',
                 'text': Template('$count packages ($percent% of ${count_total}) ' + \
-                                 'failed to built from source in total, $tot of them  in the last 48h:'),
+                                 'failed to built from source in total, $tot of them  in the last 48h in $suite/$arch:'),
                 'timely': True
             },
             {
@@ -231,15 +234,27 @@ pages = {
                 'query': 'reproducible_last48h',
                 'query2': 'reproducible_all',
                 'text': Template('$count packages ($percent% of ${count_total}) ' + \
-                                 'successfully built reproducibly in total, $tot of them in the last 48h:'),
+                                 'successfully built reproducibly in total, $tot of them in the last 48h in $suite/$arch:'),
                 'timely': True
             },
         ]
     }
 }
 
+global_pages = {
+    'scheduled': {
+        'title': 'Overview of packages currently scheduled for testing for build reproducibility',
+        'body': [
+            {
+                'query': 'scheduled',
+                'text': Template('$tot packages are currently scheduled for testing in $suite/$arch:')
+            }
+        ]
+    }
+}
 
-def build_leading_text_section(section, rows):
+
+def build_leading_text_section(section, rows, suite, arch):
     html = '<p>\n' + tab
     total = len(rows)
     try:
@@ -261,36 +276,33 @@ def build_leading_text_section(section, rows):
         html += '</a>'
     html += '\n' + tab
     if section.get('text') and section.get('timely') and section['timely']:
-        count = len(query_db(queries[section['query2']].format(suite=suite)))
+        count = len(query_db(queries[section['query2']].format(suite=suite, arch=arch)))
         percent = round(((count/count_total)*100), 1)
         html += section['text'].substitute(tot=total, percent=percent,
                                            count_total=count_total,
-                                           count=count)
+                                           count=count, suite=suite, arch=arch)
     elif section.get('text'):
-        html += section['text'].substitute(tot=total, percent=percent)
+        html += section['text'].substitute(tot=total, percent=percent,
+                                           suite=suite, arch=arch)
     else:
         log.warning('There is no text for this section')
     html += '\n</p>\n'
     return html
 
 
-def build_page_section(section):
+def build_page_section(section, suite, arch):
     try:
-        rows = query_db(queries[section['query']].format(suite=suite))
-        # remember: this is a list of tuples! so while looping the package
-        # name will be pkg[0] and not simply pkg.
+        rows = query_db(queries[section['query']].format(suite=suite, arch=arch))
     except:
         print_critical_message('A query failed: ' + queries[section['query']])
         raise
     html = ''
     if not rows:     # there are no package in this set
         return html  # do not output anything on the page.
-    html += build_leading_text_section(section, rows)
+    html += build_leading_text_section(section, rows, suite, arch)
     html += '<p>\n' + tab + '<code>\n'
     for row in rows:
         pkg = row[0]
-        #suite = row[1]  # FIXME currently we build indexes pages only for sid
-        arch = row[2]
         url = RB_PKG_URI + '/' + suite + '/' + arch + '/' + pkg + '.html'
         html += tab*2 + '<a href="' + url + '" class="'
         if package_has_notes(pkg):
@@ -305,25 +317,41 @@ def build_page_section(section):
     return html
 
 
-def build_page(page):
-    log.info('Building the ' + page + ' index page...')
+def build_page(page, suite=None, arch=None):
+    if not suite:  # global page
+        log.info('Building the ' + page + ' global index page...')
+    else:
+        log.info('Building the ' + page + ' index page for ' + suite + '/' +
+                 arch + '...')
     html = ''
     for section in pages[page]['body']:
-        html += build_page_section(section)
+        if not suite:  # global page
+            for lsuite in SUITES:
+                for larch in ARCHES:
+                    html += build_page_section(section, lsuite, larch)
+        else:
+            html += build_page_section(section, suite, arch)
     try:
         title = pages[page]['title']
     except KeyError:
         title = page
-    destfilename = page
-    destfile = BASE + '/index_' + destfilename + '.html'
-    desturl = REPRODUCIBLE_URL + '/index_' + destfilename + '.html'
+    if not suite:  # global page
+        destfile = BASE + '/index_' + page + '.html'
+        desturl = REPRODUCIBLE_URL + '/index_' + page + '.html'
+    else:
+        destfile = BASE + '/' + suite + '/' + arch + '/index_' + page + '.html'
+        desturl = REPRODUCIBLE_URL + '/' + suite + '/' + arch + '/index_' + \
+                  page + '.html'
     write_html_page(title=title, body=html, destfile=destfile, style_note=True)
     log.info('"' + title + '" now available at ' + desturl)
 
 
 bugs = get_bugs()
-suite = 'sid'
 
 if __name__ == '__main__':
-    for page in pages.keys():
+    for suite in SUITES:
+        for arch in ARCHES:
+            for page in pages.keys():
+                build_page(page, suite, arch)
+    for page in global_pages.keys():
         build_page(page)