reproducible: move the generation of index_notes and index_no_notes from _homl_notes to _html_indexes

2 files changed, 82 insertions, 79 deletions

diff --git a/bin/reproducible_html_indexes.py b/bin/reproducible_html_indexes.py
index 56369062..262f2a08 100755
--- a/bin/reproducible_html_indexes.py
+++ b/bin/reproducible_html_indexes.py
@@ -76,7 +76,9 @@ queries = {
     '404_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "404" ORDER BY name',
     'not_for_us_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "not for us" ORDER BY build_date DESC',
     'not_for_us_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "not for us" ORDER BY name',
-    'blacklisted_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "blacklisted" ORDER BY name'
+    'blacklisted_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND status = "blacklisted" ORDER BY name',
+    'notes': 'SELECT s.name FROM sources AS s JOIN notes AS n ON n.package_id=s.id JOIN results AS r ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="{status}" ORDER BY s.name',
+    'no_notes': 'SELECT s.name FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE s.suite="{suite}" AND s.architecture="{arch}" AND r.status="{status}" AND s.id NOT IN (SELECT package_id FROM notes) ORDER BY s.name'
 }
 
 pages = {
@@ -250,6 +252,83 @@ pages = {
                 'timely': True
             },
         ]
+    },
+    'notes': {
+        'global': True,
+        'title': 'Packages with notes',
+        'header': '<p>There are {tot} packages with notes.</p>',
+        'header_query': 'SELECT count(*) FROM (SELECT * FROM sources AS s JOIN notes AS n ON n.package_id=s.id GROUP BY s.name) AS tmp',
+        'body': [
+            {
+                'icon_status': 'FTBR',
+                'db_status': 'unreproducible',
+                'icon_link': '/index_FTBR.html',
+                'query': 'notes',
+                'nosuite': True,
+                'text': Template('$tot unreproducible packages in $suite/$arch :')
+            },
+            {
+                'icon_status': 'FTBFS',
+                'db_status': 'FTBFS',
+                'icon_link': '/index_FTBFS.html',
+                'query': 'notes',
+                'nosuite': True,
+                'text': Template('$tot FTBFS packages in $suite/$arch:')
+            },
+            {
+                'icon_status': 'not_for_us',
+                'db_status': 'not for us',
+                'icon_link': '/index_not_for_us.html',
+                'query': 'notes',
+                'nosuite': True,
+                'text': Template('$tot not for us packages in $suite/$arch:')
+            },
+            {
+                'icon_status': 'blacklisted',
+                'db_status': 'blacklisted',
+                'icon_link': '/index_blacklisted.html',
+                'query': 'notes',
+                'nosuite': True,
+                'text': Template('$tot blacklisted packages in $suite/$arch:')
+            },
+            {
+                'icon_status': 'reproducible',
+                'db_status': 'reproducible',
+                'icon_link': '/index_reproducible.html',
+                'query': 'notes',
+                'nosuite': True,
+                'text': Template('$tot reproducible packages in $suite/$arch:')
+            }
+        ]
+    },
+    'no_notes': {
+        'global': True,
+        'title': 'Packages without notes',
+        'header': '<p>There are {tot} faulty packages without notes, in all suites. These are the packages with failures that still need to be investigated.</p>',
+        'header_query': 'SELECT COUNT(*) FROM (SELECT s.id FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status IN ("unreproducible", "FTBFS", "blacklisted") AND s.id NOT IN (SELECT package_id FROM notes))',
+        'body': [
+            {
+                'icon_status': 'FTBR',
+                'db_status': 'unreproducible',
+                'icon_link': '/index_FTBR.html',
+                'query': 'no_notes',
+                'text': Template('$tot unreproducible packages in $suite/$arch:')
+            },
+            {
+                'icon_status': 'FTBFS',
+                'db_status': 'FTBFS',
+                'icon_link': '/index_FTBFS.html',
+                'query': 'no_notes',
+                'text': Template('$tot FTBFS packages in $suite/$arch:')
+            },
+            {
+                'icon_status': 'blacklisted',
+                'db_status': 'blacklisted',
+                'icon_link': '/index_blacklisted.html',
+                'query': 'no_notes',
+                'text': Template('$tot blacklisted packages in $suite/$arch:')
+            }
+        ]
     }
 }
 
diff --git a/bin/reproducible_html_notes.py b/bin/reproducible_html_notes.py
index 4d496fb2..de56c3f9 100755
--- a/bin/reproducible_html_notes.py
+++ b/bin/reproducible_html_notes.py
@@ -402,82 +402,6 @@ def index_issues(issues):
     log.info('Issues index now available at ' + desturl)
 
 
-def index_notes(notes, bugs):
-    log.debug('Building the index_notes page...')
-    suite = 'unstable'
-    arch = 'amd64'
-    with_notes = [x for x in all_pkgs if x[2] == suite and x[3] == arch and x[0] in notes]
-    html = '\n<p>There are ' + str(len(notes)) + ' packages with notes.</p>\n'
-    for status in ['unreproducible', 'FTBFS', 'not for us', 'blacklisted', 'reproducible']:
-        pkgs = [x[0] for x in with_notes if x[1] == status]
-        if not pkgs:
-            continue
-        html += '<p>\n'
-        html += tab + '<img src="/static/' + join_status_icon(status)[1] + '"'
-        html += ' alt="' + status + ' icon" />\n'
-        html += tab + str(len(pkgs)) + ' ' + status + ' packages in ' + suite + '/' + arch + ':\n'
-        html += tab + '<code>\n'
-        for pkg in sorted(pkgs):
-            url = RB_PKG_URI + '/' + suite + '/' + arch + '/' + pkg + '.html'
-            html += tab*2 + '<a href="' + url + '" class="noted">' + pkg
-            html += '</a>' + get_trailing_icon(pkg, bugs) + '\n'
-        html += tab + '</code>\n'
-        html += '</p>\n'
-    html += '<p>Notes are stored in <a href="https://anonscm.debian.org/cgit/reproducible/notes.git" target="_parent">notes.git</a>.</p>'
-    html = (tab*2).join(html.splitlines(True))
-    title = 'Packages with notes'
-    destfile = BASE + '/index_notes.html'
-    desturl = REPRODUCIBLE_URL + '/index_notes.html'
-    write_html_page(title=title, body=html, destfile=destfile,
-                    style_note=True)
-    log.info('Notes index now available at ' + desturl)
-
-
-def index_no_notes_section(notes, bugs, packages, suite, arch):
-    html = ''
-    for status in ['unreproducible', 'FTBFS']:
-        pkgs = [x for x in packages if x[3] == status]
-        if not pkgs:
-            continue
-        html += '<p>\n'
-        html += tab + '<img src="/static/' + join_status_icon(status)[1] + '"'
-        html += ' alt="' + status + ' icon" />\n'
-        html += tab + str(len(pkgs)) + ' ' + status + ' packages in ' + suite + '/' + arch + ':\n'
-        html += tab + '<code>\n'
-        for pkg in pkgs:
-            # 0: name, 1: suite, 2: arch, 3: status
-            url = RB_PKG_URI + '/' + pkg[1] + '/' + pkg[2] + '/' + pkg[0] + '.html'
-            html += tab*2 + '<a href="' + url + '" class="package">' + pkg[0]
-            html += '</a>' + get_trailing_icon(pkg[0], bugs) + '\n'
-        html += tab + '</code>\n'
-        html += '</p>\n'
-    return html
-
-
-def index_no_notes(notes, bugs):
-    log.debug('Building the index_no_notes page...')
-    all_bad_pkgs = query_db('SELECT s.name, s.suite, s.architecture, r.status ' +
-                        'FROM results AS r JOIN sources AS s ON r.package_id=s.id ' +
-                        'WHERE r.status = "unreproducible" OR r.status = "FTBFS"' +
-                        'ORDER BY r.build_date DESC')
-    without_notes = [x for x in all_bad_pkgs if x[0] not in notes]
-    html = '\n<p>There are ' + str(len(without_notes)) + ' faulty ' \
-           + 'packages without notes, in all suites. These are the packages ' \
-           + 'with failures that still need to be investigated.</p>\n'
-    for suite in SUITES:
-        for arch in ARCHES:
-            pkgs = [x for x in without_notes if x[1] == suite and x[2] == arch]
-            html += index_no_notes_section(notes, bugs, pkgs, suite, arch)
-    html += '<p>Notes are stored in <a href="https://anonscm.debian.org/cgit/reproducible/notes.git" target="_parent">notes.git</a>.</p>'
-    html = (tab*2).join(html.splitlines(True))
-    title = 'Packages without notes'
-    destfile = BASE + '/index_no_notes.html'
-    desturl = REPRODUCIBLE_URL + '/index_no_notes.html'
-    write_html_page(title=title, body=html, destfile=destfile,
-                    style_note=True)
-    log.info('Packages without notes index now available at ' + desturl)
-
-
 if __name__ == '__main__':
     all_pkgs = query_db('SELECT s.name, r.status, s.suite, s.architecture ' +
                         'FROM results AS r JOIN sources AS s ON r.package_id=s.id ' +
@@ -489,10 +413,10 @@ if __name__ == '__main__':
     iterate_over_notes(notes)
     iterate_over_issues(issues)
     index_issues(issues)
-    index_notes(notes, bugs)
-    index_no_notes(notes, bugs)
     purge_old_notes(notes)
     gen_packages_html(notes) # regenerate all rb-pkg/ pages
+    build_page('notes')
+    build_page('no_notes')
     for suite in SUITES:
         for arch in ARCHES:
             build_page('FTBFS', suite, arch)