1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
|
#!/usr/bin/python3
# -*- coding: utf-8 -*-
#
# Copyright © 2015 Mattia Rizzolo <mattia@mapreri.org>
# Based on reproducible_html_indexes.sh © 2014 Holger Levsen <holger@layer-acht.org>
# Licensed under GPL-2
#
# Depends: python3
#
# Build quite all index_* pages
from reproducible_common import *
"""
Reference doc for the folowing lists:
* queries is just a list of queries. They are referred further below.
* pages is just a list of pages. It is actually a dictionary, where every
element is a page. Every page has:
+ `title`: The page title
+ `body`: a list of dicts containing every section that made up the page.
Every section has:
- `icon_status`: the name of a icon (see join_status_icon())
- `icon_link`: a link to hide below the icon
- `query`: query to perform against the reproducible db to get the list of
packages to show
- `text` a string.Template instance with $tot (total of packages listed)
and $percent (percentual on all sid packages)
Technically speaking, a package can be empty (we all love nonsense) but every
section must have at least a `query` defining what to file in.
"""
queries = {
'reproducible_all': 'SELECT name FROM source_packages WHERE status = "reproducible" ORDER BY build_date DESC',
'reproducible_last24h': 'SELECT name FROM source_packages WHERE status = "reproducible" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
'reproducible_last48h': 'SELECT name FROM source_packages WHERE status = "reproducible" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
'reproducible_all_abc': 'SELECT name FROM source_packages WHERE status = "reproducible" ORDER BY name',
'FTBR_all': 'SELECT name FROM source_packages WHERE status = "unreproducible" ORDER BY build_date DESC',
'FTBR_last24h': 'SELECT name FROM source_packages WHERE status = "unreproducible" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
'FTBR_last48h': 'SELECT name FROM source_packages WHERE status = "unreproducible" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
'FTBR_all_abc': 'SELECT name FROM source_packages WHERE status = "unreproducible" ORDER BY name',
'FTBFS_all': 'SELECT name FROM source_packages WHERE status = "FTBFS" ORDER BY build_date DESC',
'FTBFS_last24h': 'SELECT name FROM source_packages WHERE status = "FTBFS" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
'FTBFS_last48h': 'SELECT name FROM source_packages WHERE status = "FTBFS" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
'FTBFS_all_abc': 'SELECT name FROM source_packages WHERE status = "FTBFS" ORDER BY name',
'404_all': 'SELECT name FROM source_packages WHERE status = "404" ORDER BY build_date DESC',
'404_all_abc': 'SELECT name FROM source_packages WHERE status = "404" ORDER BY name',
'not_for_us_all': 'SELECT name FROM source_packages WHERE status = "not for us" ORDER BY build_date DESC',
'not_for_us_all_abc': 'SELECT name FROM source_packages WHERE status = "not for us" ORDER BY name',
'blacklisted_all': 'SELECT name FROM source_packages WHERE status = "blacklisted" ORDER BY name'
}
pages = {
'reproducible': {
'title': 'Overview of packages which built reproducibly',
'body': [
{
'icon_status': 'reproducible',
'icon_link': '/index_reproducible.html',
'query': 'reproducible_all',
'text': Template('$tot ($percent%) packages which built reproducibly.')
}
]
},
'FTBR': {
'title': 'Overview of packages which failed to build reproducibly',
'body': [
{
'icon_status': 'FTBR',
'query': 'FTBR_all',
'text': Template('$tot ($percent%) packages which failed to build reproducibly.')
}
]
},
'FTBFS': {
'title': 'Overview of packages which failed to build from source',
'body': [
{
'icon_status': 'FTBFS',
'query': 'FTBFS_all',
'text': Template('$tot ($percent%) packages where the sources failed to download.')
}
]
},
'404': {
'title': 'Overview of packages where the sources failed to download',
'body': [
{
'icon_status': '404',
'query': '404_all',
'text': Template('$tot ($percent%) packages which failed to build from source.')
}
]
},
'not_for_us': {
'title': 'Overview of packages which should not be build on "amd64"',
'body': [
{
'icon_status': 'not_for_us',
'query': 'not_for_us_all',
'text': Template('$tot ($percent%) packages which should not be build on "amd64".')
}
]
},
'blacklisted': {
'title': 'Overview of packages which have been blacklisted',
'body': [
{
'icon_status': 'blacklisted',
'query': 'blacklisted_all',
'text': Template('$tot ($percent%) packages which have been blacklisted.')
}
]
}
}
def build_leading_text_section(section, rows):
html = '<p>\n' + tab
total = len(rows)
percent = round(((total/count_total)*100), 1) # count_total is
try: # defined in common
html += '<a href="' + section['icon_link'] + '" target="_parent">'
no_icon_link = False
except KeyError:
no_icon_link = True # to avoid closing the </a> tag below
if section.get('icon_status'):
html += '<img src="/static/'
html += join_status_icon(section['icon_status'])[1]
html += '" alt="reproducible icon" />'
if not no_icon_link:
html += '</a>'
html += '\n' + tab
if section.get('text') and section.get('timely') and section['timely']:
count = len(query_db(queries[section['query2']]))
percent = round(((count/count_total)*100), 1)
html += section['text'].substitute(tot=total, percent=percent,
count_total=count_total,
count=count)
elif section.get('text'):
html += section['text'].substitute(tot=total, percent=percent)
else:
log.warning('There is no text for this section')
html += '\n</p>\n'
return html
def build_page_section(section):
try:
rows = query_db(queries[section['query']])
# remember: this is a list of tuples! so while looping the package
# name will be pkg[0] and not simply pkg.
except:
print_critical_message('A query failed: ' + queries[section['query']])
raise
html = ''
html += build_leading_text_section(section, rows)
html += '<p>\n' + tab + '<code>\n'
for pkg in rows:
url = RB_PKG_URI + '/' + pkg[0] + '.html'
html += tab*2 + '<a href="' + url + '" class="'
if package_has_notes(pkg[0]):
html += 'noted'
else:
html += 'package'
html += '">' + pkg[0] + '</a>'
html += get_trailing_icon(pkg[0], bugs) + '\n'
html += tab + '</code>\n'
html += '</p>'
html = (tab*2).join(html.splitlines(True))
return html
def build_page(page):
log.info('Building the ' + page + ' index page...')
html = ''
for section in pages[page]['body']:
html += build_page_section(section)
try:
title = pages[page]['title']
except KeyError:
title = page
destfilename = page
destfile = BASE + '/index_' + destfilename + '.html'
desturl = REPRODUCIBLE_URL + '/index_' + destfilename + '.html'
write_html_page(title=title, body=html, destfile=destfile, style_note=True)
log.info('"' + title + '" now available at ' + desturl)
if __name__ == '__main__':
bugs = get_bugs()
print(bugs)
for page in pages.keys():
build_page(page)
|