diff options
| -rwxr-xr-x | bin/reproducible_common.py | 88 | ||||
| -rwxr-xr-x | bin/reproducible_html_packages.py | 93 |
2 files changed, 87 insertions, 94 deletions
diff --git a/bin/reproducible_common.py b/bin/reproducible_common.py index 519ac56d..b6294928 100755 --- a/bin/reproducible_common.py +++ b/bin/reproducible_common.py @@ -191,14 +191,49 @@ def write_html_page(title, body, destfile, noheader=False, style_note=False, noe with open(destfile, 'w') as fd: fd.write(html) -def init_conn(): +def start_db_connection(): return sqlite3.connect(REPRODUCIBLE_DB) def query_db(query): - cursor = conn.cursor() + cursor = conn_db.cursor() cursor.execute(query) return cursor.fetchall() +def start_udd_connection(): + username = "public-udd-mirror" + password = "public-udd-mirror" + host = "public-udd-mirror.xvm.mit.edu" + port = 5432 + db = "udd" + try: + log.debug("Starting connection to the UDD database") + conn = psycopg2.connect("dbname=" + db + + " user=" + username + + " host=" + host + + " password=" + password) + except: + log.error("Erorr connecting to the UDD database replica") + raise + conn.set_client_encoding('utf8') + return conn + +def query_udd(query): + cursor = conn_udd.cursor() + cursor.execute(query) + return cursor.fetchall() + +def is_virtual_package(package): + rows = query_udd("""SELECT source FROM sources WHERE source='%s'""" % package) + if len(rows) > 0: + return False + return True + +def bug_has_patch(bug): + query = """SELECT id FROM bugs_tags WHERE id=%s AND tag='patch'""" % bug + if len(query_udd(query)) > 0: + return True + return False + def join_status_icon(status, package=None, version=None): table = {'reproducible' : 'weather-clear.png', 'FTBFS': 'weather-storm.png', @@ -253,8 +288,55 @@ def pkg_has_buildinfo(package, version=False): else: return False +def get_bugs(): + """ + This function returns a dict: + { "package_name": { + bug1: {patch: True, done: False}, + bug2: {patch: False, done: False}, + } + } + """ + query = """ + SELECT bugs.id, bugs.source, bugs.done + FROM bugs JOIN bugs_tags on bugs.id = bugs_tags.id + JOIN bugs_usertags on bugs_tags.id = bugs_usertags.id + WHERE bugs_usertags.email = 'reproducible-builds@lists.alioth.debian.org' + AND bugs.id NOT IN ( + SELECT id + FROM bugs_usertags + WHERE email = 'reproducible-builds@lists.alioth.debian.org' + AND ( + bugs_usertags.tag = 'toolchain' + OR bugs_usertags.tag = 'infrastructure') + ) + """ + # returns a list of tuples [(id, source, done)] + rows = query_udd(query) + log.info("finding out which usertagged bugs have been closed or at least have patches") + packages = {} + + for bug in rows: + if bug[1] not in packages: + packages[bug[1]] = {} + # bug[0] = bug_id, bug[1] = source_name, bug[2] = who_when_done + if is_virtual_package(bug[1]): + continue + packages[bug[1]][bug[0]] = {'done': False, 'patch': False} + if bug[2]: # if the bug is done + packages[bug[1]][bug[0]]['done'] = True + try: + if bug_has_patch(bug[0]): + packages[bug[1]][bug[0]]['patch'] = True + except KeyError: + log.error('item: ' + str(bug)) + return packages + +# init the databases connections +conn_db = start_db_connection() # the local sqlite3 reproducible db +conn_udd = start_udd_connection() + # do the db querying -conn = init_conn() amount = int(query_db('SELECT count(name) FROM sources')[0][0]) count_total = int(query_db('SELECT COUNT(name) FROM source_packages')[0][0]) count_good = int(query_db( diff --git a/bin/reproducible_html_packages.py b/bin/reproducible_html_packages.py index 060d43ea..424d6507 100755 --- a/bin/reproducible_html_packages.py +++ b/bin/reproducible_html_packages.py @@ -55,59 +55,6 @@ def sizeof_fmt(num): num /= 1024.0 return str(int(round(float("%f" % num), 0))) + "%s" % ('Yi') -def start_udd_connection(): - username = "public-udd-mirror" - password = "public-udd-mirror" - host = "public-udd-mirror.xvm.mit.edu" - port = 5432 - db = "udd" - try: - log.debug("Starting connection to the UDD database") - conn = psycopg2.connect("dbname=" + db + - " user=" + username + - " host=" + host + - " password=" + password) - except: - log.error("Erorr connecting to the UDD database replica") - raise - conn.set_client_encoding('utf8') - return conn - -def query_udd(query): - cursor = conn.cursor() - cursor.execute(query) - return cursor.fetchall() - -def is_virtual_package(package): - rows = query_udd("""SELECT source FROM sources WHERE source='%s'""" % package) - if len(rows) > 0: - return False - return True - -def bug_has_patch(bug): - query = """SELECT id FROM bugs_tags WHERE id=%s AND tag='patch'""" % bug - if len(query_udd(query)) > 0: - return True - return False - -def get_bugs(): - query = """ - SELECT bugs.id, bugs.source, bugs.done - FROM bugs JOIN bugs_tags on bugs.id = bugs_tags.id - JOIN bugs_usertags on bugs_tags.id = bugs_usertags.id - WHERE bugs_usertags.email = 'reproducible-builds@lists.alioth.debian.org' - AND bugs.id NOT IN ( - SELECT id - FROM bugs_usertags - WHERE email = 'reproducible-builds@lists.alioth.debian.org' - AND ( - bugs_usertags.tag = 'toolchain' - OR bugs_usertags.tag = 'infrastructure') - ) - """ - # returns a list of tuples [(id, source, done)] - return query_udd(query) - def check_package_status(package): """ This returns a tuple containing status, version and build_date of the last @@ -167,38 +114,6 @@ def gen_extra_links(package, version): ' did not produce any buildlog! Check ' + rbuild) return (links, default_view) -def parse_bugs(bugs): - """ - This function returns a dict: - { "package_name": { - bug1: {patch: True, done: False}, - bug2: {patch: False, done: False}, - } - } - - The `bugs` argument is the list of tuples returned by the get_bugs() above - """ - log.info("finding out which usertagged bugs have been closed or at least have patches") - packages = {} - - for bug in bugs: - if bug[1] not in packages: - packages[bug[1]] = {} - # bug[0] = bug_id, bug[1] = source_name, bug[2] = who_when_done - if is_virtual_package(bug[1]): - continue - packages[bug[1]][bug[0]] = {'done': False, 'patch': False} - if bug[2]: # if the bug is done - packages[bug[1]][bug[0]]['done'] = True - try: - if bug_has_patch(bug[0]): - packages[bug[1]][bug[0]]['patch'] = True - except KeyError: - log.error('item: ' + str(bug)) - return packages - - - def gen_bugs_links(package, bugs): html = '' if package in bugs: @@ -219,7 +134,8 @@ def process_packages(packages, no_clean=False): generate the /rb-pkg/package.html page packages should be a list """ - bugs = parse_bugs(get_bugs()) + bugs = get_bugs() + log.debug(str(len(bugs)) + ' bugs found: ' + str(bugs)) total = len(packages) log.info('Generating the pages of ' + str(total) + ' package(s)') for pkg in sorted(packages): @@ -268,8 +184,3 @@ def purge_old_pages(): ' Removing old page.') os.remove(RB_PKG_PATH + '/' + page) - -try: - conn = start_udd_connection() -except: - raise |