#!/usr/bin/python3 # -*- coding: utf-8 -*- # # Copyright © 2015 Mattia Rizzolo # Licensed under GPL-2 # # Depends: python3 # # Build a page full of CI issues to investigate from reproducible_common import * class bcolors: BOLD = '\033[1m' UNDERLINE = '\033[4m' WARN = '\033[93m' + UNDERLINE FAIL = '\033[91m' + BOLD + UNDERLINE ENDC = '\033[0m' def unrep_with_dbd_issues(): log.info('running unrep_with_dbd_issues check...') without_dbd = [] bad_dbd = [] query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status="unreproducible" ORDER BY s.name ASC, s.suite DESC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \ eversion + '.debbindiff.html' if not os.access(dbd, os.R_OK): without_dbd.append((pkg, version, suite, arch)) log.warning(pkg + '/' + suite + ' (' + version + ') is ' 'unreproducible without debbindiff file.') else: log.debug(dbd + ' found.') data = open(dbd, 'br').read(3) if b'<' not in data: bad_dbd.append((pkg, version, suite, arch)) log.warning(pkg + '/' + suite + ' (' + version + ') has ' 'debbindiff output, but it does not seem to ' 'be an html page.') return without_dbd, bad_dbd def not_unrep_with_dbd_file(): log.info('running not_unrep_with_dbd_file check...') bad_pkgs = [] query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status != "unreproducible" ORDER BY s.name ASC, s.suite DESC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \ eversion + '.debbindiff.html' if os.access(dbd, os.R_OK): bad_pkgs.append((pkg, version, suite, arch)) log.warning(pkg + '/' + suite + ' (' + version + ') has a ' 'debbindiff file but it\'s not unreproducible.') return bad_pkgs def lack_rbuild(): log.info('running lack_rbuild check...') bad_pkgs = [] query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status NOT IN ("blacklisted", "not for us", "") ORDER BY s.name ASC, s.suite DESC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) rbuild = RBUILD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \ eversion + '.rbuild.log' if not os.access(rbuild, os.R_OK): bad_pkgs.append((pkg, version, suite, arch)) log.warning(pkg + '/' + suite + ' (' + version + ') has been ' 'built, but a buildlog is missing.') return bad_pkgs def lack_buildinfo(): log.info('running lack_buildinfo check...') bad_pkgs = [] query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status NOT IN ("blacklisted", "not for us", "FTBFS", "") ORDER BY s.name ASC, s.suite DESC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) buildinfo = BUILDINFO_PATH + '/' + suite + '/' + arch + '/' + pkg + \ '_' + eversion + '_' + arch + '.buildinfo' if not os.access(buildinfo, os.R_OK): bad_pkgs.append((pkg, version, suite, arch)) log.warning(pkg + '/' + suite + ' (' + version + ') has been ' 'successfully built, but a .buildinfo is missing') return bad_pkgs def pbuilder_dep_fail(): log.info('running pbuilder_dep_fail check...') bad_pkgs = [] query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status = "FTBFS" ORDER BY s.name ASC, s.suite DESC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) rbuild = RBUILD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \ eversion + '.rbuild.log' if os.access(rbuild, os.R_OK): log.debug('\tlooking at ' + rbuild) with open(rbuild, "br") as fd: for line in fd: if re.search(b'E: pbuilder-satisfydepends failed.', line): bad_pkgs.append((pkg, version, suite, arch)) log.warning(pkg + '/' + suite + ' (' + version + ') failed to met its dependencies.') return bad_pkgs def alien_rbuild(): log.info('running alien_rbuild check...') query = '''SELECT s.name FROM sources AS s JOIN results AS r on r.package_id=s.id WHERE r.status != "" AND s.name="{pkg}" AND s.suite="{suite}" AND s.architecture="{arch}" ORDER BY s.name ASC, s.suite DESC''' bad_files = [] for root, dirs, files in os.walk(RBUILD_PATH): if not files: continue suite, arch = root.rsplit('/', 2)[1:] for file in files: try: pkg, version = file.rsplit('.', 2)[0].rsplit('_', 1) except ValueError: log.critical(bcolors.FAIL + '/'.join([root, file]) + ' does not seem to be a file that should be there' + bcolors.ENDC) if not query_db(query.format(pkg=pkg, suite=suite, arch=arch)): bad_files.append('/'.join([root, file])) log.warning('/'.join([root, file]) + ' should not be there') return bad_files def alien_dbd(): log.info('running alien_dbd check...') query = '''SELECT r.status FROM sources AS s JOIN results AS r on r.package_id=s.id WHERE s.name="{pkg}" AND s.suite="{suite}" AND s.architecture="{arch}" ORDER BY s.name ASC, s.suite DESC''' bad_files = [] for root, dirs, files in os.walk(DBD_PATH): if not files: continue suite, arch = root.rsplit('/', 2)[1:] for file in files: try: pkg, version = file.rsplit('.', 2)[0].rsplit('_', 1) except ValueError: log.critical(bcolors.FAIL + '/'.join([root, file]) + ' does not seem to be a file that should be there' + bcolors.ENDC) result = query_db(query.format(pkg=pkg, suite=suite, arch=arch)) try: if result[0][0] != 'unreproducible': bad_files.append('/'.join([root, file]) + ' (' + str(result[0][0]) + ' package)') log.warning('/'.join([root, file]) + ' should not be ' 'there (' + str(result[0][0]) + ' package)') except IndexError: bad_files.append('/'.join([root, file]) + ' (' + 'missing package)') log.warning(bcolors.WARN + '/'.join([root, file]) + ' should ' 'not be there (missing package)' + bcolors.ENDC) return bad_files def alien_rbpkg(): log.info('running alien_rbpkg check...') query = '''SELECT s.name FROM sources AS s WHERE s.name="{pkg}" AND s.suite="{suite}" AND s.architecture="{arch}" ORDER BY s.name ASC, s.suite DESC''' bad_files = [] for root, dirs, files in os.walk(RB_PKG_PATH): if not files: continue suite, arch = root.rsplit('/', 2)[1:] for file in files: pkg = file.rsplit('.', 1)[0] if not query_db(query.format(pkg=pkg, suite=suite, arch=arch)): bad_files.append('/'.join([root, file])) log.warning('/'.join([root, file]) + ' should not be there') return bad_files def _gen_section(header, pkgs, entries=None): if not pkgs and not entries: return '' if pkgs: html = '

' + str(len(pkgs)) + ' ' html += header html += '

\n'
        for pkg in pkgs:
            html += tab + link_package(pkg[0], pkg[2], pkg[3]).strip()
            html += ' (' + pkg[1] + ' in ' + pkg[2] + '/' + pkg[3] + ')\n'
    elif entries:
        html = '

' + str(len(entries)) + ' ' html += header html += '

\n'
        for entry in entries:
            html += tab + entry + '\n'
    html += '

\n' return html def gen_html(): html = '' # files that should not be there (e.g. removed package without cleanup) html += _gen_section('rbuild file that should not be there', None, entries=alien_rbuild()) html += _gen_section('debbindiff files that should not be there:', None, entries=alien_dbd()) html += _gen_section('rb-pkg pages that should not be there:', None, entries=alien_rbpkg()) # debbindiff troubles without_dbd, bad_dbd = unrep_with_dbd_issues() html += _gen_section('are marked as unreproducible, but without ' + 'debbindiff output:', without_dbd) html += _gen_section('are marked as unreproducible, but their ' + 'debbindiff output does not seem to be an html ' + 'file:', bad_dbd) # debbindiff report where it shouldn't be html += _gen_section('are not marked as unreproducible, but they ' + 'have a debbindiff file:', not_unrep_with_dbd_file()) # missing files html += _gen_section('are built but don\'t have a buildlog:', lack_rbuild()) html += _gen_section('are built but don\'t have a .buildinfo file:', lack_buildinfo()) # pbuilder-satisfydepends failed html += _gen_section('failed to match their build-dependencies:', pbuilder_dep_fail()) return html if __name__ == '__main__': bugs = get_bugs() html = '

This page lists unexpected things a human should look at and ' html += 'fix, like packages with an incoherent status or files that ' html += 'should not be there.
Please help with empting this page!

' breakages = gen_html() if breakages: html += breakages else: html += '

COOL!!! Everything is GOOD and not a issue was ' html += 'detected. Enjoy!

' title = 'CI breakages' destfile = BASE + '/index_breakages.html' desturl = REPRODUCIBLE_URL + '/index_breakages.html' write_html_page(title, html, destfile, style_note=True) log.info('CI breackages page created at ' + desturl)