#!/usr/bin/python3 # -*- coding: utf-8 -*- # # Copyright © 2015-2016 Mattia Rizzolo # Licensed under GPL-2 # # Depends: python3 # # Build a page full of CI issues to investigate from reproducible_common import * import time def unrep_with_dbd_issues(): log.info('running unrep_with_dbd_issues check...') without_dbd = [] bad_dbd = [] query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status="unreproducible" ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \ eversion + '.diffoscope.html' if not os.access(dbd, os.R_OK): without_dbd.append((pkg, version, suite, arch)) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') is ' 'unreproducible without diffoscope file.') else: log.debug(dbd + ' found.') data = open(dbd, 'br').read(3) if b'<' not in data: bad_dbd.append((pkg, version, suite, arch)) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') has ' 'diffoscope output, but it does not seem to ' 'be an html page.') return without_dbd, bad_dbd def not_unrep_with_dbd_file(): log.info('running not_unrep_with_dbd_file check...') bad_pkgs = [] query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status != "unreproducible" ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) dbd = DBD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \ eversion + '.diffoscope.html' if os.access(dbd, os.R_OK): bad_pkgs.append((pkg, version, suite, arch)) log.warning(dbd + ' exists but ' + suite + '/' + arch + '/' + pkg + ' (' + version + ')' ' is not unreproducible.') return bad_pkgs def lack_rbuild(): log.info('running lack_rbuild check...') bad_pkgs = [] query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status NOT IN ("blacklisted", "") ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: if not pkg_has_rbuild(pkg, version, suite, arch): bad_pkgs.append((pkg, version, suite, arch)) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') has been ' 'built, but a buildlog is missing.') return bad_pkgs def lack_buildinfo(): log.info('running lack_buildinfo check...') bad_pkgs = [] query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status NOT IN ("blacklisted", "not for us", "FTBFS", "depwait", "404", "") ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) buildinfo = BUILDINFO_PATH + '/' + suite + '/' + arch + '/' + pkg + \ '_' + eversion + '_' + arch + '.buildinfo' if not os.access(buildinfo, os.R_OK): bad_pkgs.append((pkg, version, suite, arch)) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') has been ' 'successfully built, but a .buildinfo is missing') return bad_pkgs def pbuilder_dep_fail(): log.info('running pbuilder_dep_fail check...') bad_pkgs = [] # we only care about these failures in the testing suite as they happen # all the time in other suites, as packages are buggy # and specific versions also come and go query = '''SELECT s.name, r.version, s.suite, s.architecture FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status = "FTBFS" AND s.suite = "testing" ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' results = query_db(query) for pkg, version, suite, arch in results: eversion = strip_epoch(version) rbuild = RBUILD_PATH + '/' + suite + '/' + arch + '/' + pkg + '_' + \ eversion + '.rbuild.log' if os.access(rbuild, os.R_OK): log.debug('\tlooking at ' + rbuild) with open(rbuild, "br") as fd: for line in fd: if re.search(b'E: pbuilder-satisfydepends failed.', line): bad_pkgs.append((pkg, version, suite, arch)) log.warning(suite + '/' + arch + '/' + pkg + ' (' + version + ') failed to satisfy its dependencies.') return bad_pkgs def alien_log(directory=None): if directory is None: bad_files = [] for path in RBUILD_PATH, LOGS_PATH, DIFFS_PATH: bad_files.extend(alien_log(path)) return bad_files log.info('running alien_log check over ' + directory + '...') query = '''SELECT r.version FROM sources AS s JOIN results AS r ON r.package_id=s.id WHERE r.status != "" AND s.name="{pkg}" AND s.suite="{suite}" AND s.architecture="{arch}" ORDER BY s.name ASC, s.suite DESC, s.architecture ASC''' bad_files = [] for root, dirs, files in os.walk(directory): if not files: continue suite, arch = root.rsplit('/', 2)[1:] for file in files: # different file have differnt name patterns and different splitting needs if file.endswith('.diff.gz'): rsplit_level = 2 elif file.endswith('.gz'): rsplit_level = 3 else: rsplit_level = 2 try: pkg, version = file.rsplit('.', rsplit_level)[0].rsplit('_', 1) except ValueError: log.critical(bcolors.FAIL + '/'.join([root, file]) + ' does not seem to be a file that should be there' + bcolors.ENDC) continue try: rversion = query_db(query.format(pkg=pkg, suite=suite, arch=arch))[0][0] except IndexError: # that package is not known (or not yet tested) rversion = '' # continue towards the "bad file" path if strip_epoch(rversion) != version: try: if os.path.getmtime('/'.join([root, file])) ' html += header html += '
\n'
        for pkg in pkgs:
            html += tab + link_package(pkg[0], pkg[2], pkg[3]).strip()
            html += ' (' + pkg[1] + ' in ' + pkg[2] + '/' + pkg[3] + ')\n'
    elif entries:
        html = '

' + str(len(entries)) + ' ' html += header html += '

\n'
        for entry in entries:
            html += tab + entry + '\n'
    html += '

\n' return html def gen_html(): html = '' # files that should not be there (e.g. removed package without cleanup) html += _gen_section('log files that should not be there:', None, entries=alien_log()) html += _gen_section('diffoscope files that should not be there:', None, entries=alien_dbd()) html += _gen_section('rb-pkg pages that should not be there:', None, entries=alien_rbpkg()) html += _gen_section('buildinfo files that should not be there:', None, entries=alien_buildinfo()) html += _gen_section('history tables that should not be there:', None, entries=alien_history()) # diffoscope report where it shouldn't be html += _gen_section('are not marked as unreproducible, but they ' + 'have a diffoscope file:', not_unrep_with_dbd_file()) # missing files html += _gen_section('have been built but don\'t have a buildlog:', lack_rbuild()) html += _gen_section('have been built but don\'t have a .buildinfo file:', lack_buildinfo()) # diffoscope troubles without_dbd, bad_dbd = unrep_with_dbd_issues() html += _gen_section('are marked as unreproducible, but there is no ' + 'diffoscope output - so probably diffoscope ' + 'crashed:', without_dbd) html += _gen_section('are marked as unreproducible, but their ' + 'diffoscope output does not seem to be an html ' + 'file - so probably diffoscope ran into a ' + 'timeout:', bad_dbd) # pbuilder-satisfydepends failed html += _gen_section('failed to satisfy their build-dependencies:', pbuilder_dep_fail()) return html if __name__ == '__main__': bugs = get_bugs() html = '

This page lists unexpected things a human should look at and ' html += 'fix, like packages with an incoherent status or files that ' html += 'should not be there. Some of these breakages are caused by ' html += 'bugs in diffoscope ' html += 'while others are probably due to bugs in the scripts run by jenkins. ' html += 'Please help making this page empty!

\n' breakages = gen_html() if breakages: html += breakages else: html += '

COOL!!! Everything is GOOD and not a single issue was ' html += 'detected. Enjoy!

' title = 'Breakage on the Debian pages of tests.reproducible-builds.org' destfile = DEBIAN_BASE + '/index_breakages.html' desturl = DEBIAN_URL + '/index_breakages.html' write_html_page(title, html, destfile, style_note=True, displayed_page='breakages') log.info('Breackages page created at ' + desturl)