diff options
| -rwxr-xr-x | bin/reproducible_build.sh | 2 | ||||
| -rwxr-xr-x | bin/reproducible_common.sh | 8 | ||||
| -rwxr-xr-x | bin/reproducible_html_dd_list.sh | 2 | ||||
| -rwxr-xr-x | bin/reproducible_html_graphs.sh | 10 | ||||
| -rwxr-xr-x | bin/reproducible_html_indexes.py | 51 | ||||
| -rwxr-xr-x | bin/reproducible_html_notes.py | 9 | ||||
| -rwxr-xr-x | bin/reproducible_html_packages.py | 37 |
7 files changed, 71 insertions, 48 deletions
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh index ce4da14f..c74d416e 100755 --- a/bin/reproducible_build.sh +++ b/bin/reproducible_build.sh @@ -41,7 +41,7 @@ update_db_and_html() { set +x gen_packages_html $SRCPACKAGE echo - echo "Successfully updated the database and updated $REPRODUCIBLE_URL/rb-pkg/$SRCPACKAGE.html" + echo "Successfully updated the database and updated $REPRODUCIBLE_URL/rb-pkg/${SUITE}/$SRCPACKAGE.html" echo } diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index 5b8b62f4..363ca745 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -199,10 +199,14 @@ set_package_class() { } set_linktarget() { + local SUITE="$1" + local ARCH="$2" + shift + shift for PKG in $@ ; do - if [ -f $RB_PATH/$PKG.html ] ; then + if [ -f $RB_PATH/$SUITE/$PKG.html ] ; then set_package_class - LINKTARGET[$PKG]="<a href=\"/userContent/rb-pkg/$PKG.html\" $CLASS>$PKG</a>" + LINKTARGET[$PKG]="<a href=\"/userContent/rb-pkg/$SUITE/$ARCH/$PKG.html\" $CLASS>$PKG</a>" else LINKTARGET[$PKG]="$PKG" fi diff --git a/bin/reproducible_html_dd_list.sh b/bin/reproducible_html_dd_list.sh index 2ae21bc4..518a9e24 100755 --- a/bin/reproducible_html_dd_list.sh +++ b/bin/reproducible_html_dd_list.sh @@ -29,7 +29,7 @@ while IFS= read -r LINE ; do if [ "$UPLOADERS" = "$PACKAGE" ] ; then UPLOADERS="" fi - write_page " <a href=\"/rb-pkg/$PACKAGE.html\">$PACKAGE</a> $UPLOADERS" + write_page " <a href=\"/rb-pkg/sid/$PACKAGE.html\">$PACKAGE</a> $UPLOADERS" else LINE="$(echo $LINE | sed 's#&#\&#g ; s#<#\<#g ; s#>#\>#g')" write_page "$LINE" diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_graphs.sh index 1c177726..c9912c73 100755 --- a/bin/reproducible_html_graphs.sh +++ b/bin/reproducible_html_graphs.sh @@ -15,6 +15,7 @@ init_html gather_stats SUITE="sid" # we only care about sid status here (for now) +ARCH="amd64" # we only care about amd64 status here (for now) # # create stats @@ -269,6 +270,7 @@ write_icon write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64' will not be build here" write_page "and those " set_icon blacklisted + write_icon write_page "$COUNT_BLACKLISTED blacklisted packages neither.</p>" write_page "<p>" @@ -315,20 +317,20 @@ for i in $(seq 1 ${#META_PKGSET[@]}) ; do set_icon reproducible write_icon write_page "$COUNT_META_GOOD packages ($PERCENT_META_GOOD%) successfully built reproducibly:" - set_linktarget $META_GOOD + set_linktarget $SUITE $ARCH $META_GOOD link_packages $META_GOOD write_page "<br />" set_icon unreproducible write_icon write_page "$COUNT_META_BAD ($PERCENT_META_BAD%) packages failed to built reproducibly:" - set_linktarget $META_BAD + set_linktarget $SUITE $ARCH $META_BAD link_packages $META_BAD write_page "<br />" if [ $COUNT_META_UGLY -gt 0 ] ; then set_icon FTBFS write_icon write_page "$COUNT_META_UGLY ($PERCENT_META_UGLY%) packages failed to build from source:" - set_linktarget $META_UGLY + set_linktarget $SUITE $ARCH $META_UGLY link_packages $META_UGLY write_page "<br />" fi @@ -340,7 +342,7 @@ for i in $(seq 1 ${#META_PKGSET[@]}) ; do set_icon 404 write_icon write_page "$COUNT_META_REST ($PERCENT_META_REST%) packages are either blacklisted, not for us or cannot be downloaded:" - set_linktarget $META_REST + set_linktarget $SUITE $ARCH $META_REST link_packages $META_REST write_page "<br />" fi diff --git a/bin/reproducible_html_indexes.py b/bin/reproducible_html_indexes.py index 43e078ed..d02b4fe5 100755 --- a/bin/reproducible_html_indexes.py +++ b/bin/reproducible_html_indexes.py @@ -15,6 +15,8 @@ from reproducible_common import * Reference doc for the folowing lists: * queries is just a list of queries. They are referred further below. + + every query return the following tuple: + (package_name, suite, architecture) * pages is just a list of pages. It is actually a dictionary, where every element is a page. Every page has: + `title`: The page title @@ -38,24 +40,24 @@ section must have at least a `query` defining what to file in. """ queries = { - 'scheduled': 'SELECT sources.name FROM schedule JOIN sources ON schedule.package_id=sources.id ORDER BY schedule.date_scheduled', - 'reproducible_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY r.build_date DESC', - 'reproducible_last24h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-24 hours") ORDER BY r.build_date DESC', - 'reproducible_last48h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-48 hours") ORDER BY r.build_date DESC', - 'reproducible_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY name', - 'FTBR_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY build_date DESC', - 'FTBR_last24h': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC', - 'FTBR_last48h': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC', - 'FTBR_all_abc': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY name', - 'FTBFS_all': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY build_date DESC', - 'FTBFS_last24h': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC', - 'FTBFS_last48h': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC', - 'FTBFS_all_abc': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY name', - '404_all': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY build_date DESC', - '404_all_abc': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY name', - 'not_for_us_all': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY build_date DESC', - 'not_for_us_all_abc': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY name', - 'blacklisted_all': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "blacklisted" ORDER BY name' + 'scheduled': 'SELECT sources.name, sources.suite, sources.architecture FROM schedule JOIN sources ON schedule.package_id=sources.id ORDER BY schedule.date_scheduled', + 'reproducible_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY r.build_date DESC', + 'reproducible_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-24 hours") ORDER BY r.build_date DESC', + 'reproducible_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-48 hours") ORDER BY r.build_date DESC', + 'reproducible_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY name', + 'FTBR_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY build_date DESC', + 'FTBR_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC', + 'FTBR_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC', + 'FTBR_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY name', + 'FTBFS_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY build_date DESC', + 'FTBFS_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC', + 'FTBFS_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC', + 'FTBFS_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY name', + '404_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY build_date DESC', + '404_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY name', + 'not_for_us_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY build_date DESC', + 'not_for_us_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY name', + 'blacklisted_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "blacklisted" ORDER BY name' } pages = { @@ -285,15 +287,18 @@ def build_page_section(section): return html # do not output anything on the page. html += build_leading_text_section(section, rows) html += '<p>\n' + tab + '<code>\n' - for pkg in rows: - url = RB_PKG_URI + '/' + pkg[0] + '.html' + for row in rows: + pkg = row[0] + #suite = row[1] # FIXME currently we build indexes pages only for sid + arch = row[2] + url = RB_PKG_URI + '/' + suite + '/' + arch + '/' + pkg + '.html' html += tab*2 + '<a href="' + url + '" class="' - if package_has_notes(pkg[0]): + if package_has_notes(pkg): html += 'noted' else: html += 'package' - html += '">' + pkg[0] + '</a>' - html += get_trailing_icon(pkg[0], bugs) + '\n' + html += '">' + pkg + '</a>' + html += get_trailing_icon(pkg, bugs) + '\n' html += tab + '</code>\n' html += '</p>' html = (tab*2).join(html.splitlines(True)) diff --git a/bin/reproducible_html_notes.py b/bin/reproducible_html_notes.py index 56acd3ff..456fc4f9 100755 --- a/bin/reproducible_html_notes.py +++ b/bin/reproducible_html_notes.py @@ -242,7 +242,8 @@ def gen_html_issue(issue): try: affected += tab*4 + '<b>' + str(len(issues_count[issue])) + '</b>:\n' for pkg in sorted(issues_count[issue]): - affected += tab*7 + '<a href="%s/%s.html" class="noted">%s</a>\n' \ + # FIXME we currently consider notes only for sid + affected += tab*7 + '<a href="%s/sid/%s.html" class="noted">%s</a>\n' \ % (RB_PKG_URI, pkg, pkg) except KeyError: # The note is not listed in any package, that is affected = '<i>None</i>' @@ -373,7 +374,8 @@ def index_notes(notes, bugs): html += tab + str(len(pkgs)) + ' ' + status + ' packages:\n' html += tab + '<code>\n' for pkg in pkgs: - url = RB_PKG_URI + '/' + pkg + '.html' + # FIXME we currently consider notes only for sid + url = RB_PKG_URI + '/sid/' + pkg + '.html' html += tab*2 + '<a href="' + url + '" class="noted">' + pkg html += '</a>' + get_trailing_icon(pkg, bugs) + '\n' html += tab + '</code>\n' @@ -410,7 +412,8 @@ def index_no_notes(notes, bugs): html += tab + str(len(pkgs)) + ' ' + status + ' packages:\n' html += tab + '<code>\n' for pkg in pkgs: - url = RB_PKG_URI + '/' + pkg + '.html' + # FIXME we currently consider notes only for sid + url = RB_PKG_URI + '/sid/' + pkg + '.html' html += tab*2 + '<a href="' + url + '" class="package">' + pkg html += '</a>' + get_trailing_icon(pkg, bugs) + '\n' html += tab + '</code>\n' diff --git a/bin/reproducible_html_packages.py b/bin/reproducible_html_packages.py index 346d567d..a97d6573 100755 --- a/bin/reproducible_html_packages.py +++ b/bin/reproducible_html_packages.py @@ -166,8 +166,9 @@ def gen_packages_html(packages, suite='sid', arch='amd64', no_clean=False): links=links, bugs_links=bugs_links, default_view=default_view) - destfile = RB_PKG_PATH + '/' + pkg + '.html' - desturl = REPRODUCIBLE_URL + RB_PKG_URI + '/' + pkg + '.html' + destfile = RB_PKG_PATH + '/' + suite + '/' + arch + '/' + pkg + '.html' + desturl = REPRODUCIBLE_URL + RB_PKG_URI + '/' + suite + '/' + \ + arch + '/' + pkg + '.html' title = pkg + ' - reproducible build results' write_html_page(title=title, body=html, destfile=destfile, noheader=True, noendpage=True) @@ -185,16 +186,24 @@ def gen_all_rb_pkg_pages(suite='sid', arch='amd64', no_clean=False): gen_packages_html(pkgs, suite=suite, arch=arch, no_clean=no_clean) def purge_old_pages(): - presents = sorted(os.listdir(RB_PKG_PATH)) - for page in presents: - pkg = page.rsplit('.', 1)[0] - log.debug('Checking if ' + page + ' (from ' + pkg + ') is still needed') - query = 'SELECT s.name ' + \ - 'FROM results AS r JOIN sources AS s ON r.package_id=s.id ' + \ - 'WHERE s.name="%s" AND r.status != "" AND s.suite="sid"' % pkg - result = query_db(query) - if not result: # actually, the query produces no results - log.info('There is no package named ' + pkg + ' in the database.' + - ' Removing old page.') - os.remove(RB_PKG_PATH + '/' + page) + for suite in SUITES: + for arch in ARCHES: + log.info('Removing old pages from ' + suite + '...') + presents = sorted(os.listdir(RB_PKG_PATH + '/' + suite + '/' + + arch)) + for page in presents: + pkg = page.rsplit('.', 1)[0] + query = 'SELECT s.name ' + \ + 'FROM results AS r ' + \ + 'JOIN sources AS s ON r.package_id=s.id ' + \ + 'WHERE s.name="{name}" AND r.status != "" ' + \ + 'AND s.suite="{suite}" AND s.architecture="{arch}"' + query = query.format(name=pkg, suite=suite, arch=arch) + result = query_db(query) + if not result: # actually, the query produces no results + log.info('There is no package named ' + pkg + ' from ' + + suite + '/' + arch + ' in the database. ' + + 'Removing old page.') + os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' + + page) |