summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMattia Rizzolo <mattia@mapreri.org>2015-07-08 22:51:07 +0000
committerHolger Levsen <holger@layer-acht.org>2015-07-09 11:26:44 +0200
commite1a355b23ea0cad72ea1935f955376fe5230145a (patch)
tree621a1967001a9c3169664bdfdb7334f696c93a5d
parente4771abaf0d97fbe7c0356c59e25f9e27551a4e5 (diff)
downloadjenkins.debian.net-e1a355b23ea0cad72ea1935f955376fe5230145a.tar.xz
reproducible: rewrite reproducible_html_dd_list into python. This allow us to get the "informative package links" on that page too
-rwxr-xr-xbin/reproducible_html_dd_list.py70
-rwxr-xr-xbin/reproducible_html_dd_list.sh49
2 files changed, 70 insertions, 49 deletions
diff --git a/bin/reproducible_html_dd_list.py b/bin/reproducible_html_dd_list.py
new file mode 100755
index 00000000..7e7ce6b5
--- /dev/null
+++ b/bin/reproducible_html_dd_list.py
@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Holger Levsen <holger@layer-acht.org>
+# © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# Licensed under GPL-2
+#
+# Depends: python3
+#
+# Get the output of dd-list(1) and turn it into some nice html
+
+import lzma
+from urllib.request import urlopen
+from subprocess import Popen, PIPE
+from tempfile import NamedTemporaryFile
+
+from reproducible_common import *
+
+
+arch = 'amd64'
+mirror = 'http://ftp.de.debian.org/debian'
+
+bugs = get_bugs()
+
+for suite in SUITES:
+ remotefile = mirror + '/dists/' + suite + '/main/source/Sources.xz'
+ os.makedirs('/tmp/reproducible', exist_ok=True)
+ with NamedTemporaryFile(dir='/tmp/reproducible') as sources:
+ log.info('Downloading sources file for ' + suite + ': ' + remotefile)
+ xfile = lzma.decompress(urlopen(remotefile).read())
+ if xfile:
+ sources.write(xfile)
+ else:
+ log.error('Failed to get the ' + suite + 'sources')
+ continue
+ query = "SELECT s.name " + \
+ "FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \
+ "WHERE r.status='unreproducible' AND s.suite='{suite}'"
+ try:
+ pkgs = [x[0] for x in query_db(query.format(suite=suite))]
+ except IndexError:
+ log.error('Looks like there are no unreproducible packages...')
+ p = Popen(('dd-list --stdin --sources ' + sources.name).split(),
+ stdout=PIPE, stdin=PIPE, stderr=PIPE)
+ out, err = p.communicate(input=('\n'.join(pkgs)).encode())
+ if err:
+ log.error('dd-list errored:\n' + err.decode())
+ log.debug('dd-list output:\n' + out.decode())
+
+ html = '<p>The following maintainers and uploaders are listed '
+ html += 'for packages in ' + suite + ' which have built '
+ html += 'unreproducibly:</p>\n<p><pre>'
+ out = out.decode().splitlines()
+ for line in out:
+ if line[0:3] == ' ':
+ line = line.strip().split(None, 1)
+ html += ' '
+ # the final strip() is to avoid a newline
+ html += link_package(line[0], suite, arch, bugs).strip()
+ try:
+ html += ' ' + line[1] # eventual uploaders sign
+ except IndexError:
+ pass
+ else:
+ html += HTML.escape(line.strip())
+ html += '\n'
+ html += '</pre></p>'
+ title = 'Maintainers of unreproducible packages in ' + suite
+ destfile = BASE + '/' + suite + '/index_dd-list.html'
+ write_html_page(title, html, destfile, suite, arch)
diff --git a/bin/reproducible_html_dd_list.sh b/bin/reproducible_html_dd_list.sh
deleted file mode 100755
index 4d819d9b..00000000
--- a/bin/reproducible_html_dd_list.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 Holger Levsen <holger@layer-acht.org>
-# © 2015 Mattia Rizzolo <mattia@mapreri.org>
-# released under the GPLv=2
-
-DEBUG=false
-. /srv/jenkins/bin/common-functions.sh
-common_init "$@"
-
-# common code defining db access
-. /srv/jenkins/bin/reproducible_common.sh
-
-SUITE="unstable"
-ARCH="amd64"
-
-VIEW=dd-list
-for SUITE in $SUITES ; do
- PAGE=index_${VIEW}.html
- echo "$(date) - starting to write $PAGE page."
- write_page_header $VIEW "Maintainers of unreproducible packages in $SUITE"
- TMPFILE=$(mktemp --tmpdir=$TEMPDIR dd-list-XXXXXXXX)
- SOURCES=$(mktemp --tmpdir=$TEMPDIR dd-list-XXXXXXXX)
- schroot --directory /tmp -c source:jenkins-reproducible-$SUITE cat /var/lib/apt/lists/*_source_Sources > $SOURCES || \
- wget ${MIRROR}/dists/$SUITE/main/source/Sources.xz -O - | xzcat > $SOURCES
- BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.status='unreproducible' AND s.suite='$SUITE' ORDER BY r.build_date DESC" | xargs echo)
- echo "${BAD}" | dd-list --stdin --sources $SOURCES > $TMPFILE || true
- write_page "<p>The following maintainers and uploaders are listed for packages in $SUITE which have built unreproducibly:</p><p><pre>"
- while IFS= read -r LINE ; do
- if [ "${LINE:0:3}" = " " ] ; then
- PACKAGE=$(echo "${LINE:3}" | cut -d " " -f1)
- UPLOADERS=$(echo "${LINE:3}" | cut -d " " -f2-)
- if [ "$UPLOADERS" = "$PACKAGE" ] ; then
- UPLOADERS=""
- fi
- write_page " <a href=\"/rb-pkg/$SUITE/$ARCH/$PACKAGE.html\">$PACKAGE</a> $UPLOADERS"
- else
- LINE="$(echo $LINE | sed 's#&#\&amp;#g ; s#<#\&lt;#g ; s#>#\&gt;#g')"
- write_page "$LINE"
- fi
- done < $TMPFILE
- write_page "</pre></p>"
- rm $TMPFILE
- rm $SOURCES
- write_page_footer
- publish_page $SUITE
- echo
-done
-