reproducible: rewrite reproducible_html_dd_list into python. This allow us to get the "informative package links" on that page too

2 files changed, 70 insertions, 49 deletions

diff --git a/bin/reproducible_html_dd_list.py b/bin/reproducible_html_dd_list.py
new file mode 100755
index 00000000..7e7ce6b5
--- /dev/null
+++ b/bin/reproducible_html_dd_list.py
@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014 Holger Levsen <holger@layer-acht.org>
+#           © 2015 Mattia Rizzolo <mattia@mapreri.org>
+# Licensed under GPL-2
+#
+# Depends: python3
+#
+# Get the output of dd-list(1) and turn it into some nice html
+
+import lzma
+from urllib.request import urlopen
+from subprocess import Popen, PIPE
+from tempfile import NamedTemporaryFile
+
+from reproducible_common import *
+
+
+arch = 'amd64'
+mirror = 'http://ftp.de.debian.org/debian'
+
+bugs = get_bugs()
+
+for suite in SUITES:
+    remotefile = mirror + '/dists/' + suite + '/main/source/Sources.xz'
+    os.makedirs('/tmp/reproducible', exist_ok=True)
+    with NamedTemporaryFile(dir='/tmp/reproducible') as sources:
+        log.info('Downloading sources file for ' + suite + ': ' + remotefile)
+        xfile = lzma.decompress(urlopen(remotefile).read())
+        if xfile:
+            sources.write(xfile)
+        else:
+            log.error('Failed to get the ' + suite + 'sources')
+            continue
+        query = "SELECT s.name " + \
+                "FROM results AS r JOIN sources AS s ON r.package_id=s.id " + \
+                "WHERE r.status='unreproducible' AND s.suite='{suite}'"
+        try:
+            pkgs = [x[0] for x in query_db(query.format(suite=suite))]
+        except IndexError:
+            log.error('Looks like there are no unreproducible packages...')
+        p = Popen(('dd-list --stdin --sources ' + sources.name).split(),
+                  stdout=PIPE, stdin=PIPE, stderr=PIPE)
+        out, err = p.communicate(input=('\n'.join(pkgs)).encode())
+        if err:
+            log.error('dd-list errored:\n' + err.decode())
+        log.debug('dd-list output:\n' + out.decode())
+
+        html = '<p>The following maintainers and uploaders are listed '
+        html += 'for packages in ' + suite + ' which have built '
+        html += 'unreproducibly:</p>\n<p><pre>'
+        out = out.decode().splitlines()
+        for line in out:
+            if line[0:3] == '   ':
+                line = line.strip().split(None, 1)
+                html += '    '
+                # the final strip() is to avoid a newline
+                html += link_package(line[0], suite, arch, bugs).strip()
+                try:
+                    html += ' ' + line[1]  # eventual uploaders sign
+                except IndexError:
+                    pass
+            else:
+                html += HTML.escape(line.strip())
+            html += '\n'
+        html += '</pre></p>'
+        title = 'Maintainers of unreproducible packages in ' + suite
+        destfile = BASE + '/' + suite + '/index_dd-list.html'
+        write_html_page(title, html, destfile, suite, arch)
diff --git a/bin/reproducible_html_dd_list.sh b/bin/reproducible_html_dd_list.sh
deleted file mode 100755
index 4d819d9b..00000000
--- a/bin/reproducible_html_dd_list.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 Holger Levsen <holger@layer-acht.org>
-#         © 2015 Mattia Rizzolo <mattia@mapreri.org>
-# released under the GPLv=2
-
-DEBUG=false
-. /srv/jenkins/bin/common-functions.sh
-common_init "$@"
-
-# common code defining db access
-. /srv/jenkins/bin/reproducible_common.sh
-
-SUITE="unstable"
-ARCH="amd64"
-
-VIEW=dd-list
-for SUITE in $SUITES ; do
-	PAGE=index_${VIEW}.html
-	echo "$(date) - starting to write $PAGE page."
-	write_page_header $VIEW "Maintainers of unreproducible packages in $SUITE"
-	TMPFILE=$(mktemp --tmpdir=$TEMPDIR dd-list-XXXXXXXX)
-	SOURCES=$(mktemp --tmpdir=$TEMPDIR dd-list-XXXXXXXX)
-	schroot --directory /tmp -c source:jenkins-reproducible-$SUITE cat /var/lib/apt/lists/*_source_Sources > $SOURCES || \
-	    wget ${MIRROR}/dists/$SUITE/main/source/Sources.xz -O - | xzcat > $SOURCES
-	BAD=$(sqlite3 -init $INIT $PACKAGES_DB "SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE r.status='unreproducible' AND s.suite='$SUITE' ORDER BY r.build_date DESC" | xargs echo)
-	echo "${BAD}" | dd-list --stdin --sources $SOURCES > $TMPFILE || true
-	write_page "<p>The following maintainers and uploaders are listed for packages in $SUITE which have built unreproducibly:</p><p><pre>"
-	while IFS= read -r LINE ; do
-		if [ "${LINE:0:3}" = "   " ] ; then
-			PACKAGE=$(echo "${LINE:3}" | cut -d " " -f1)
-			UPLOADERS=$(echo "${LINE:3}" | cut -d " " -f2-)
-			if [ "$UPLOADERS" = "$PACKAGE" ] ; then
-				UPLOADERS=""
-			fi
-			write_page "   <a href=\"/rb-pkg/$SUITE/$ARCH/$PACKAGE.html\">$PACKAGE</a> $UPLOADERS"
-		else
-			LINE="$(echo $LINE | sed 's#&#\&amp;#g ; s#<#\&lt;#g ; s#>#\&gt;#g')"
-			write_page "$LINE"
-		fi
-	done < $TMPFILE
-	write_page "</pre></p>"
-	rm $TMPFILE
-	rm $SOURCES
-	write_page_footer
-	publish_page $SUITE
-	echo
-done
-