reproducible: move packages files under a suite-named directory, e.g. /rb-pkg/sid/

7 files changed, 71 insertions, 48 deletions

diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index ce4da14f..c74d416e 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -41,7 +41,7 @@ update_db_and_html() {
 	set +x
 	gen_packages_html $SRCPACKAGE
 	echo
-	echo "Successfully updated the database and updated $REPRODUCIBLE_URL/rb-pkg/$SRCPACKAGE.html"
+	echo "Successfully updated the database and updated $REPRODUCIBLE_URL/rb-pkg/${SUITE}/$SRCPACKAGE.html"
 	echo
 }
 
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index 5b8b62f4..363ca745 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -199,10 +199,14 @@ set_package_class() {
 }
 
 set_linktarget() {
+	local SUITE="$1"
+	local ARCH="$2"
+	shift
+	shift
 	for PKG in $@ ; do
-		if [ -f $RB_PATH/$PKG.html ] ; then
+		if [ -f $RB_PATH/$SUITE/$PKG.html ] ; then
 			set_package_class
-			LINKTARGET[$PKG]="<a href=\"/userContent/rb-pkg/$PKG.html\" $CLASS>$PKG</a>"
+			LINKTARGET[$PKG]="<a href=\"/userContent/rb-pkg/$SUITE/$ARCH/$PKG.html\" $CLASS>$PKG</a>"
 		else
 			LINKTARGET[$PKG]="$PKG"
 		fi
diff --git a/bin/reproducible_html_dd_list.sh b/bin/reproducible_html_dd_list.sh
index 2ae21bc4..518a9e24 100755
--- a/bin/reproducible_html_dd_list.sh
+++ b/bin/reproducible_html_dd_list.sh
@@ -29,7 +29,7 @@ while IFS= read -r LINE ; do
 		if [ "$UPLOADERS" = "$PACKAGE" ] ; then
 			UPLOADERS=""
 		fi
-		write_page "   <a href=\"/rb-pkg/$PACKAGE.html\">$PACKAGE</a> $UPLOADERS"
+		write_page "   <a href=\"/rb-pkg/sid/$PACKAGE.html\">$PACKAGE</a> $UPLOADERS"
 	else
 		LINE="$(echo $LINE | sed 's#&#\&amp;#g ; s#<#\&lt;#g ; s#>#\&gt;#g')"
 		write_page "$LINE"
diff --git a/bin/reproducible_html_graphs.sh b/bin/reproducible_html_graphs.sh
index 1c177726..c9912c73 100755
--- a/bin/reproducible_html_graphs.sh
+++ b/bin/reproducible_html_graphs.sh
@@ -15,6 +15,7 @@ init_html
 gather_stats
 
 SUITE="sid"  # we only care about sid status here (for now)
+ARCH="amd64"  # we only care about amd64 status here (for now)
 
 #
 # create stats
@@ -269,6 +270,7 @@ write_icon
 write_page "$COUNT_NOTFORUS ($PERCENT_NOTFORUS%) packages which are neither Architecture: 'any', 'all', 'amd64', 'linux-any', 'linux-amd64' nor 'any-amd64' will not be build here"
 write_page "and those "
 set_icon blacklisted
+
 write_icon
 write_page "$COUNT_BLACKLISTED blacklisted packages neither.</p>"
 write_page "<p>"
@@ -315,20 +317,20 @@ for i in $(seq 1 ${#META_PKGSET[@]}) ; do
 		set_icon reproducible
 		write_icon
 		write_page "$COUNT_META_GOOD packages ($PERCENT_META_GOOD%) successfully built reproducibly:"
-		set_linktarget $META_GOOD
+		set_linktarget $SUITE $ARCH $META_GOOD
 		link_packages $META_GOOD
 		write_page "<br />"
 		set_icon unreproducible
 		write_icon
 		write_page "$COUNT_META_BAD ($PERCENT_META_BAD%) packages failed to built reproducibly:"
-		set_linktarget $META_BAD
+		set_linktarget $SUITE $ARCH $META_BAD
 		link_packages $META_BAD
 		write_page "<br />"
 		if [ $COUNT_META_UGLY -gt 0 ] ; then
 			set_icon FTBFS
 			write_icon
 			write_page "$COUNT_META_UGLY ($PERCENT_META_UGLY%) packages failed to build from source:"
-			set_linktarget $META_UGLY
+			set_linktarget $SUITE $ARCH $META_UGLY
 			link_packages $META_UGLY
 			write_page "<br />"
 		fi
@@ -340,7 +342,7 @@ for i in $(seq 1 ${#META_PKGSET[@]}) ; do
 			set_icon 404
 			write_icon
 			write_page "$COUNT_META_REST ($PERCENT_META_REST%) packages are either blacklisted, not for us or cannot be downloaded:"
-			set_linktarget $META_REST
+			set_linktarget $SUITE $ARCH $META_REST
 			link_packages $META_REST
 			write_page "<br />"
 		fi
diff --git a/bin/reproducible_html_indexes.py b/bin/reproducible_html_indexes.py
index 43e078ed..d02b4fe5 100755
--- a/bin/reproducible_html_indexes.py
+++ b/bin/reproducible_html_indexes.py
@@ -15,6 +15,8 @@ from reproducible_common import *
 Reference doc for the folowing lists:
 
 * queries is just a list of queries. They are referred further below.
+  + every query return the following tuple:
+    (package_name, suite, architecture)
 * pages is just a list of pages. It is actually a dictionary, where every
   element is a page. Every page has:
   + `title`: The page title
@@ -38,24 +40,24 @@ section must have at least a `query` defining what to file in.
 """
 
 queries = {
-    'scheduled': 'SELECT sources.name FROM schedule JOIN sources ON schedule.package_id=sources.id ORDER BY schedule.date_scheduled',
-    'reproducible_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY r.build_date DESC',
-    'reproducible_last24h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-24 hours") ORDER BY r.build_date DESC',
-    'reproducible_last48h': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-48 hours") ORDER BY r.build_date DESC',
-    'reproducible_all_abc': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY name',
-    'FTBR_all': 'SELECT s.name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY build_date DESC',
-    'FTBR_last24h': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
-    'FTBR_last48h': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
-    'FTBR_all_abc': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY name',
-    'FTBFS_all': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY build_date DESC',
-    'FTBFS_last24h': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
-    'FTBFS_last48h': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
-    'FTBFS_all_abc': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY name',
-    '404_all': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY build_date DESC',
-    '404_all_abc': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY name',
-    'not_for_us_all': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY build_date DESC',
-    'not_for_us_all_abc': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY name',
-    'blacklisted_all': 'SELECT name FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "blacklisted" ORDER BY name'
+    'scheduled': 'SELECT sources.name, sources.suite, sources.architecture FROM schedule JOIN sources ON schedule.package_id=sources.id ORDER BY schedule.date_scheduled',
+    'reproducible_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY r.build_date DESC',
+    'reproducible_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-24 hours") ORDER BY r.build_date DESC',
+    'reproducible_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" AND r.build_date > datetime("now", "-48 hours") ORDER BY r.build_date DESC',
+    'reproducible_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND r.status="reproducible" ORDER BY name',
+    'FTBR_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY build_date DESC',
+    'FTBR_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
+    'FTBR_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
+    'FTBR_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "unreproducible" ORDER BY name',
+    'FTBFS_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY build_date DESC',
+    'FTBFS_last24h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-24 hours") ORDER BY build_date DESC',
+    'FTBFS_last48h': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" AND build_date > datetime("now", "-48 hours") ORDER BY build_date DESC',
+    'FTBFS_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "FTBFS" ORDER BY name',
+    '404_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY build_date DESC',
+    '404_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "404" ORDER BY name',
+    'not_for_us_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY build_date DESC',
+    'not_for_us_all_abc': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "not for us" ORDER BY name',
+    'blacklisted_all': 'SELECT s.name, s.suite, s.architecture FROM results AS r JOIN sources AS s ON r.package_id=s.id WHERE s.suite="{suite}" AND status = "blacklisted" ORDER BY name'
 }
 
 pages = {
@@ -285,15 +287,18 @@ def build_page_section(section):
         return html  # do not output anything on the page.
     html += build_leading_text_section(section, rows)
     html += '<p>\n' + tab + '<code>\n'
-    for pkg in rows:
-        url = RB_PKG_URI + '/' + pkg[0] + '.html'
+    for row in rows:
+        pkg = row[0]
+        #suite = row[1]  # FIXME currently we build indexes pages only for sid
+        arch = row[2]
+        url = RB_PKG_URI + '/' + suite + '/' + arch + '/' + pkg + '.html'
         html += tab*2 + '<a href="' + url + '" class="'
-        if package_has_notes(pkg[0]):
+        if package_has_notes(pkg):
             html += 'noted'
         else:
             html += 'package'
-        html += '">' + pkg[0] + '</a>'
-        html += get_trailing_icon(pkg[0], bugs) + '\n'
+        html += '">' + pkg + '</a>'
+        html += get_trailing_icon(pkg, bugs) + '\n'
     html += tab + '</code>\n'
     html += '</p>'
     html = (tab*2).join(html.splitlines(True))
diff --git a/bin/reproducible_html_notes.py b/bin/reproducible_html_notes.py
index 56acd3ff..456fc4f9 100755
--- a/bin/reproducible_html_notes.py
+++ b/bin/reproducible_html_notes.py
@@ -242,7 +242,8 @@ def gen_html_issue(issue):
     try:
         affected += tab*4 + '<b>' + str(len(issues_count[issue])) + '</b>:\n'
         for pkg in sorted(issues_count[issue]):
-            affected += tab*7 + '<a href="%s/%s.html" class="noted">%s</a>\n' \
+            # FIXME we currently consider notes only for sid
+            affected += tab*7 + '<a href="%s/sid/%s.html" class="noted">%s</a>\n' \
                         % (RB_PKG_URI, pkg, pkg)
     except KeyError:    # The note is not listed in any package, that is
         affected = '<i>None</i>'
@@ -373,7 +374,8 @@ def index_notes(notes, bugs):
         html += tab + str(len(pkgs)) + ' ' + status + ' packages:\n'
         html += tab + '<code>\n'
         for pkg in pkgs:
-            url = RB_PKG_URI + '/' + pkg + '.html'
+            # FIXME we currently consider notes only for sid
+            url = RB_PKG_URI + '/sid/' + pkg + '.html'
             html += tab*2 + '<a href="' + url + '" class="noted">' + pkg
             html += '</a>' + get_trailing_icon(pkg, bugs) + '\n'
         html += tab + '</code>\n'
@@ -410,7 +412,8 @@ def index_no_notes(notes, bugs):
         html += tab + str(len(pkgs)) + ' ' + status + ' packages:\n'
         html += tab + '<code>\n'
         for pkg in pkgs:
-            url = RB_PKG_URI + '/' + pkg + '.html'
+            # FIXME we currently consider notes only for sid
+            url = RB_PKG_URI + '/sid/' + pkg + '.html'
             html += tab*2 + '<a href="' + url + '" class="package">' + pkg
             html += '</a>' + get_trailing_icon(pkg, bugs) + '\n'
         html += tab + '</code>\n'
diff --git a/bin/reproducible_html_packages.py b/bin/reproducible_html_packages.py
index 346d567d..a97d6573 100755
--- a/bin/reproducible_html_packages.py
+++ b/bin/reproducible_html_packages.py
@@ -166,8 +166,9 @@ def gen_packages_html(packages, suite='sid', arch='amd64', no_clean=False):
                                             links=links,
                                             bugs_links=bugs_links,
                                             default_view=default_view)
-        destfile = RB_PKG_PATH + '/' + pkg + '.html'
-        desturl = REPRODUCIBLE_URL + RB_PKG_URI + '/' + pkg + '.html'
+        destfile = RB_PKG_PATH + '/' + suite + '/' + arch + '/' + pkg + '.html'
+        desturl = REPRODUCIBLE_URL + RB_PKG_URI + '/' + suite + '/' + \
+                  arch + '/' + pkg + '.html'
         title = pkg + ' - reproducible build results'
         write_html_page(title=title, body=html, destfile=destfile,
                         noheader=True, noendpage=True)
@@ -185,16 +186,24 @@ def gen_all_rb_pkg_pages(suite='sid', arch='amd64', no_clean=False):
     gen_packages_html(pkgs, suite=suite, arch=arch, no_clean=no_clean)
 
 def purge_old_pages():
-    presents = sorted(os.listdir(RB_PKG_PATH))
-    for page in presents:
-        pkg = page.rsplit('.', 1)[0]
-        log.debug('Checking if ' + page + ' (from ' + pkg + ') is still needed')
-        query = 'SELECT s.name ' + \
-                'FROM results AS r JOIN sources AS s ON r.package_id=s.id ' + \
-                'WHERE s.name="%s" AND r.status != "" AND s.suite="sid"' % pkg
-        result = query_db(query)
-        if not result: # actually, the query produces no results
-            log.info('There is no package named ' + pkg + ' in the database.' +
-                     ' Removing old page.')
-            os.remove(RB_PKG_PATH + '/' + page)
+    for suite in SUITES:
+        for arch in ARCHES:
+            log.info('Removing old pages from ' + suite + '...')
+            presents = sorted(os.listdir(RB_PKG_PATH + '/' + suite + '/' +
+                              arch))
+            for page in presents:
+                pkg = page.rsplit('.', 1)[0]
+                query = 'SELECT s.name ' + \
+                    'FROM results AS r ' + \
+                    'JOIN sources AS s ON r.package_id=s.id ' + \
+                    'WHERE s.name="{name}" AND r.status != "" ' + \
+                    'AND s.suite="{suite}" AND s.architecture="{arch}"'
+                query = query.format(name=pkg, suite=suite, arch=arch)
+                result = query_db(query)
+                if not result: # actually, the query produces no results
+                    log.info('There is no package named ' + pkg + ' from ' +
+                             suite + '/' + arch + ' in the database. ' +
+                             'Removing old page.')
+                    os.remove(RB_PKG_PATH + '/' + suite + '/' + arch + '/' +
+                              page)