reproducible: new db table: removed_packages, to track removed package and subsequently clean up files

the db now stores tuple of (pkgname, suite, architecture) of every removed
packages, then reproducible_maintenance.sh pick up entries from it and
find+remove files.

For now the reproducible_maintenance only output what he would have remove.

3 files changed, 26 insertions, 15 deletions

diff --git a/bin/reproducible_db_maintenance.py b/bin/reproducible_db_maintenance.py
index 17740398..003ae32b 100755
--- a/bin/reproducible_db_maintenance.py
+++ b/bin/reproducible_db_maintenance.py
@@ -354,6 +354,13 @@ schema_updates = {
             url TEXT,
             PRIMARY KEY (name))''',
         'INSERT INTO rb_schema VALUES ("10", "' + now + '")'],
+    11: [ # table with removed packages, to enable the maintenance job to do clean up
+        '''CREATE TABLE removed_packages (
+            name TEXT NOT NULL,
+            suite TEXT NOT NULL,
+            architecture TEXT NOT NULL,
+            PRIMARY KEY (name, suite, architecture))''',
+        'INSERT INTO rb_schema VALUES ("11", "' + now + '")'],
 }
 
 
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index 53799b20..a4362650 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -185,26 +185,25 @@ fi
 rm $PACKAGES
 
 # find packages which have been removed from unstable
-# commented out for now. This can't be done using the database anymore
-QUERY="SELECT source_packages.name FROM source_packages
-		WHERE source_packages.name NOT IN
-		(SELECT sources.name FROM sources)
-	LIMIT 25"
-#PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY")
-PACKAGES=''
+QUERY="SELECT name, suite, architecture FROM removed_packages
+		LIMIT 25"
+PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY")
 if [ ! -z "$PACKAGES" ] ; then
+	DIRTY=true
 	echo
+	echo "Warning: found files relative to old packages, no more in the archive:"
 	echo "Removing these removed packages from database:"
 	echo $PACKAGES
 	echo
-	QUERY="DELETE FROM source_packages
-			WHERE source_packages.name NOT IN
-			(SELECT sources.name FROM sources)
-		LIMIT 25"
-	sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY"
-	cd /var/lib/jenkins/userContent
-	for i in PACKAGES ; do
-		find rb-pkg/ rbuild/ notes/ dbd/ -name "${i}_*" -exec rm -v {} \;
+	for pkg in "$PACKAGES" ; do
+		PKGNAME=$(echo "$pkg" | cut -f '|' -d 1)
+		SUITE=$(echo "$pkg" | cut -f '|' -d 2)
+		ARCH=$(echo "$pkg" | cut -f '|' -d 3)
+		QUERY="DELETE FROM removed_packages
+			WHERE name='$PKGNAME' AND suite='$SUITE' AND architecture='$ARCH'"
+		sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY"
+		cd /var/lib/jenkins/userContent
+		find rb-pkg/$SUITE/$ARCH  rbuild/$SUITE/$ARCH notes/ dbd/$SUITE/$ARCH buildinfo/$SUITE/$ARCH -name "${PKGNAME}_*" | xargs echo rm -v
 	done
 	cd -
 fi
diff --git a/bin/reproducible_scheduler.py b/bin/reproducible_scheduler.py
index bb465ea6..78d6fcb6 100755
--- a/bin/reproducible_scheduler.py
+++ b/bin/reproducible_scheduler.py
@@ -102,11 +102,13 @@ def update_sources_tables(suite):
     rmed_pkgs = [x for x in cur_pkgs_name if x not in new_pkgs_name]
     log.info('Now deleting ' + str(len(rmed_pkgs)) + ' removed packages: ' + str(rmed_pkgs))
     rmed_pkgs_id = []
+    pkgs_to_rm = []
     for pkg in rmed_pkgs:
         result = query_db(('SELECT id FROM sources ' +
                           'WHERE name="{name}" ' +
                           'AND suite="{suite}"').format(name=pkg, suite=suite))
         rmed_pkgs_id.extend(result)
+        pkgs_to_rm.extend([(x[0], x[1], 'amd64') for x in result])
     log.debug('removed packages ID: ' + str([str(x[0]) for x in rmed_pkgs_id]))
     cursor.executemany('DELETE FROM sources ' +
                        'WHERE id=?', rmed_pkgs_id)
@@ -114,6 +116,9 @@ def update_sources_tables(suite):
                        'WHERE package_id=?', rmed_pkgs_id)
     cursor.executemany('DELETE FROM schedule ' +
                        'WHERE package_id=?', rmed_pkgs_id)
+    cursor.executemany('INSERT INTO removed_packages '  +
+                       '(name, suite, architecture) ' +
+                       'VALUES (?, ?, ?)', pkgs_to_rm)
     conn_db.commit()
     # finally check whether the db has the correct number of packages
     pkgs_end = query_db('SELECT count(*) FROM sources WHERE suite="%s"' % suite)