summaryrefslogtreecommitdiffstats
path: root/bin
diff options
context:
space:
mode:
authorMattia Rizzolo <mattia@mapreri.org>2015-04-10 12:39:12 +0200
committerHolger Levsen <holger@layer-acht.org>2015-04-16 16:47:01 +0200
commit36798da6b02677f0d8d3808bb0541148d8e1c314 (patch)
tree1eae382af734b145cc29382b550fce25870e7314 /bin
parentefff11a45908152891feb1a1b071ad9871208190 (diff)
downloadjenkins.debian.net-36798da6b02677f0d8d3808bb0541148d8e1c314.tar.xz
reproducible: new db table: removed_packages, to track removed package and subsequently clean up files
the db now stores tuple of (pkgname, suite, architecture) of every removed packages, then reproducible_maintenance.sh pick up entries from it and find+remove files. For now the reproducible_maintenance only output what he would have remove.
Diffstat (limited to 'bin')
-rwxr-xr-xbin/reproducible_db_maintenance.py7
-rwxr-xr-xbin/reproducible_maintenance.sh29
-rwxr-xr-xbin/reproducible_scheduler.py5
3 files changed, 26 insertions, 15 deletions
diff --git a/bin/reproducible_db_maintenance.py b/bin/reproducible_db_maintenance.py
index 17740398..003ae32b 100755
--- a/bin/reproducible_db_maintenance.py
+++ b/bin/reproducible_db_maintenance.py
@@ -354,6 +354,13 @@ schema_updates = {
url TEXT,
PRIMARY KEY (name))''',
'INSERT INTO rb_schema VALUES ("10", "' + now + '")'],
+ 11: [ # table with removed packages, to enable the maintenance job to do clean up
+ '''CREATE TABLE removed_packages (
+ name TEXT NOT NULL,
+ suite TEXT NOT NULL,
+ architecture TEXT NOT NULL,
+ PRIMARY KEY (name, suite, architecture))''',
+ 'INSERT INTO rb_schema VALUES ("11", "' + now + '")'],
}
diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh
index 53799b20..a4362650 100755
--- a/bin/reproducible_maintenance.sh
+++ b/bin/reproducible_maintenance.sh
@@ -185,26 +185,25 @@ fi
rm $PACKAGES
# find packages which have been removed from unstable
-# commented out for now. This can't be done using the database anymore
-QUERY="SELECT source_packages.name FROM source_packages
- WHERE source_packages.name NOT IN
- (SELECT sources.name FROM sources)
- LIMIT 25"
-#PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY")
-PACKAGES=''
+QUERY="SELECT name, suite, architecture FROM removed_packages
+ LIMIT 25"
+PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY")
if [ ! -z "$PACKAGES" ] ; then
+ DIRTY=true
echo
+ echo "Warning: found files relative to old packages, no more in the archive:"
echo "Removing these removed packages from database:"
echo $PACKAGES
echo
- QUERY="DELETE FROM source_packages
- WHERE source_packages.name NOT IN
- (SELECT sources.name FROM sources)
- LIMIT 25"
- sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY"
- cd /var/lib/jenkins/userContent
- for i in PACKAGES ; do
- find rb-pkg/ rbuild/ notes/ dbd/ -name "${i}_*" -exec rm -v {} \;
+ for pkg in "$PACKAGES" ; do
+ PKGNAME=$(echo "$pkg" | cut -f '|' -d 1)
+ SUITE=$(echo "$pkg" | cut -f '|' -d 2)
+ ARCH=$(echo "$pkg" | cut -f '|' -d 3)
+ QUERY="DELETE FROM removed_packages
+ WHERE name='$PKGNAME' AND suite='$SUITE' AND architecture='$ARCH'"
+ sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY"
+ cd /var/lib/jenkins/userContent
+ find rb-pkg/$SUITE/$ARCH rbuild/$SUITE/$ARCH notes/ dbd/$SUITE/$ARCH buildinfo/$SUITE/$ARCH -name "${PKGNAME}_*" | xargs echo rm -v
done
cd -
fi
diff --git a/bin/reproducible_scheduler.py b/bin/reproducible_scheduler.py
index bb465ea6..78d6fcb6 100755
--- a/bin/reproducible_scheduler.py
+++ b/bin/reproducible_scheduler.py
@@ -102,11 +102,13 @@ def update_sources_tables(suite):
rmed_pkgs = [x for x in cur_pkgs_name if x not in new_pkgs_name]
log.info('Now deleting ' + str(len(rmed_pkgs)) + ' removed packages: ' + str(rmed_pkgs))
rmed_pkgs_id = []
+ pkgs_to_rm = []
for pkg in rmed_pkgs:
result = query_db(('SELECT id FROM sources ' +
'WHERE name="{name}" ' +
'AND suite="{suite}"').format(name=pkg, suite=suite))
rmed_pkgs_id.extend(result)
+ pkgs_to_rm.extend([(x[0], x[1], 'amd64') for x in result])
log.debug('removed packages ID: ' + str([str(x[0]) for x in rmed_pkgs_id]))
cursor.executemany('DELETE FROM sources ' +
'WHERE id=?', rmed_pkgs_id)
@@ -114,6 +116,9 @@ def update_sources_tables(suite):
'WHERE package_id=?', rmed_pkgs_id)
cursor.executemany('DELETE FROM schedule ' +
'WHERE package_id=?', rmed_pkgs_id)
+ cursor.executemany('INSERT INTO removed_packages ' +
+ '(name, suite, architecture) ' +
+ 'VALUES (?, ?, ?)', pkgs_to_rm)
conn_db.commit()
# finally check whether the db has the correct number of packages
pkgs_end = query_db('SELECT count(*) FROM sources WHERE suite="%s"' % suite)