diff options
| -rw-r--r-- | TODO | 1 | ||||
| -rwxr-xr-x | bin/reproducible_db_maintenance.py | 7 | ||||
| -rwxr-xr-x | bin/reproducible_maintenance.sh | 29 | ||||
| -rwxr-xr-x | bin/reproducible_scheduler.py | 5 | ||||
| -rw-r--r-- | logparse/reproducible.rules | 1 |
5 files changed, 27 insertions, 16 deletions
@@ -138,7 +138,6 @@ properties: ** meld bin/reproducible_setup_schroot.sh into bin/schroot-create.sh and alias the former to the latter ** "fork" etc/schroot/default into etc/schroot/reproducible ** use one css for all, not two minimally different ones -** restore the "find packages which have been removed from sid" part of reproducible_maintenance.sh ** repo-comparison: check for binaries without source ** turn job-cfg/g-i.yaml into .yaml.py diff --git a/bin/reproducible_db_maintenance.py b/bin/reproducible_db_maintenance.py index 17740398..003ae32b 100755 --- a/bin/reproducible_db_maintenance.py +++ b/bin/reproducible_db_maintenance.py @@ -354,6 +354,13 @@ schema_updates = { url TEXT, PRIMARY KEY (name))''', 'INSERT INTO rb_schema VALUES ("10", "' + now + '")'], + 11: [ # table with removed packages, to enable the maintenance job to do clean up + '''CREATE TABLE removed_packages ( + name TEXT NOT NULL, + suite TEXT NOT NULL, + architecture TEXT NOT NULL, + PRIMARY KEY (name, suite, architecture))''', + 'INSERT INTO rb_schema VALUES ("11", "' + now + '")'], } diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh index 53799b20..a4362650 100755 --- a/bin/reproducible_maintenance.sh +++ b/bin/reproducible_maintenance.sh @@ -185,26 +185,25 @@ fi rm $PACKAGES # find packages which have been removed from unstable -# commented out for now. This can't be done using the database anymore -QUERY="SELECT source_packages.name FROM source_packages - WHERE source_packages.name NOT IN - (SELECT sources.name FROM sources) - LIMIT 25" -#PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY") -PACKAGES='' +QUERY="SELECT name, suite, architecture FROM removed_packages + LIMIT 25" +PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY") if [ ! -z "$PACKAGES" ] ; then + DIRTY=true echo + echo "Warning: found files relative to old packages, no more in the archive:" echo "Removing these removed packages from database:" echo $PACKAGES echo - QUERY="DELETE FROM source_packages - WHERE source_packages.name NOT IN - (SELECT sources.name FROM sources) - LIMIT 25" - sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY" - cd /var/lib/jenkins/userContent - for i in PACKAGES ; do - find rb-pkg/ rbuild/ notes/ dbd/ -name "${i}_*" -exec rm -v {} \; + for pkg in "$PACKAGES" ; do + PKGNAME=$(echo "$pkg" | cut -f '|' -d 1) + SUITE=$(echo "$pkg" | cut -f '|' -d 2) + ARCH=$(echo "$pkg" | cut -f '|' -d 3) + QUERY="DELETE FROM removed_packages + WHERE name='$PKGNAME' AND suite='$SUITE' AND architecture='$ARCH'" + sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY" + cd /var/lib/jenkins/userContent + find rb-pkg/$SUITE/$ARCH rbuild/$SUITE/$ARCH notes/ dbd/$SUITE/$ARCH buildinfo/$SUITE/$ARCH -name "${PKGNAME}_*" | xargs echo rm -v done cd - fi diff --git a/bin/reproducible_scheduler.py b/bin/reproducible_scheduler.py index bb465ea6..78d6fcb6 100755 --- a/bin/reproducible_scheduler.py +++ b/bin/reproducible_scheduler.py @@ -102,11 +102,13 @@ def update_sources_tables(suite): rmed_pkgs = [x for x in cur_pkgs_name if x not in new_pkgs_name] log.info('Now deleting ' + str(len(rmed_pkgs)) + ' removed packages: ' + str(rmed_pkgs)) rmed_pkgs_id = [] + pkgs_to_rm = [] for pkg in rmed_pkgs: result = query_db(('SELECT id FROM sources ' + 'WHERE name="{name}" ' + 'AND suite="{suite}"').format(name=pkg, suite=suite)) rmed_pkgs_id.extend(result) + pkgs_to_rm.extend([(x[0], x[1], 'amd64') for x in result]) log.debug('removed packages ID: ' + str([str(x[0]) for x in rmed_pkgs_id])) cursor.executemany('DELETE FROM sources ' + 'WHERE id=?', rmed_pkgs_id) @@ -114,6 +116,9 @@ def update_sources_tables(suite): 'WHERE package_id=?', rmed_pkgs_id) cursor.executemany('DELETE FROM schedule ' + 'WHERE package_id=?', rmed_pkgs_id) + cursor.executemany('INSERT INTO removed_packages ' + + '(name, suite, architecture) ' + + 'VALUES (?, ?, ?)', pkgs_to_rm) conn_db.commit() # finally check whether the db has the correct number of packages pkgs_end = query_db('SELECT count(*) FROM sources WHERE suite="%s"' % suite) diff --git a/logparse/reproducible.rules b/logparse/reproducible.rules index 176836c5..b9a5c564 100644 --- a/logparse/reproducible.rules +++ b/logparse/reproducible.rules @@ -9,6 +9,7 @@ warning /Warning: lock .+ still exists, exiting./ warning /Warning: old schroots found in /schroots, which have been deleted:/ warning /Warning: processes found which should not be there/ warning /Warning: packages found where the build was started more than 24h ago:/ +warning /Warning: found files relative to old packages, no more in the archive:/ warning /Warning: Found files with bad permissions.+/ warning /Warning: .+ could not be fully removed./ warning /Warning: could not download.+/ |