diff options
| -rwxr-xr-x | bin/housekeeping.sh | 65 | ||||
| -rw-r--r-- | job-cfg/g-i-installation.yaml | 7 |
2 files changed, 49 insertions, 23 deletions
diff --git a/bin/housekeeping.sh b/bin/housekeeping.sh index d123ebcf..91430011 100755 --- a/bin/housekeeping.sh +++ b/bin/housekeeping.sh @@ -7,9 +7,6 @@ DEBUG=false . /srv/jenkins/bin/common-functions.sh common_init "$@" -# let's have readable output here -set +x - check_for_mounted_chroots() { CHROOT_PATTERN="/chroots/${1}-*" OUTPUT=$(mktemp) @@ -38,22 +35,45 @@ chroot_checks() { } report_disk_usage() { + if [ -z "$WATCHED_JOBS" ] ; then + echo "File system usage for all ${1} jobs:" + else + echo "File system usage for all ${1} jobs (including those currently running):" + fi du -schx /var/lib/jenkins/jobs/${1}* |grep total |sed -s "s#total#${1} jobs#" + echo + if [ ! -z "$WATCHED_JOBS" ] ; then + TMPFILE=$(mktemp) + for JOB in $(cat $WATCHED_JOBS) ; do + du -shx --exclude='*/archive/*' $JOB | grep G >> $TMPFILE || true + done + if [ -s $TMPFILE ] ; then + echo + echo "${1} jobs with filesystem usage over 1G, excluding their archives and those currently running:" + cat $TMPFILE + echo + fi + rm $TMPFILE + fi } report_filetype_usage() { OUTPUT=$(mktemp) - echo "File system use in $1 for $2 files:" - echo - find /var/lib/jenkins/jobs/${1}* -type f -name "*.${2}" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$1 .$2 files#" > $OUTPUT - if [ "$3" = "warn" ] && [ -s $OUTPUT ] ; then - echo "Warning: there are $2 files and there should not be any." + for JOB in $(cat $WATCHED_JOBS) ; do + if [ "$2" != "bak" ] && [ "$2" != "ppm" ] ; then + find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" ! -path "*/archive/*" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT + else + # find archived .bak + .ppm files too + find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT + fi + done + if [ -s $OUTPUT ] ; then + echo "File system use in $1 for $2 files:" cat $OUTPUT + if [ "$3" = "warn" ] ; then + echo "Warning: there are $2 files and there should not be any." + fi echo - echo "Checking for running QEMU processes: (might be causing these files to be there right now)" - ps fax | grep [q]emu-system | grep -v grep || true - else - cat $OUTPUT fi rm $OUTPUT } @@ -76,7 +96,6 @@ wait4idle() { } general_housekeeping() { - echo uptime echo @@ -110,14 +129,28 @@ else report_disk_usage $1 chroot_checks $1 ;; - g-i-installation) wait4idle $1 + g-i-installation) ACTIVE_JOBS=$(mktemp) + WATCHED_JOBS=$(mktemp) + RUNNING=$(mktemp) + ps fax > $RUNNING + cd /var/lib/jenkins/jobs + for GIJ in g-i-installation_* ; do + if grep -q "$GIJ/workspace" $RUNNING ; then + echo "$GIJ" >> $ACTIVE_JOBS + echo "Ignoring $GIJ job as it's currently running." + else + echo "$GIJ" >> $WATCHED_JOBS + fi + done + echo report_disk_usage $1 - report_filetype_usage $1 raw warn - report_filetype_usage $1 iso report_filetype_usage $1 png report_filetype_usage $1 ppm warn report_filetype_usage $1 bak warn + report_filetype_usage $1 raw warn + report_filetype_usage $1 iso warn echo "WARNING: there is no check / handling on stale lvm volumes" + rm $ACTIVE_JOBS $WATCHED_JOBS $RUNNING ;; squid) report_squid_usage ;; diff --git a/job-cfg/g-i-installation.yaml b/job-cfg/g-i-installation.yaml index da4a811c..40f5de9a 100644 --- a/job-cfg/g-i-installation.yaml +++ b/job-cfg/g-i-installation.yaml @@ -86,13 +86,6 @@ icon: /userContent/images/profitbricks-24x24.png - priority: job-prio: '175' - - throttle: - max-total: 4 - max-per-node: 4 - enabled: True - option: category - categories: - - g-i-installation - job-template: defaults: g-i-installation-housekeeping |