g-i housekeeping: properly report disk usage for each job, exclude running jobs, so this can be run also when g-i-i jobs are running

2 files changed, 49 insertions, 23 deletions

diff --git a/bin/housekeeping.sh b/bin/housekeeping.sh
index d123ebcf..91430011 100755
--- a/bin/housekeeping.sh
+++ b/bin/housekeeping.sh
@@ -7,9 +7,6 @@ DEBUG=false
 . /srv/jenkins/bin/common-functions.sh
 common_init "$@"
 
-# let's have readable output here
-set +x
-
 check_for_mounted_chroots() {
 	CHROOT_PATTERN="/chroots/${1}-*"
 	OUTPUT=$(mktemp)
@@ -38,22 +35,45 @@ chroot_checks() {
 }
 
 report_disk_usage() {
+	if [ -z "$WATCHED_JOBS" ] ; then
+		echo "File system usage for all ${1} jobs:"
+	else
+		echo "File system usage for all ${1} jobs (including those currently running):"
+	fi
 	du -schx /var/lib/jenkins/jobs/${1}* |grep total |sed -s "s#total#${1} jobs#"
+	echo
+	if [ ! -z "$WATCHED_JOBS" ] ; then
+		TMPFILE=$(mktemp)
+		for JOB in $(cat $WATCHED_JOBS) ; do
+			du -shx --exclude='*/archive/*' $JOB | grep G >> $TMPFILE || true
+		done
+		if [ -s $TMPFILE ] ; then
+			echo
+			echo "${1} jobs with filesystem usage over 1G, excluding their archives and those currently running:"
+			cat $TMPFILE
+			echo
+		fi
+		rm $TMPFILE
+	fi
 }
 
 report_filetype_usage() {
 	OUTPUT=$(mktemp)
-	echo "File system use in $1 for $2 files:"
-	echo
-	find /var/lib/jenkins/jobs/${1}* -type f -name "*.${2}" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$1 .$2 files#" > $OUTPUT
-	if [ "$3" = "warn" ] && [ -s $OUTPUT ] ; then
-		echo "Warning: there are $2 files and there should not be any."
+	for JOB in $(cat $WATCHED_JOBS) ; do
+		if [ "$2" != "bak" ] && [ "$2" != "ppm" ] ; then
+			find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" ! -path "*/archive/*" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT
+		else
+			# find archived .bak + .ppm files too
+			find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT
+		fi
+	done
+	if [ -s $OUTPUT ] ; then
+		echo "File system use in $1 for $2 files:"
 		cat $OUTPUT
+		if [ "$3" = "warn" ] ; then
+			echo "Warning: there are $2 files and there should not be any."
+		fi
 		echo
-		echo "Checking for running QEMU processes: (might be causing these files to be there right now)"
-		ps fax | grep [q]emu-system | grep -v grep || true
-	else
-		cat $OUTPUT
 	fi
 	rm $OUTPUT
 }
@@ -76,7 +96,6 @@ wait4idle() {
 }
 
 general_housekeeping() {
-	echo
 	uptime
 
 	echo
@@ -110,14 +129,28 @@ else
 						report_disk_usage $1
 						chroot_checks $1
 						;;
-		g-i-installation)		wait4idle $1
+		g-i-installation)		ACTIVE_JOBS=$(mktemp)
+						WATCHED_JOBS=$(mktemp)
+						RUNNING=$(mktemp)
+						ps fax > $RUNNING
+						cd /var/lib/jenkins/jobs
+						for GIJ in g-i-installation_* ; do
+							if grep -q "$GIJ/workspace" $RUNNING ; then
+								echo "$GIJ" >> $ACTIVE_JOBS
+								echo "Ignoring $GIJ job as it's currently running."
+							else
+								echo "$GIJ" >> $WATCHED_JOBS
+							fi
+						done
+						echo
 						report_disk_usage $1
-						report_filetype_usage $1 raw warn
-						report_filetype_usage $1 iso
 						report_filetype_usage $1 png
 						report_filetype_usage $1 ppm warn
 						report_filetype_usage $1 bak warn
+						report_filetype_usage $1 raw warn
+						report_filetype_usage $1 iso warn
 						echo "WARNING: there is no check / handling on stale lvm volumes"
+						rm $ACTIVE_JOBS $WATCHED_JOBS $RUNNING
 						;;
 		squid)				report_squid_usage
 						;;
diff --git a/job-cfg/g-i-installation.yaml b/job-cfg/g-i-installation.yaml
index da4a811c..40f5de9a 100644
--- a/job-cfg/g-i-installation.yaml
+++ b/job-cfg/g-i-installation.yaml
@@ -86,13 +86,6 @@
           icon: /userContent/images/profitbricks-24x24.png
       - priority:
           job-prio: '175'
-      - throttle:
-          max-total: 4
-          max-per-node: 4
-          enabled: True
-          option: category
-          categories:
-            - g-i-installation
 
 - job-template:
     defaults: g-i-installation-housekeeping