summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHolger Levsen <holger@layer-acht.org>2014-11-01 14:40:43 +0100
committerHolger Levsen <holger@layer-acht.org>2014-11-01 14:40:43 +0100
commit7420684ef0f4a7ed0aec5c6341f3a6f15e9ebcda (patch)
tree25d75b7523df0363841697a3a63c0a96a5ad14bd
parentdf6562b006cfe074017a247e66335e2b816b1cb6 (diff)
downloadjenkins.debian.net-7420684ef0f4a7ed0aec5c6341f3a6f15e9ebcda.tar.xz
g-i housekeeping: properly report disk usage for each job, exclude running jobs, so this can be run also when g-i-i jobs are running
-rwxr-xr-xbin/housekeeping.sh65
-rw-r--r--job-cfg/g-i-installation.yaml7
2 files changed, 49 insertions, 23 deletions
diff --git a/bin/housekeeping.sh b/bin/housekeeping.sh
index d123ebcf..91430011 100755
--- a/bin/housekeeping.sh
+++ b/bin/housekeeping.sh
@@ -7,9 +7,6 @@ DEBUG=false
. /srv/jenkins/bin/common-functions.sh
common_init "$@"
-# let's have readable output here
-set +x
-
check_for_mounted_chroots() {
CHROOT_PATTERN="/chroots/${1}-*"
OUTPUT=$(mktemp)
@@ -38,22 +35,45 @@ chroot_checks() {
}
report_disk_usage() {
+ if [ -z "$WATCHED_JOBS" ] ; then
+ echo "File system usage for all ${1} jobs:"
+ else
+ echo "File system usage for all ${1} jobs (including those currently running):"
+ fi
du -schx /var/lib/jenkins/jobs/${1}* |grep total |sed -s "s#total#${1} jobs#"
+ echo
+ if [ ! -z "$WATCHED_JOBS" ] ; then
+ TMPFILE=$(mktemp)
+ for JOB in $(cat $WATCHED_JOBS) ; do
+ du -shx --exclude='*/archive/*' $JOB | grep G >> $TMPFILE || true
+ done
+ if [ -s $TMPFILE ] ; then
+ echo
+ echo "${1} jobs with filesystem usage over 1G, excluding their archives and those currently running:"
+ cat $TMPFILE
+ echo
+ fi
+ rm $TMPFILE
+ fi
}
report_filetype_usage() {
OUTPUT=$(mktemp)
- echo "File system use in $1 for $2 files:"
- echo
- find /var/lib/jenkins/jobs/${1}* -type f -name "*.${2}" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$1 .$2 files#" > $OUTPUT
- if [ "$3" = "warn" ] && [ -s $OUTPUT ] ; then
- echo "Warning: there are $2 files and there should not be any."
+ for JOB in $(cat $WATCHED_JOBS) ; do
+ if [ "$2" != "bak" ] && [ "$2" != "ppm" ] ; then
+ find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" ! -path "*/archive/*" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT
+ else
+ # find archived .bak + .ppm files too
+ find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT
+ fi
+ done
+ if [ -s $OUTPUT ] ; then
+ echo "File system use in $1 for $2 files:"
cat $OUTPUT
+ if [ "$3" = "warn" ] ; then
+ echo "Warning: there are $2 files and there should not be any."
+ fi
echo
- echo "Checking for running QEMU processes: (might be causing these files to be there right now)"
- ps fax | grep [q]emu-system | grep -v grep || true
- else
- cat $OUTPUT
fi
rm $OUTPUT
}
@@ -76,7 +96,6 @@ wait4idle() {
}
general_housekeeping() {
- echo
uptime
echo
@@ -110,14 +129,28 @@ else
report_disk_usage $1
chroot_checks $1
;;
- g-i-installation) wait4idle $1
+ g-i-installation) ACTIVE_JOBS=$(mktemp)
+ WATCHED_JOBS=$(mktemp)
+ RUNNING=$(mktemp)
+ ps fax > $RUNNING
+ cd /var/lib/jenkins/jobs
+ for GIJ in g-i-installation_* ; do
+ if grep -q "$GIJ/workspace" $RUNNING ; then
+ echo "$GIJ" >> $ACTIVE_JOBS
+ echo "Ignoring $GIJ job as it's currently running."
+ else
+ echo "$GIJ" >> $WATCHED_JOBS
+ fi
+ done
+ echo
report_disk_usage $1
- report_filetype_usage $1 raw warn
- report_filetype_usage $1 iso
report_filetype_usage $1 png
report_filetype_usage $1 ppm warn
report_filetype_usage $1 bak warn
+ report_filetype_usage $1 raw warn
+ report_filetype_usage $1 iso warn
echo "WARNING: there is no check / handling on stale lvm volumes"
+ rm $ACTIVE_JOBS $WATCHED_JOBS $RUNNING
;;
squid) report_squid_usage
;;
diff --git a/job-cfg/g-i-installation.yaml b/job-cfg/g-i-installation.yaml
index da4a811c..40f5de9a 100644
--- a/job-cfg/g-i-installation.yaml
+++ b/job-cfg/g-i-installation.yaml
@@ -86,13 +86,6 @@
icon: /userContent/images/profitbricks-24x24.png
- priority:
job-prio: '175'
- - throttle:
- max-total: 4
- max-per-node: 4
- enabled: True
- option: category
- categories:
- - g-i-installation
- job-template:
defaults: g-i-installation-housekeeping