From bcc697c4ebb0f7ae0098607bd0b38773ee7bdf59 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Tue, 31 Mar 2015 13:18:29 +0200 Subject: fix language: s#maintainance#maintenance#g --- bin/maintenance.sh | 208 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100755 bin/maintenance.sh (limited to 'bin/maintenance.sh') diff --git a/bin/maintenance.sh b/bin/maintenance.sh new file mode 100755 index 00000000..c123a9bd --- /dev/null +++ b/bin/maintenance.sh @@ -0,0 +1,208 @@ +#!/bin/bash + +# Copyright 2012-2015 Holger Levsen +# released under the GPLv=2 + +DEBUG=false +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +check_for_mounted_chroots() { + CHROOT_PATTERN="/chroots/${1}-*" + OUTPUT=$(mktemp) + ls $CHROOT_PATTERN 2>/dev/null > $OUTPUT || true + if [ -s $OUTPUT ] ; then + figlet "Warning:" + echo + echo "Probably manual cleanup needed:" + echo + echo "$ ls -la $CHROOT_PATTERN" + # List the processes using the partition + echo + fuser -mv $CHROOT_PATTERN + cat $OUTPUT + rm $OUTPUT + exit 1 + fi + rm $OUTPUT +} + +chroot_checks() { + check_for_mounted_chroots $1 + report_disk_usage /chroots + report_disk_usage /schroots + echo "WARNING: should remove directories in /(s)chroots which are older than a month." +} + +compress_old_jenkins_logs() { + local COMPRESSED + # compress logs to save space + COMPRESSED=$(find /var/lib/jenkins/jobs/*/builds/ -maxdepth 2 -mindepth 2 -mtime +1 -name log -exec gzip -9 -v {} \;) + if [ ! -z "$COMPRESSED" ] ; then + echo "Logs have been compressed:" + echo + echo "$COMPRESSED" + echo + fi +} + +remove_old_rebootstrap_logs() { + local OLDSTUFF + # delete old html logs to save space + OLDSTUFF=$(find /var/lib/jenkins/jobs/rebootstrap_* -maxdepth 3 -mtime +7 -name log_content.html -exec rm -v {} \;) + if [ ! -z "$OLDSTUFF" ] ; then + echo "Old html logs have been deleted:" + echo + echo "$OLDSTUFF" + echo + fi +} + +report_old_directories() { + # find and warn about old temp directories + if [ -z "$3" ] ; then + OLDSTUFF=$(find $1/* -maxdepth 0 -type d -mtime +$2 -exec ls -lad {} \;) + else + # if $3 is given, ignore it + OLDSTUFF=$(find $1/* -maxdepth 0 -type d -mtime +$2 ! -path "$3*" -exec ls -lad {} \;) + fi + if [ ! -z "$OLDSTUFF" ] ; then + echo "Warning: old temp directories found in $REP_RESULTS" + echo + echo "$OLDSTUFF" + echo "Please cleanup manually." + echo + fi +} + +report_disk_usage() { + if [ -z "$WATCHED_JOBS" ] ; then + echo "File system usage for all ${1} jobs:" + else + echo "File system usage for all ${1} jobs (including those currently running):" + fi + du -schx /var/lib/jenkins/jobs/${1}* |grep total |sed -s "s#total#${1} jobs#" + echo + if [ ! -z "$WATCHED_JOBS" ] ; then + TMPFILE=$(mktemp) + for JOB in $(cat $WATCHED_JOBS) ; do + du -shx --exclude='*/archive/*' $JOB | grep G >> $TMPFILE || true + done + if [ -s $TMPFILE ] ; then + echo + echo "${1} jobs with filesystem usage over 1G, excluding their archives and those currently running:" + cat $TMPFILE + echo + fi + rm $TMPFILE + fi +} + +report_filetype_usage() { + OUTPUT=$(mktemp) + for JOB in $(cat $WATCHED_JOBS) ; do + if [ "$2" != "bak" ] && [ "$2" != "png" ] ; then + find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" ! -path "*/archive/*" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT + else + # find archived .bak + .png files too + find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT + fi + done + if [ -s $OUTPUT ] ; then + echo "File system use in $1 for $2 files:" + cat $OUTPUT + if [ "$3" = "warn" ] ; then + echo "Warning: there are $2 files and there should not be any." + fi + echo + fi + rm $OUTPUT +} + +report_squid_usage() { + REPORT=/var/www/calamaris/calamaris.txt + if [ -z $1 ] ; then + cat $REPORT + else + head -31 $REPORT + fi +} + +wait4idle() { + echo "Waiting until no $1.sh process runs.... $(date)" + while [ $(ps fax | grep -c $1.sh) -gt 1 ] ; do + sleep 30 + done + echo "Done waiting: $(date)" +} + +general_maintenance() { + uptime + + echo + # ignore unreadable /media fuse mountpoints from guestmount + df -h 2>/dev/null || true + + echo + for DIR in /var/cache/apt/archives/ /var/spool/squid/ /var/cache/pbuilder/build/ /var/lib/jenkins/jobs/ /chroots /schroots ; do + sudo du -shx $DIR 2>/dev/null + done + JOB_PREFIXES=$(ls -1 /var/lib/jenkins/jobs/|cut -d "_" -f1|sort -f -u) + for PREFIX in $JOB_PREFIXES ; do + report_disk_usage $PREFIX + done + + echo + vnstat + + (df 2>/dev/null || true ) | grep tmpfs > /dev/null || ( echo ; echo "Warning: no tmpfs mounts in use. Please investigate the host system." ; exit 1 ) +} + +# +# if $1 is empty, we do general maintenance, else for some subgroup of all jobs +# +if [ -z $1 ] ; then + general_maintenance + compress_old_jenkins_logs + report_squid_usage brief +else + case $1 in + chroot-installation*) wait4idle $1 + report_disk_usage $1 + chroot_checks $1 + ;; + g-i-installation) ACTIVE_JOBS=$(mktemp) + WATCHED_JOBS=$(mktemp) + RUNNING=$(mktemp) + ps fax > $RUNNING + cd /var/lib/jenkins/jobs + for GIJ in g-i-installation_* ; do + if grep -q "$GIJ/workspace" $RUNNING ; then + echo "$GIJ" >> $ACTIVE_JOBS + echo "Ignoring $GIJ job as it's currently running." + else + echo "$GIJ" >> $WATCHED_JOBS + fi + done + echo + report_disk_usage $1 + report_filetype_usage $1 png warn + report_filetype_usage $1 bak warn + report_filetype_usage $1 raw warn + report_filetype_usage $1 iso warn + echo "WARNING: there is no check / handling on stale lvm volumes" + rm $ACTIVE_JOBS $WATCHED_JOBS $RUNNING + ;; + d-i) report_old_directories /srv/d-i 7 /srv/d-i/workspace + ;; + squid) report_squid_usage + ;; + rebootstrap) remove_old_rebootstrap_logs + ;; + *) ;; + esac +fi + +echo +echo "No (big) problems found, all seems good." +figlet "Ok." -- cgit v1.2.3-54-g00ecf