summaryrefslogtreecommitdiffstats
path: root/bin/maintainance.sh
blob: 89bf5fd11e0a164cb59c35a5e1f460dcc437ab81 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
#!/bin/bash

# Copyright 2012-2015 Holger Levsen <holger@layer-acht.org>
# released under the GPLv=2

DEBUG=false
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

check_for_mounted_chroots() {
	CHROOT_PATTERN="/chroots/${1}-*"
	OUTPUT=$(mktemp)
	ls $CHROOT_PATTERN 2>/dev/null > $OUTPUT || true
	if [ -s $OUTPUT ] ; then
		figlet "Warning:"
		echo
		echo "Probably manual cleanup needed:"
		echo
		echo "$ ls -la $CHROOT_PATTERN"
		# List the processes using the partition
		echo
		fuser -mv $CHROOT_PATTERN
		cat $OUTPUT
		rm $OUTPUT
		exit 1
	fi
	rm $OUTPUT
}

chroot_checks() {
	check_for_mounted_chroots $1
	report_disk_usage /chroots
	report_disk_usage /schroots
	echo "WARNING: should remove directories in /(s)chroots which are older than a month."
}

compress_old_jenkins_logs() {
	local COMPRESSED
	# compress logs to save space
	COMPRESSED=$(find /var/lib/jenkins/jobs/*/builds/ -maxdepth 2 -mindepth 2 -mtime +1 -name log -exec gzip -9 -v {} \;)
	if [ ! -z "$COMPRESSED" ] ; then
		echo "Logs have been compressed:"
		echo
		echo "$COMPRESSED"
		echo
	fi
}

remove_old_rebootstrap_logs() {
	local OLDSTUFF
	# delete old html logs to save space
	OLDSTUFF=$(find /var/lib/jenkins/jobs/rebootstrap_* -maxdepth 3 -mtime +7 -name log_content.html  -exec rm -v {} \;)
	if [ ! -z "$OLDSTUFF" ] ; then
		echo "Old html logs have been deleted:"
		echo
		echo "$OLDSTUFF"
		echo
	fi
}

report_old_directories() {
	# find and warn about old temp directories
	if [ -z "$3" ] ; then
		OLDSTUFF=$(find $1/* -maxdepth 0 -type d -mtime +$2 -exec ls -lad {} \;)
	else
		# if $3 is given, ignore it
		OLDSTUFF=$(find $1/* -maxdepth 0 -type d -mtime +$2 ! -path "$3*" -exec ls -lad {} \;)
	fi
	if [ ! -z "$OLDSTUFF" ] ; then
		echo "Warning: old temp directories found in $REP_RESULTS"
		echo
		echo "$OLDSTUFF"
		echo "Please cleanup manually."
		echo
	fi
}

report_disk_usage() {
	if [ -z "$WATCHED_JOBS" ] ; then
		echo "File system usage for all ${1} jobs:"
	else
		echo "File system usage for all ${1} jobs (including those currently running):"
	fi
	du -schx /var/lib/jenkins/jobs/${1}* |grep total |sed -s "s#total#${1} jobs#"
	echo
	if [ ! -z "$WATCHED_JOBS" ] ; then
		TMPFILE=$(mktemp)
		for JOB in $(cat $WATCHED_JOBS) ; do
			du -shx --exclude='*/archive/*' $JOB | grep G >> $TMPFILE || true
		done
		if [ -s $TMPFILE ] ; then
			echo
			echo "${1} jobs with filesystem usage over 1G, excluding their archives and those currently running:"
			cat $TMPFILE
			echo
		fi
		rm $TMPFILE
	fi
}

report_filetype_usage() {
	OUTPUT=$(mktemp)
	for JOB in $(cat $WATCHED_JOBS) ; do
		if [ "$2" != "bak" ] && [ "$2" != "png" ] ; then
			find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" ! -path "*/archive/*" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT
		else
			# find archived .bak + .png files too
			find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT
		fi
	done
	if [ -s $OUTPUT ] ; then
		echo "File system use in $1 for $2 files:"
		cat $OUTPUT
		if [ "$3" = "warn" ] ; then
			echo "Warning: there are $2 files and there should not be any."
		fi
		echo
	fi
	rm $OUTPUT
}

report_squid_usage() {
	REPORT=/var/www/calamaris/calamaris.txt
	if [ -z $1 ] ; then
		cat $REPORT
	else
		head -31 $REPORT
	fi
}

wait4idle() {
	echo "Waiting until no $1.sh process runs.... $(date)"
	while [ $(ps fax | grep -c $1.sh) -gt 1 ] ; do
		sleep 30
	done
	echo "Done waiting: $(date)"
}

general_maintainance() {
	uptime

	echo
	# ignore unreadable /media fuse mountpoints from guestmount
	df -h 2>/dev/null || true

	echo
	for DIR in /var/cache/apt/archives/ /var/spool/squid/ /var/cache/pbuilder/build/ /var/lib/jenkins/jobs/ /chroots /schroots ; do
		sudo du -shx $DIR 2>/dev/null
	done
	JOB_PREFIXES=$(ls -1 /var/lib/jenkins/jobs/|cut -d "_" -f1|sort -f -u)
	for PREFIX in $JOB_PREFIXES ; do
		report_disk_usage $PREFIX
	done

	echo
	vnstat

	(df 2>/dev/null || true ) | grep tmpfs > /dev/null || ( echo ; echo "Warning: no tmpfs mounts in use. Please investigate the host system." ; exit 1 )
}

#
# if $1 is empty, we do general maintainance, else for some subgroup of all jobs
#
if [ -z $1 ] ; then
	general_maintainance
	compress_old_jenkins_logs
	report_squid_usage brief
else
	case $1 in
		chroot-installation*)		wait4idle $1
						report_disk_usage $1
						chroot_checks $1
						;;
		g-i-installation)		ACTIVE_JOBS=$(mktemp)
						WATCHED_JOBS=$(mktemp)
						RUNNING=$(mktemp)
						ps fax > $RUNNING
						cd /var/lib/jenkins/jobs
						for GIJ in g-i-installation_* ; do
							if grep -q "$GIJ/workspace" $RUNNING ; then
								echo "$GIJ" >> $ACTIVE_JOBS
								echo "Ignoring $GIJ job as it's currently running."
							else
								echo "$GIJ" >> $WATCHED_JOBS
							fi
						done
						echo
						report_disk_usage $1
						report_filetype_usage $1 png warn
						report_filetype_usage $1 bak warn
						report_filetype_usage $1 raw warn
						report_filetype_usage $1 iso warn
						echo "WARNING: there is no check / handling on stale lvm volumes"
						rm $ACTIVE_JOBS $WATCHED_JOBS $RUNNING
						;;
		d-i)				report_old_directories /srv/d-i 7 /srv/d-i/workspace
						;;
		squid)				report_squid_usage
						;;
		rebootstrap)			remove_old_rebootstrap_logs
						;;
		*)				;;
	esac
fi

echo
echo "No (big) problems found, all seems good."
figlet "Ok."