blob: 41c166b5406186ccd9cd94bdc9386d91d6cd47b7 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
|
#!/bin/bash
# Copyright 2012-2015 Holger Levsen <holger@layer-acht.org>
# released under the GPLv=2
DEBUG=false
. /srv/jenkins/bin/common-functions.sh
common_init "$@"
check_for_mounted_chroots() {
CHROOT_PATTERN="/chroots/${1}-*"
OUTPUT=$(mktemp)
ls $CHROOT_PATTERN 2>/dev/null > $OUTPUT || true
if [ -s $OUTPUT ] ; then
figlet "Warning:"
echo
echo "Probably manual cleanup needed:"
echo
echo "$ ls -la $CHROOT_PATTERN"
# List the processes using the partition
echo
fuser -mv $CHROOT_PATTERN
cat $OUTPUT
rm $OUTPUT
exit 1
fi
rm $OUTPUT
}
chroot_checks() {
check_for_mounted_chroots $1
report_disk_usage /chroots
report_disk_usage /schroots
echo "WARNING: should remove directories in /(s)chroots which are older than a month."
}
compress_old_jenkins_logs() {
local COMPRESSED
# compress logs to save space
COMPRESSED=$(find /var/lib/jenkins/jobs/*/builds/ -maxdepth 2 -mindepth 2 -mtime +1 -name log -exec gzip -9 -v {} \;)
if [ ! -z "$COMPRESSED" ] ; then
echo "Logs have been compressed:"
echo
echo "$COMPRESSED"
echo
fi
}
remove_old_rebootstrap_logs() {
local OLDSTUFF
# delete old html logs to save space
OLDSTUFF=$(find /var/lib/jenkins/jobs/rebootstrap_* -maxdepth 3 -mtime +7 -name log_content.html -exec rm -v {} \;)
if [ ! -z "$OLDSTUFF" ] ; then
echo "Old html logs have been deleted:"
echo
echo "$OLDSTUFF"
echo
fi
}
report_old_directories() {
# find and warn about old temp directories
if [ -z "$3" ] ; then
OLDSTUFF=$(find $1/* -maxdepth 0 -type d -mtime +$2 -exec ls -lad {} \;)
elif [ -z "$4" ] ; then
# if $3 is given, ignore it
OLDSTUFF=$(find $1/* -maxdepth 0 -type d -mtime +$2 ! -path "$3*" -exec ls -lad {} \;)
else
# if $3 + $4 are given, ignore them
OLDSTUFF=$(find $1/* -maxdepth 0 -type d -mtime +$2 ! -path "$3*" ! -path "$4*" -exec ls -lad {} \;)
fi
if [ ! -z "$OLDSTUFF" ] ; then
echo "Warning: old temp directories found in $1"
echo
echo "$OLDSTUFF"
echo "Please cleanup manually."
echo
fi
}
report_disk_usage() {
if [ -z "$WATCHED_JOBS" ] ; then
echo "File system usage for all ${1} jobs:"
else
echo "File system usage for all ${1} jobs (including those currently running):"
fi
du -schx /var/lib/jenkins/jobs/${1}* |grep total |sed -s "s#total#${1} jobs#"
echo
if [ ! -z "$WATCHED_JOBS" ] ; then
TMPFILE=$(mktemp)
for JOB in $(cat $WATCHED_JOBS) ; do
du -shx --exclude='*/archive/*' $JOB | grep G >> $TMPFILE || true
done
if [ -s $TMPFILE ] ; then
echo
echo "${1} jobs with filesystem usage over 1G, excluding their archives and those currently running:"
cat $TMPFILE
echo
fi
rm $TMPFILE
fi
}
report_filetype_usage() {
OUTPUT=$(mktemp)
for JOB in $(cat $WATCHED_JOBS) ; do
if [ "$2" != "bak" ] && [ "$2" != "png" ] ; then
find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" ! -path "*/archive/*" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT
else
# find archived .bak + .png files too
find /var/lib/jenkins/jobs/$JOB -type f -name "*.${2}" 2>/dev/null|xargs -r du -sch |grep total |sed -s "s#total#$JOB .$2 files#" >> $OUTPUT
fi
done
if [ -s $OUTPUT ] ; then
echo "File system use in $1 for $2 files:"
cat $OUTPUT
if [ "$3" = "warn" ] ; then
echo "Warning: there are $2 files and there should not be any."
fi
echo
fi
rm $OUTPUT
}
report_squid_usage() {
REPORT=/var/www/calamaris/calamaris.txt
if [ -z $1 ] ; then
cat $REPORT
else
head -31 $REPORT
fi
}
wait4idle() {
echo "Waiting until no $1.sh process runs.... $(date)"
while [ $(ps fax | grep -c $1.sh) -gt 1 ] ; do
sleep 30
done
echo "Done waiting: $(date)"
}
general_maintenance() {
uptime
echo
# ignore unreadable /media fuse mountpoints from guestmount
df -h 2>/dev/null || true
echo
for DIR in /var/cache/apt/archives/ /var/spool/squid3/ /var/cache/pbuilder/build/ /var/lib/jenkins/jobs/ /chroots /schroots ; do
sudo du -shx $DIR 2>/dev/null
done
JOB_PREFIXES=$(ls -1 /var/lib/jenkins/jobs/|cut -d "_" -f1|sort -f -u)
for PREFIX in $JOB_PREFIXES ; do
report_disk_usage $PREFIX
done
echo
vnstat
(df 2>/dev/null || true ) | grep tmpfs > /dev/null || ( echo ; echo "Warning: no tmpfs mounts in use. Please investigate the host system." ; exit 1 )
}
#
# if $1 is empty, we do general maintenance, else for some subgroup of all jobs
#
if [ -z $1 ] ; then
general_maintenance
compress_old_jenkins_logs
report_squid_usage brief
else
case $1 in
chroot-installation*) wait4idle $1
report_disk_usage $1
chroot_checks $1
;;
g-i-installation) ACTIVE_JOBS=$(mktemp)
WATCHED_JOBS=$(mktemp)
RUNNING=$(mktemp)
ps fax > $RUNNING
cd /var/lib/jenkins/jobs
for GIJ in g-i-installation_* ; do
if grep -q "$GIJ/workspace" $RUNNING ; then
echo "$GIJ" >> $ACTIVE_JOBS
echo "Ignoring $GIJ job as it's currently running."
else
echo "$GIJ" >> $WATCHED_JOBS
fi
done
echo
report_disk_usage $1
report_filetype_usage $1 png warn
report_filetype_usage $1 bak warn
report_filetype_usage $1 raw warn
report_filetype_usage $1 iso warn
echo "WARNING: there is no check / handling on stale lvm volumes"
rm $ACTIVE_JOBS $WATCHED_JOBS $RUNNING
;;
d-i) report_old_directories /srv/d-i 7 /srv/d-i/workspace /srv/d-i/isos
;;
squid) report_squid_usage
;;
rebootstrap) remove_old_rebootstrap_logs
;;
*) ;;
esac
fi
echo
echo "No (big) problems found, all seems good."
figlet "Ok."
|