diff options
| author | Holger Levsen <holger@layer-acht.org> | 2017-07-15 13:58:34 +0200 |
|---|---|---|
| committer | Holger Levsen <holger@layer-acht.org> | 2017-07-15 13:58:34 +0200 |
| commit | 9b381aff743a88c9ebc2e400a345ee836a25bd7e (patch) | |
| tree | 5ad2ba92bd7c821d6c33c62ec0964cca820fa300 | |
| parent | 36d88b06612905aa394a7cfbbc07a1c335a25f07 (diff) | |
| download | jenkins.debian.net-9b381aff743a88c9ebc2e400a345ee836a25bd7e.tar.xz | |
reproducible Debian: add node health check for too big jenkins.logs
Signed-off-by: Holger Levsen <holger@layer-acht.org>
| -rwxr-xr-x | bin/reproducible_node_health_check.sh | 14 | ||||
| -rw-r--r-- | logparse/reproducible.rules | 2 |
2 files changed, 16 insertions, 0 deletions
diff --git a/bin/reproducible_node_health_check.sh b/bin/reproducible_node_health_check.sh index 1fc61fb8..cc87584f 100755 --- a/bin/reproducible_node_health_check.sh +++ b/bin/reproducible_node_health_check.sh @@ -152,6 +152,20 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then DIRTY=true echo fi + # + # /var/log/jenkins/jenkins.log sometimes grows very fast + # and we don't yet know why, so let's monitor this for now. + JENKINSLOG="find /var/log/jenkins -name jenkins.log -size +42G" + if [ -z "JENKINSLOG" ] ; then + echo "Warning, jenkins.log is larger than 42G, please fix, erroring out now." + exit 1 + else + JENKINSLOG="find /var/log/jenkins -name jenkins.log -size +23G" + if [ -z "JENKINSLOG" ] ; then + echo "Warning, jenkins.log is larger than 23G, please do something…" + DIRTY=true + fi + fi fi diff --git a/logparse/reproducible.rules b/logparse/reproducible.rules index ef19fef9..eef8d5c0 100644 --- a/logparse/reproducible.rules +++ b/logparse/reproducible.rules @@ -25,4 +25,6 @@ warning /Warning, today is the wrong future:.+/ warning /Warning: curl http:..www.debian.org failed, probably the proxy is down for.+/ warning /Warning, more than one kernel in.+/ warning /Warning, rise of the jenkins job zombies.+/ +warning /Warning, jenkins.log is larger than.+/ + |