From 9b381aff743a88c9ebc2e400a345ee836a25bd7e Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Sat, 15 Jul 2017 13:58:34 +0200 Subject: reproducible Debian: add node health check for too big jenkins.logs Signed-off-by: Holger Levsen --- bin/reproducible_node_health_check.sh | 14 ++++++++++++++ logparse/reproducible.rules | 2 ++ 2 files changed, 16 insertions(+) diff --git a/bin/reproducible_node_health_check.sh b/bin/reproducible_node_health_check.sh index 1fc61fb8..cc87584f 100755 --- a/bin/reproducible_node_health_check.sh +++ b/bin/reproducible_node_health_check.sh @@ -152,6 +152,20 @@ if [ "$HOSTNAME" = "$MAINNODE" ] ; then DIRTY=true echo fi + # + # /var/log/jenkins/jenkins.log sometimes grows very fast + # and we don't yet know why, so let's monitor this for now. + JENKINSLOG="find /var/log/jenkins -name jenkins.log -size +42G" + if [ -z "JENKINSLOG" ] ; then + echo "Warning, jenkins.log is larger than 42G, please fix, erroring out now." + exit 1 + else + JENKINSLOG="find /var/log/jenkins -name jenkins.log -size +23G" + if [ -z "JENKINSLOG" ] ; then + echo "Warning, jenkins.log is larger than 23G, please do something…" + DIRTY=true + fi + fi fi diff --git a/logparse/reproducible.rules b/logparse/reproducible.rules index ef19fef9..eef8d5c0 100644 --- a/logparse/reproducible.rules +++ b/logparse/reproducible.rules @@ -25,4 +25,6 @@ warning /Warning, today is the wrong future:.+/ warning /Warning: curl http:..www.debian.org failed, probably the proxy is down for.+/ warning /Warning, more than one kernel in.+/ warning /Warning, rise of the jenkins job zombies.+/ +warning /Warning, jenkins.log is larger than.+/ + -- cgit v1.2.3-54-g00ecf