From abc5d6ee19068fc4c4a19bb61c466ac3251e9f94 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Mon, 12 Jun 2017 23:52:52 +0200 Subject: reproducible debian: rename _nodes_info to _html_nodes_info Signed-off-by: Holger Levsen --- bin/reproducible_html_nodes_info.sh | 215 ++++++++++++++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100755 bin/reproducible_html_nodes_info.sh (limited to 'bin/reproducible_html_nodes_info.sh') diff --git a/bin/reproducible_html_nodes_info.sh b/bin/reproducible_html_nodes_info.sh new file mode 100755 index 00000000..6f30ed76 --- /dev/null +++ b/bin/reproducible_html_nodes_info.sh @@ -0,0 +1,215 @@ +#!/bin/bash + +# Copyright © 2015-2016 Holger Levsen +# released under the GPLv=2 + +DEBUG=false +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code defining db access +. /srv/jenkins/bin/reproducible_common.sh + +TARGET_DIR=/srv/reproducible-results/node-information/ +mkdir -p $TARGET_DIR +TMPFILE_SRC=$(mktemp) +TMPFILE_NODE=$(mktemp) + +# +# build static webpages +# +VIEW=nodes_health +PAGE=index_${VIEW}.html +ARCH=amd64 +SUITE=unstable +echo "$(date -u) - starting to write $PAGE page." +write_page_header $VIEW "Nodes health overview" +DISCLAIMER="

This page is still under development. Please provide feedback, which other information (be it from munin or elsewhere) should be displayed and how this page should be split further, eg, the graphs could all be on another page and/or we should split this page into four for the four architectures being tested…

" +write_page $DISCLAIMER +write_page "

" +for ARCH in ${ARCHS} ; do + write_page "

$ARCH nodes

" + write_page "" + write_page "" + for SUITE in ${SUITES} ; do + if [ "$SUITE" = "experimental" ] ; then + continue + fi + write_page "" + done + for SUITE in ${SUITES} ; do + write_page "" + done + write_page "" + for NODE in jenkins $BUILD_NODES ; do + if [ -z "$(echo $NODE | grep $ARCH || true)" ] && [ "$NODE" != "jenkins" ] ; then + continue + elif [ "$NODE" = "jenkins" ] && [ "$ARCH" != "amd64" ] ; then + continue + fi + if [ "$NODE" = "jenkins" ] ; then + JENKINS_NODENAME=jenkins + NODE="jenkins.debian.net" + else + case $ARCH in + amd64|i386) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1-2|sed 's#-build##' ) ;; + arm64) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1-2|sed 's#-sled##' ) ;; + armhf) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1) ;; + esac + fi + write_page "" + URL="https://jenkins.debian.net/view/reproducible/view/Node_maintenance/job/reproducible_maintenance_${ARCH}_${JENKINS_NODENAME}" + BADGE="$URL/badge/icon" + write_page "" + if [ "$JENKINS_NODENAME" = "jenkins" ] ; then + write_page "" + else + write_page "" + fi + for SUITE in ${SUITES} ; do + if [ "$SUITE" = "experimental" ] ; then + continue + fi + URL="https://jenkins.debian.net/view/reproducible/view/Debian_setup_${ARCH}/job/reproducible_setup_schroot_${SUITE}_${ARCH}_${JENKINS_NODENAME}" + BADGE="$URL/badge/icon" + write_page "" + done + for SUITE in ${SUITES} ; do + if [ "$JENKINS_NODENAME" = "jenkins" ] ; then + write_page "" + else + URL="https://jenkins.debian.net/view/reproducible/view/Debian_setup_${ARCH}/job/reproducible_setup_pbuilder_${SUITE}_${ARCH}_${JENKINS_NODENAME}" + BADGE="$URL/badge/icon" + write_page "" + fi + done + write_page "" + done + write_page "
Namemaintenanceworker.log linksschroot setup $SUITEpbuilder setup $SUITE
$JENKINS_NODENAME" + SHORTNAME=$(echo $NODE | cut -d '.' -f1) + for WORKER in $(grep "${ARCH}_" /srv/jenkins/bin/reproducible_build_service.sh | grep -v \# |grep $SHORTNAME | cut -d ')' -f1) ; do + write_page "" + write_page "$(echo $WORKER |cut -d '_' -f2) " + done + write_page "
" +done +write_page "

" +write_page_footer +publish_page debian + +for TYPE in daily weekly ; do + VIEW=nodes_${TYPE}_graphs + PAGE=index_${VIEW}.html + ARCH=amd64 + SUITE=unstable + echo "$(date -u) - starting to write $PAGE page." + write_page_header $VIEW "Nodes $TYPE graphs" + write_page $DISCLAIMER + write_page "

" + for ARCH in ${ARCHS} ; do + write_page "

$ARCH nodes

" + write_page "" + write_page "" + write_page "" + for NODE in jenkins $BUILD_NODES ; do + if [ -z "$(echo $NODE | grep $ARCH || true)" ] && [ "$NODE" != "jenkins" ] ; then + continue + elif [ "$NODE" = "jenkins" ] && [ "$ARCH" != "amd64" ] ; then + continue + fi + if [ "$NODE" = "jenkins" ] ; then + JENKINS_NODENAME=jenkins + NODE="jenkins.debian.net" + else + case $ARCH in + amd64|i386) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1-2|sed 's#-build##' ) ;; + arm64) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1-2|sed 's#-sled##' ) ;; + armhf) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1) ;; + esac + fi + write_page "" + for GRAPH in jenkins_reproducible_builds cpu memory df swap load ; do + if [ "$JENKINS_NODENAME" = "jenkins" ] && [ "$GRAPH" = "jenkins_reproducible_builds" ] ; then + write_page "" + else + write_page "" + fi + done + write_page "" + + done + write_page "
Name
$JENKINS_NODENAME" + if [ "$TYPE" = "daily" ] ; then + IMG=day.png + else + IMG=week.png + fi + write_page "
" + done + write_page "

" + write_page_footer + publish_page debian +done + +# +# collect node information +# +echo "$(date -u) - Collecting information from nodes" +for NODE in $BUILD_NODES jenkins.debian.net ; do + if [ "$NODE" = "jenkins.debian.net" ] ; then + echo "$(date -u) - Trying to update $TARGET_DIR/$NODE." + /srv/jenkins/bin/reproducible_info.sh > $TARGET_DIR/$NODE + echo "$(date -u) - $TARGET_DIR/$NODE updated:" + cat $TARGET_DIR/$NODE + continue + fi + # call jenkins_master_wrapper.sh so we only need to track different ssh ports in one place + # jenkins_master_wrapper.sh needs NODE_NAME and JOB_NAME + export NODE_NAME=$NODE + export JOB_NAME=$JOB_NAME + echo "$(date -u) - Trying to update $TARGET_DIR/$NODE." + set +e + /srv/jenkins/bin/jenkins_master_wrapper.sh /srv/jenkins/bin/reproducible_info.sh > $TMPFILE_SRC + if [ $? -eq 1 ] ; then + echo "$(date -u) - Warning: could not update $TARGET_DIR/$NODE." + continue + fi + set -e + for KEY in $BUILD_ENV_VARS ; do + VALUE=$(egrep "^$KEY=" $TMPFILE_SRC | cut -d "=" -f2-) + if [ ! -z "$VALUE" ] ; then + echo "$KEY=$VALUE" >> $TMPFILE_NODE + fi + done + if [ -s $TMPFILE_NODE ] ; then + mv $TMPFILE_NODE $TARGET_DIR/$NODE + echo "$(date -u) - $TARGET_DIR/$NODE updated:" + cat $TARGET_DIR/$NODE + fi + rm -f $TMPFILE_SRC $TMPFILE_NODE +done +echo + +echo "$(date -u) - Showing node performance:" +TMPFILE1=$(mktemp) +TMPFILE2=$(mktemp) +TMPFILE3=$(mktemp) +NOW=$(date -u '+%Y-%m-%d %H:%m') +for i in $BUILD_NODES ; do + query_db "SELECT build_date FROM stats_build AS r WHERE ( r.node1='$i' OR r.node2='$i' )" > $TMPFILE1 2>/dev/null + j=$(wc -l $TMPFILE1|cut -d " " -f1) + k=$(cat $TMPFILE1|cut -d " " -f1|sort -u|wc -l) + l=$(echo "scale=1 ; ($j/$k)" | bc) + echo "$l builds/day ($j/$k) on $i" >> $TMPFILE2 + DATE=$(date '+%Y-%m-%d %H:%M' -d "-1 days") + m=$(query_db "SELECT count(build_date) FROM stats_build AS r WHERE ( r.node1='$i' OR r.node2='$i' ) AND r.build_date > '$DATE' " 2>/dev/null) + if [ "$m" = "" ] ; then m=0 ; fi + echo "$m builds in the last 24h on $i" >> $TMPFILE3 +done +rm $TMPFILE1 >/dev/null +sort -g -r $TMPFILE2 +echo +sort -g -r $TMPFILE3 +rm $TMPFILE2 $TMPFILE3 >/dev/null +echo + -- cgit v1.2.3-54-g00ecf