From abc5d6ee19068fc4c4a19bb61c466ac3251e9f94 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Mon, 12 Jun 2017 23:52:52 +0200 Subject: reproducible debian: rename _nodes_info to _html_nodes_info Signed-off-by: Holger Levsen --- bin/jenkins_node_definitions.sh | 2 +- bin/reproducible_html_dashboard.sh | 2 +- bin/reproducible_html_nodes_info.sh | 215 +++++++++++++++++++++++++++++++++++ bin/reproducible_nodes_info.sh | 216 ------------------------------------ job-cfg/reproducible.yaml | 6 +- 5 files changed, 220 insertions(+), 221 deletions(-) create mode 100755 bin/reproducible_html_nodes_info.sh delete mode 100755 bin/reproducible_nodes_info.sh diff --git a/bin/jenkins_node_definitions.sh b/bin/jenkins_node_definitions.sh index 9b4eaf0b..46848fbb 100755 --- a/bin/jenkins_node_definitions.sh +++ b/bin/jenkins_node_definitions.sh @@ -5,7 +5,7 @@ # define Debian build nodes in use for tests.reproducible-builds.org/debian/ # FIXME: this is used differently in two places, -# - bin/reproducible_nodes_info.sh +# - bin/reproducible_html_nodes_info.sh # where it *must* only contain the Debian nodes as it's used # to generate the variations… and # - bin/reproducible_cleanup_nodes.sh where it would be diff --git a/bin/reproducible_html_dashboard.sh b/bin/reproducible_html_dashboard.sh index 4dc7b5d5..3855a8c7 100755 --- a/bin/reproducible_html_dashboard.sh +++ b/bin/reproducible_html_dashboard.sh @@ -676,7 +676,7 @@ create_performance_page() { write_page "

" done # the end - write_page "Daily individual build node performance stats are available as well as oldest results for" + write_page "Daily individual build node performance stats are available as well as oldest results for" for ARCH in ${ARCHS} ; do write_page " $ARCH" done diff --git a/bin/reproducible_html_nodes_info.sh b/bin/reproducible_html_nodes_info.sh new file mode 100755 index 00000000..6f30ed76 --- /dev/null +++ b/bin/reproducible_html_nodes_info.sh @@ -0,0 +1,215 @@ +#!/bin/bash + +# Copyright © 2015-2016 Holger Levsen +# released under the GPLv=2 + +DEBUG=false +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code defining db access +. /srv/jenkins/bin/reproducible_common.sh + +TARGET_DIR=/srv/reproducible-results/node-information/ +mkdir -p $TARGET_DIR +TMPFILE_SRC=$(mktemp) +TMPFILE_NODE=$(mktemp) + +# +# build static webpages +# +VIEW=nodes_health +PAGE=index_${VIEW}.html +ARCH=amd64 +SUITE=unstable +echo "$(date -u) - starting to write $PAGE page." +write_page_header $VIEW "Nodes health overview" +DISCLAIMER="

This page is still under development. Please provide feedback, which other information (be it from munin or elsewhere) should be displayed and how this page should be split further, eg, the graphs could all be on another page and/or we should split this page into four for the four architectures being tested…

" +write_page $DISCLAIMER +write_page "

" +for ARCH in ${ARCHS} ; do + write_page "

$ARCH nodes

" + write_page "" + write_page "" + for SUITE in ${SUITES} ; do + if [ "$SUITE" = "experimental" ] ; then + continue + fi + write_page "" + done + for SUITE in ${SUITES} ; do + write_page "" + done + write_page "" + for NODE in jenkins $BUILD_NODES ; do + if [ -z "$(echo $NODE | grep $ARCH || true)" ] && [ "$NODE" != "jenkins" ] ; then + continue + elif [ "$NODE" = "jenkins" ] && [ "$ARCH" != "amd64" ] ; then + continue + fi + if [ "$NODE" = "jenkins" ] ; then + JENKINS_NODENAME=jenkins + NODE="jenkins.debian.net" + else + case $ARCH in + amd64|i386) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1-2|sed 's#-build##' ) ;; + arm64) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1-2|sed 's#-sled##' ) ;; + armhf) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1) ;; + esac + fi + write_page "" + URL="https://jenkins.debian.net/view/reproducible/view/Node_maintenance/job/reproducible_maintenance_${ARCH}_${JENKINS_NODENAME}" + BADGE="$URL/badge/icon" + write_page "" + if [ "$JENKINS_NODENAME" = "jenkins" ] ; then + write_page "" + else + write_page "" + fi + for SUITE in ${SUITES} ; do + if [ "$SUITE" = "experimental" ] ; then + continue + fi + URL="https://jenkins.debian.net/view/reproducible/view/Debian_setup_${ARCH}/job/reproducible_setup_schroot_${SUITE}_${ARCH}_${JENKINS_NODENAME}" + BADGE="$URL/badge/icon" + write_page "" + done + for SUITE in ${SUITES} ; do + if [ "$JENKINS_NODENAME" = "jenkins" ] ; then + write_page "" + else + URL="https://jenkins.debian.net/view/reproducible/view/Debian_setup_${ARCH}/job/reproducible_setup_pbuilder_${SUITE}_${ARCH}_${JENKINS_NODENAME}" + BADGE="$URL/badge/icon" + write_page "" + fi + done + write_page "" + done + write_page "
Namemaintenanceworker.log linksschroot setup $SUITEpbuilder setup $SUITE
$JENKINS_NODENAME" + SHORTNAME=$(echo $NODE | cut -d '.' -f1) + for WORKER in $(grep "${ARCH}_" /srv/jenkins/bin/reproducible_build_service.sh | grep -v \# |grep $SHORTNAME | cut -d ')' -f1) ; do + write_page "" + write_page "$(echo $WORKER |cut -d '_' -f2) " + done + write_page "
" +done +write_page "

" +write_page_footer +publish_page debian + +for TYPE in daily weekly ; do + VIEW=nodes_${TYPE}_graphs + PAGE=index_${VIEW}.html + ARCH=amd64 + SUITE=unstable + echo "$(date -u) - starting to write $PAGE page." + write_page_header $VIEW "Nodes $TYPE graphs" + write_page $DISCLAIMER + write_page "

" + for ARCH in ${ARCHS} ; do + write_page "

$ARCH nodes

" + write_page "" + write_page "" + write_page "" + for NODE in jenkins $BUILD_NODES ; do + if [ -z "$(echo $NODE | grep $ARCH || true)" ] && [ "$NODE" != "jenkins" ] ; then + continue + elif [ "$NODE" = "jenkins" ] && [ "$ARCH" != "amd64" ] ; then + continue + fi + if [ "$NODE" = "jenkins" ] ; then + JENKINS_NODENAME=jenkins + NODE="jenkins.debian.net" + else + case $ARCH in + amd64|i386) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1-2|sed 's#-build##' ) ;; + arm64) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1-2|sed 's#-sled##' ) ;; + armhf) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1) ;; + esac + fi + write_page "" + for GRAPH in jenkins_reproducible_builds cpu memory df swap load ; do + if [ "$JENKINS_NODENAME" = "jenkins" ] && [ "$GRAPH" = "jenkins_reproducible_builds" ] ; then + write_page "" + else + write_page "" + fi + done + write_page "" + + done + write_page "
Name
$JENKINS_NODENAME" + if [ "$TYPE" = "daily" ] ; then + IMG=day.png + else + IMG=week.png + fi + write_page "
" + done + write_page "

" + write_page_footer + publish_page debian +done + +# +# collect node information +# +echo "$(date -u) - Collecting information from nodes" +for NODE in $BUILD_NODES jenkins.debian.net ; do + if [ "$NODE" = "jenkins.debian.net" ] ; then + echo "$(date -u) - Trying to update $TARGET_DIR/$NODE." + /srv/jenkins/bin/reproducible_info.sh > $TARGET_DIR/$NODE + echo "$(date -u) - $TARGET_DIR/$NODE updated:" + cat $TARGET_DIR/$NODE + continue + fi + # call jenkins_master_wrapper.sh so we only need to track different ssh ports in one place + # jenkins_master_wrapper.sh needs NODE_NAME and JOB_NAME + export NODE_NAME=$NODE + export JOB_NAME=$JOB_NAME + echo "$(date -u) - Trying to update $TARGET_DIR/$NODE." + set +e + /srv/jenkins/bin/jenkins_master_wrapper.sh /srv/jenkins/bin/reproducible_info.sh > $TMPFILE_SRC + if [ $? -eq 1 ] ; then + echo "$(date -u) - Warning: could not update $TARGET_DIR/$NODE." + continue + fi + set -e + for KEY in $BUILD_ENV_VARS ; do + VALUE=$(egrep "^$KEY=" $TMPFILE_SRC | cut -d "=" -f2-) + if [ ! -z "$VALUE" ] ; then + echo "$KEY=$VALUE" >> $TMPFILE_NODE + fi + done + if [ -s $TMPFILE_NODE ] ; then + mv $TMPFILE_NODE $TARGET_DIR/$NODE + echo "$(date -u) - $TARGET_DIR/$NODE updated:" + cat $TARGET_DIR/$NODE + fi + rm -f $TMPFILE_SRC $TMPFILE_NODE +done +echo + +echo "$(date -u) - Showing node performance:" +TMPFILE1=$(mktemp) +TMPFILE2=$(mktemp) +TMPFILE3=$(mktemp) +NOW=$(date -u '+%Y-%m-%d %H:%m') +for i in $BUILD_NODES ; do + query_db "SELECT build_date FROM stats_build AS r WHERE ( r.node1='$i' OR r.node2='$i' )" > $TMPFILE1 2>/dev/null + j=$(wc -l $TMPFILE1|cut -d " " -f1) + k=$(cat $TMPFILE1|cut -d " " -f1|sort -u|wc -l) + l=$(echo "scale=1 ; ($j/$k)" | bc) + echo "$l builds/day ($j/$k) on $i" >> $TMPFILE2 + DATE=$(date '+%Y-%m-%d %H:%M' -d "-1 days") + m=$(query_db "SELECT count(build_date) FROM stats_build AS r WHERE ( r.node1='$i' OR r.node2='$i' ) AND r.build_date > '$DATE' " 2>/dev/null) + if [ "$m" = "" ] ; then m=0 ; fi + echo "$m builds in the last 24h on $i" >> $TMPFILE3 +done +rm $TMPFILE1 >/dev/null +sort -g -r $TMPFILE2 +echo +sort -g -r $TMPFILE3 +rm $TMPFILE2 $TMPFILE3 >/dev/null +echo + diff --git a/bin/reproducible_nodes_info.sh b/bin/reproducible_nodes_info.sh deleted file mode 100755 index 8a9d5ec2..00000000 --- a/bin/reproducible_nodes_info.sh +++ /dev/null @@ -1,216 +0,0 @@ -#!/bin/bash - -# Copyright © 2015-2016 Holger Levsen -# released under the GPLv=2 - -DEBUG=false -. /srv/jenkins/bin/common-functions.sh -common_init "$@" - -# common code defining db access -. /srv/jenkins/bin/reproducible_common.sh - -TARGET_DIR=/srv/reproducible-results/node-information/ -mkdir -p $TARGET_DIR -TMPFILE_SRC=$(mktemp) -TMPFILE_NODE=$(mktemp) - -# -# build static webpages -# -VIEW=nodes_health -PAGE=index_${VIEW}.html -ARCH=amd64 -SUITE=unstable -echo "$(date -u) - starting to write $PAGE page." -write_page_header $VIEW "Nodes health overview" -DISCLAIMER="

This page is still under development. Please provide feedback, which other information (be it from munin or elsewhere) should be displayed and how this page should be split further, eg, the graphs could all be on another page and/or we should split this page into four for the four architectures being tested…

" -write_page $DISCLAIMER -# FIXME: Also either $0 and its job needs to be renamed to include 'html' or the code needs to be moved elsewhere -write_page "

" -for ARCH in ${ARCHS} ; do - write_page "

$ARCH nodes

" - write_page "" - write_page "" - for SUITE in ${SUITES} ; do - if [ "$SUITE" = "experimental" ] ; then - continue - fi - write_page "" - done - for SUITE in ${SUITES} ; do - write_page "" - done - write_page "" - for NODE in jenkins $BUILD_NODES ; do - if [ -z "$(echo $NODE | grep $ARCH || true)" ] && [ "$NODE" != "jenkins" ] ; then - continue - elif [ "$NODE" = "jenkins" ] && [ "$ARCH" != "amd64" ] ; then - continue - fi - if [ "$NODE" = "jenkins" ] ; then - JENKINS_NODENAME=jenkins - NODE="jenkins.debian.net" - else - case $ARCH in - amd64|i386) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1-2|sed 's#-build##' ) ;; - arm64) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1-2|sed 's#-sled##' ) ;; - armhf) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1) ;; - esac - fi - write_page "" - URL="https://jenkins.debian.net/view/reproducible/view/Node_maintenance/job/reproducible_maintenance_${ARCH}_${JENKINS_NODENAME}" - BADGE="$URL/badge/icon" - write_page "" - if [ "$JENKINS_NODENAME" = "jenkins" ] ; then - write_page "" - else - write_page "" - fi - for SUITE in ${SUITES} ; do - if [ "$SUITE" = "experimental" ] ; then - continue - fi - URL="https://jenkins.debian.net/view/reproducible/view/Debian_setup_${ARCH}/job/reproducible_setup_schroot_${SUITE}_${ARCH}_${JENKINS_NODENAME}" - BADGE="$URL/badge/icon" - write_page "" - done - for SUITE in ${SUITES} ; do - if [ "$JENKINS_NODENAME" = "jenkins" ] ; then - write_page "" - else - URL="https://jenkins.debian.net/view/reproducible/view/Debian_setup_${ARCH}/job/reproducible_setup_pbuilder_${SUITE}_${ARCH}_${JENKINS_NODENAME}" - BADGE="$URL/badge/icon" - write_page "" - fi - done - write_page "" - done - write_page "
Namemaintenanceworker.log linksschroot setup $SUITEpbuilder setup $SUITE
$JENKINS_NODENAME" - SHORTNAME=$(echo $NODE | cut -d '.' -f1) - for WORKER in $(grep "${ARCH}_" /srv/jenkins/bin/reproducible_build_service.sh | grep -v \# |grep $SHORTNAME | cut -d ')' -f1) ; do - write_page "" - write_page "$(echo $WORKER |cut -d '_' -f2) " - done - write_page "
" -done -write_page "

" -write_page_footer -publish_page debian - -for TYPE in daily weekly ; do - VIEW=nodes_${TYPE}_graphs - PAGE=index_${VIEW}.html - ARCH=amd64 - SUITE=unstable - echo "$(date -u) - starting to write $PAGE page." - write_page_header $VIEW "Nodes $TYPE graphs" - write_page $DISCLAIMER - write_page "

" - for ARCH in ${ARCHS} ; do - write_page "

$ARCH nodes

" - write_page "" - write_page "" - write_page "" - for NODE in jenkins $BUILD_NODES ; do - if [ -z "$(echo $NODE | grep $ARCH || true)" ] && [ "$NODE" != "jenkins" ] ; then - continue - elif [ "$NODE" = "jenkins" ] && [ "$ARCH" != "amd64" ] ; then - continue - fi - if [ "$NODE" = "jenkins" ] ; then - JENKINS_NODENAME=jenkins - NODE="jenkins.debian.net" - else - case $ARCH in - amd64|i386) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1-2|sed 's#-build##' ) ;; - arm64) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1-2|sed 's#-sled##' ) ;; - armhf) JENKINS_NODENAME=$(echo $NODE | cut -d "-" -f1) ;; - esac - fi - write_page "" - for GRAPH in jenkins_reproducible_builds cpu memory df swap load ; do - if [ "$JENKINS_NODENAME" = "jenkins" ] && [ "$GRAPH" = "jenkins_reproducible_builds" ] ; then - write_page "" - else - write_page "" - fi - done - write_page "" - - done - write_page "
Name
$JENKINS_NODENAME" - if [ "$TYPE" = "daily" ] ; then - IMG=day.png - else - IMG=week.png - fi - write_page "
" - done - write_page "

" - write_page_footer - publish_page debian -done - -# -# collect node information -# -echo "$(date -u) - Collecting information from nodes" -for NODE in $BUILD_NODES jenkins.debian.net ; do - if [ "$NODE" = "jenkins.debian.net" ] ; then - echo "$(date -u) - Trying to update $TARGET_DIR/$NODE." - /srv/jenkins/bin/reproducible_info.sh > $TARGET_DIR/$NODE - echo "$(date -u) - $TARGET_DIR/$NODE updated:" - cat $TARGET_DIR/$NODE - continue - fi - # call jenkins_master_wrapper.sh so we only need to track different ssh ports in one place - # jenkins_master_wrapper.sh needs NODE_NAME and JOB_NAME - export NODE_NAME=$NODE - export JOB_NAME=$JOB_NAME - echo "$(date -u) - Trying to update $TARGET_DIR/$NODE." - set +e - /srv/jenkins/bin/jenkins_master_wrapper.sh /srv/jenkins/bin/reproducible_info.sh > $TMPFILE_SRC - if [ $? -eq 1 ] ; then - echo "$(date -u) - Warning: could not update $TARGET_DIR/$NODE." - continue - fi - set -e - for KEY in $BUILD_ENV_VARS ; do - VALUE=$(egrep "^$KEY=" $TMPFILE_SRC | cut -d "=" -f2-) - if [ ! -z "$VALUE" ] ; then - echo "$KEY=$VALUE" >> $TMPFILE_NODE - fi - done - if [ -s $TMPFILE_NODE ] ; then - mv $TMPFILE_NODE $TARGET_DIR/$NODE - echo "$(date -u) - $TARGET_DIR/$NODE updated:" - cat $TARGET_DIR/$NODE - fi - rm -f $TMPFILE_SRC $TMPFILE_NODE -done -echo - -echo "$(date -u) - Showing node performance:" -TMPFILE1=$(mktemp) -TMPFILE2=$(mktemp) -TMPFILE3=$(mktemp) -NOW=$(date -u '+%Y-%m-%d %H:%m') -for i in $BUILD_NODES ; do - query_db "SELECT build_date FROM stats_build AS r WHERE ( r.node1='$i' OR r.node2='$i' )" > $TMPFILE1 2>/dev/null - j=$(wc -l $TMPFILE1|cut -d " " -f1) - k=$(cat $TMPFILE1|cut -d " " -f1|sort -u|wc -l) - l=$(echo "scale=1 ; ($j/$k)" | bc) - echo "$l builds/day ($j/$k) on $i" >> $TMPFILE2 - DATE=$(date '+%Y-%m-%d %H:%M' -d "-1 days") - m=$(query_db "SELECT count(build_date) FROM stats_build AS r WHERE ( r.node1='$i' OR r.node2='$i' ) AND r.build_date > '$DATE' " 2>/dev/null) - if [ "$m" = "" ] ; then m=0 ; fi - echo "$m builds in the last 24h on $i" >> $TMPFILE3 -done -rm $TMPFILE1 >/dev/null -sort -g -r $TMPFILE2 -echo -sort -g -r $TMPFILE3 -rm $TMPFILE2 $TMPFILE3 >/dev/null -echo - diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index f1096c58..cc51a268 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -450,12 +450,12 @@ my_description: 'Generate an HTML page showing the live status of all builds happening or scheduled by humans.' my_timed: 'H/6 * * * *' my_shellext: ".py" + - 'html_nodes_info': + my_description: 'Collect various information from build nodes used for reproducible builds and generate static HTML pages with various information about nodes.' + my_timed: '0 0 * * *' - 'create_meta_pkg_sets': my_description: 'Create meta package sets for later consumption when generating HTML stats.' my_timed: '0 6 * * H/3' - - 'nodes_info': - my_description: 'Collect various information from build nodes used for reproducible builds.' - my_timed: '0 0 * * *' - 'json': my_description: 'Generate https://reproducible.debian.net/userContent/reproducible.json for consumption by tracker.debian.org.' my_timed: '1 * * * *' -- cgit v1.2.3-54-g00ecf