#!/bin/bash # # Copyright 2014-2017 Holger Levsen # # released under the GPLv=2 # # deployment helper for jenkins.debian.net and build nodes # (misses freebsd node) # disclaimers: # this script grew over the years… # the code is horrible and was always ment to be a quick local hack and never to be published # use at your own risk. it might eat your cats. you have been warned. # the HOSTS lists below is yet another code duplication… # useful things: # ./deploy_jdn all - deploy on all nodes (and handle 398 days hosts properly) # ./deploy_jdn $host - deploy on $host and jenkins (and handle 398 days hosts properly) # ./deploy_jdn all $foo - run "$foo" on all nodes (and handle 398 days hosts properly) # ./deploy_jdn - deploy on jenkins only # ./deploy_jdn jenkins - deploy on jenkins only # ./deploy_jdn jenkins pb10 - deploy on jenkins and pb10 # ./deploy_jdn jenkins 10 - deploy on jenkins and pb10 # ./deploy_jdn jenkins 3 4 - deploy on jenkins and pb4 and pb4 # ./deploy_jdn upgrade - run "apt-get update && upgrade && clean" everywhere # ./deploy_jdn upgradey - run "apt-get upgrade -y" everywhere # ./deploy_jdn rmstamp - delete stamp files everywhere START=$(date +'%s') GIT_REPO="git://anonscm.debian.org/qa/jenkins.debian.net.git" export HOSTS="codethink-sled9-arm64.debian.net codethink-sled10-arm64.debian.net codethink-sled11-arm64.debian.net codethink-sled12-arm64.debian.net codethink-sled13-arm64.debian.net codethink-sled14-arm64.debian.net codethink-sled15-arm64.debian.net codethink-sled16-arm64.debian.net profitbricks-build2-i386.debian.net profitbricks-build6-i386.debian.net profitbricks-build12-i386.debian.net profitbricks-build16-i386.debian.net profitbricks-build1-amd64.debian.net profitbricks-build3-amd64.debian.net profitbricks-build4-amd64.debian.net profitbricks-build5-amd64.debian.net profitbricks-build7-amd64.debian.net profitbricks-build9-amd64.debian.net profitbricks-build10-amd64.debian.net profitbricks-build11-amd64.debian.net profitbricks-build15-amd64.debian.net bbx15-armhf-rb.debian.net bpi0-armhf-rb.debian.net cb3a-armhf-rb.debian.net cbxi4a-armhf-rb.debian.net cbxi4b-armhf-rb.debian.net cbxi4pro0-armhf-rb.debian.net ff2a-armhf-rb.debian.net ff2b-armhf-rb.debian.net ff4a-armhf-rb.debian.net ff64a-armhf-rb.debian.net hb0-armhf-rb.debian.net jtk1a-armhf-rb.debian.net jtk1b-armhf-rb.debian.net jtx1a-armhf-rb.debian.net jtx1b-armhf-rb.debian.net jtx1c-armhf-rb.debian.net odu3a-armhf-rb.debian.net odxu4a-armhf-rb.debian.net odxu4b-armhf-rb.debian.net odxu4c-armhf-rb.debian.net opi2a-armhf-rb.debian.net opi2b-armhf-rb.debian.net opi2c-armhf-rb.debian.net p64b-armhf-rb.debian.net p64c-armhf-rb.debian.net rpi2b-armhf-rb.debian.net rpi2c-armhf-rb.debian.net wbd0-armhf-rb.debian.net wbq0-armhf-rb.debian.net root@jenkins.debian.net" ALL_HOSTS=$HOSTS echo echo -n "$(date) - " # real command, for running manually: cd ~jenkins-adm/jenkins.debian.net/ ; sudo -u jenkins-adm git pull ; ./update_jdn.sh read -r -d '' real_command <<-EOF set -e export LANG=C cd ~jenkins-adm if [ ! -d jenkins.debian.net ]; then [ -x /usr/bin/git ] || sudo apt-get install -y git sudo -u jenkins-adm git clone $GIT_REPO cd jenkins.debian.net else cd jenkins.debian.net sudo -u jenkins-adm git pull $GIT_REPO fi ./update_jdn.sh 2>&1 | sudo tee -a /var/log/jenkins/update_jdn.log EOF reset_clock=true if [ "$1" = "all" ] ; then echo -n "Running j.d.n.git updates on $HOSTS now" # reset_clock can be false as update_jdn.sh sets the time reset_clock=false shift if [ ! -z "$1" ] ; then real_command="$@" echo -n "Running '$real_command' on $HOSTS now." real_command="$@ && echo '__reallyreally=ok__'" fi elif [ "$1" = "upgrade" ] ; then real_command="export LANG=C && sudo apt-get update && sudo apt-get upgrade && sudo apt-get dist-upgrade && sudo apt-get clean" shift elif [ "$1" = "upgradey" ] ; then real_command="export LANG=C && sudo apt-get update && sudo apt-get -y upgrade && sudo apt-get -y dist-upgrade && sudo apt-get clean" shift elif [ "$1" = "rmstamp" ] ; then real_command="sudo rm -f /var/log/jenkins/*stamp && echo '__reallyreally=ok__'" reset_clock=false shift elif [ "$1" = "check" ] ; then real_command="pgrep -l -a -f _build.sh" reset_clock=false shift elif [ "$1" = "" ] ; then export HOSTS="root@jenkins.debian.net" elif [ "$1" = "jenkins" ] ; then export HOSTS="root@jenkins.debian.net" shift for i in $@ ; do case $i in 1|pb1) HOSTS="$HOSTS profitbricks-build1-amd64.debian.net" ;; 2|pb2) HOSTS="$HOSTS profitbricks-build2-i386.debian.net" ;; 3|pb3) HOSTS="$HOSTS profitbricks-build3-amd64.debian.net" ;; 3+4|pb3+pb4|pb3+4) HOSTS="$HOSTS profitbricks-build3-amd64.debian.net profitbricks-build4-amd64.debian.net" ;; 4|pb4) HOSTS="$HOSTS profitbricks-build4-amd64.debian.net" ;; 5|pb5) HOSTS="$HOSTS profitbricks-build5-amd64.debian.net" ;; 6|pb6) HOSTS="$HOSTS profitbricks-build6-i386.debian.net" ;; 7|pb7) HOSTS="$HOSTS profitbricks-build7-amd64.debian.net" ;; 9|pb9) HOSTS="$HOSTS profitbricks-build9-amd64.debian.net" ;; 10|pb10) HOSTS="$HOSTS profitbricks-build10-amd64.debian.net" ;; 11|pb11) HOSTS="$HOSTS profitbricks-build11-amd64.debian.net" ;; 15|pb15) HOSTS="$HOSTS profitbricks-build15-amd64.debian.net" ;; c9) HOSTS="$HOSTS codethink-sled9-arm64.debian.net" ;; c10) HOSTS="$HOSTS codethink-sled10-arm64.debian.net" ;; c11) HOSTS="$HOSTS codethink-sled11-arm64.debian.net" ;; c12) HOSTS="$HOSTS codethink-sled12-arm64.debian.net" ;; c13) HOSTS="$HOSTS codethink-sled13-arm64.debian.net" ;; c14) HOSTS="$HOSTS codethink-sled14-arm64.debian.net" ;; c15) HOSTS="$HOSTS codethink-sled15-arm64.debian.net" ;; c16) HOSTS="$HOSTS codethink-sled16-arm64.debian.net" ;; armhf|amd64|i386|arm64) HOSTS="$HOSTS $(echo $ALL_HOSTS | sed 's# #\n#g' | grep $i)" ;; *) if ping -c 1 $i ; then HOSTS="$HOSTS $i" ; fi ;; esac done echo -n "Running j.d.n.git ('$real_command') updates on $HOSTS now" fi BG="" get_arch_color() { case $1 in *amd64*) BG=lightgreen ;; *i386*) BG=lightblue ;; *arm64*) BG=orange ;; *armhf*) BG=lightyellow ;; *jenkins.debian.*) BG=yellow ;; *) BG=white ;; esac } LOG=$(mktemp -u) STSOH="" for i in $HOSTS ; do STSOH="$i $STSOH" done HOSTS=$STSOH for i in $HOSTS ; do echo -n "." command="$real_command" if $reset_clock ; then case $i in profitbricks-build4-amd64*|profitbricks-build5-amd64*|profitbricks-build6-i386*|profitbricks-build15-amd64*|profitbricks-build16-i386*) # set correct future date command="sudo ntpdate -b de.pool.ntp.org && $real_command && sudo date --set=\"+398 days +6 hours + 23 minutes\" && echo '__$(echo $i|cut -d '.' -f1)=ok__'" ;; codethink-sled9*|codethink-sled11*|codethink-sled13*|codethink-sled15*) # set correct future date command="sudo ntpdate -b de.pool.ntp.org && $real_command && sudo date --set=\"+398 days +6 hours + 23 minutes\" && echo '__$(echo $i|cut -d '.' -f1)=ok__'" ;; *) ;; esac fi get_arch_color $i xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -o 'BatchMode = yes' -t $i '$command' | tee $LOG.$i ; sleep 2 ; touch $LOG.$i.done" & done sleep 3 COUNTER=0 for i in $HOSTS ; do while ! test -f $LOG.$i.done ; do let COUNTER+=1 sleep 1 echo -n "." if [ $COUNTER -eq 42 ] ; then echo echo -n "$LOG.$i.done still doesnt exist, how strange…" COUNTER=0 continue fi done done echo PROBLEMS="" for i in $HOSTS ; do HNAME1=$(echo $i | cut -d "@" -f2 | cut -d "." -f1|cut -d "-" -f1) # pb nodes (h01ger) HNAME2=$(echo $i | cut -d "@" -f2 | cut -d "." -f1) # non -armhf ones (vagrant) TAIL=$(tail -1 $LOG.$i 2>/dev/null) if [ "$i" = "root@jenkins.debian.net" ] ; then if ! ( [[ "$TAIL" =~ "__$HNAME1=ok__" ]] || [[ "$TAIL" =~ "__$HNAME2=ok__" ]] || [[ "$TAIL" =~ "__reallyreally=ok__" ]] || [[ "$TAIL" =~ "0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." ]] ) ; then echo "Problems on $i:" fi cat $LOG.$i rm $LOG.$i $LOG.$i.done > /dev/null elif [[ "$TAIL" =~ "__$HNAME1=ok__" ]] || [[ "$TAIL" =~ "__$HNAME2=ok__" ]] || [[ "$TAIL" =~ "__reallyreally=ok__" ]] || [[ "$TAIL" =~ "0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." ]] || [ ! -f $LOG.$i ] ; then rm $LOG.$i $LOG.$i.done > /dev/null else echo "Problems on $i: $TAIL" if [ -z "$PROBLEMS" ] ; then PROBLEMS=" $i" else PROBLEMS=" $i\n$PROBLEMS" fi get_arch_color $i xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "less +G $LOG.$i ; rm $LOG.$i $LOG.$i.done" & fi done echo echo "$(echo $HOSTS | sed -s "s# #\n#g" | wc -l) hosts updated." if [ ! -z "$PROBLEMS" ] ; then echo "Problems on:" echo -e "$PROBLEMS" echo fi END=$(date +'%s') DURATION=$(( $END - $START )) HOUR=$(echo "$DURATION/3600"|bc) MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc) SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc) echo "$(date) - total duration: ${HOUR}h ${MIN}m ${SEC}s." echo