#!/bin/bash
#
# Copyright 2014-2016 Holger Levsen <holger@layer-acht.org>
#
# released under the GPLv=2
#
# deployment helper for jenkins.debian.net and build nodes
# (misses freebsd node)

# disclaimers:
#   this script grew over the years…
#   the code is horrible and was always ment to be a quick local hack and never to be published
#   use at your own risk. it might eat your cats. you have been warned.
#   the HOSTS lists below is yet another code duplication…

# useful things:
# ./deploy_jdn all		- deploy on all nodes (and handle 398 days hosts properly)
# ./deploy_jdn all $foo		- run "$foo" on all nodes (and handle 398 days hosts properly)
# ./deploy_jdn 			- deploy on jenkins only
# ./deploy_jdn jenkins		- deploy on jenkins only
# ./deploy_jdn jenkins pb10	- deploy on jenkins and pb10
# ./deploy_jdn jenkins 10	- deploy on jenkins and pb10
# ./deploy_jdn jenkins 4 5	- deploy on jenkins and pb4 and pb5
# ./deploy_jdn upgrade		- run "apt-get update && upgrade && clean" everywhere
# ./deploy_jdn upgradey		- run "apt-get upgrade -y" everywhere
# ./deploy_jdn rmstamp		- delete stamp files everywhere


START=$(date +'%s')
GIT_REPO="git://anonscm.debian.org/qa/jenkins.debian.net.git"
export HOSTS="profitbricks-build1-amd64.debian.net
profitbricks-build2-i386.debian.net
profitbricks-build3-amd64.debian.net
profitbricks-build4-amd64.debian.net
profitbricks-build5-amd64.debian.net
profitbricks-build6-i386.debian.net
profitbricks-build7-amd64.debian.net
profitbricks-build9-amd64.debian.net
profitbricks-build10-amd64.debian.net
profitbricks-build11-amd64.debian.net
profitbricks-build12-i386.debian.net
profitbricks-build15-amd64.debian.net
profitbricks-build16-i386.debian.net
bbx15-armhf-rb.debian.net
bpi0-armhf-rb.debian.net
cb3a-armhf-rb.debian.net
cbxi4a-armhf-rb.debian.net
cbxi4b-armhf-rb.debian.net
cbxi4pro0-armhf-rb.debian.net
ff2a-armhf-rb.debian.net
ff2b-armhf-rb.debian.net
ff4a-armhf-rb.debian.net
hb0-armhf-rb.debian.net
jtk1a-armhf-rb.debian.net
odu3a-armhf-rb.debian.net
odxu4-armhf-rb.debian.net
odxu4b-armhf-rb.debian.net
odxu4c-armhf-rb.debian.net
opi2a-armhf-rb.debian.net
opi2b-armhf-rb.debian.net
opi2c-armhf-rb.debian.net
rpi2b-armhf-rb.debian.net
rpi2c-armhf-rb.debian.net
wbd0-armhf-rb.debian.net
wbq0-armhf-rb.debian.net
codethink-sled9-arm64.debian.net
codethink-sled10-arm64.debian.net
codethink-sled11-arm64.debian.net
codethink-sled12-arm64.debian.net
codethink-sled13-arm64.debian.net
codethink-sled14-arm64.debian.net
codethink-sled15-arm64.debian.net
codethink-sled16-arm64.debian.net
root@jenkins.debian.net"

ALL_HOSTS=$HOSTS

echo
echo -n "$(date) - "
# real command, for running manually: cd ~jenkins-adm/jenkins.debian.net/ ; sudo -u jenkins-adm git pull ; ./update_jdn.sh
read -r -d '' real_command <<-EOF
set -e
export LANG=C
cd ~jenkins-adm
if [ ! -d jenkins.debian.net ]; then
    [ -x /usr/bin/git ] || sudo apt-get install -y git
    sudo -u jenkins-adm git clone $GIT_REPO
    cd jenkins.debian.net
else
    cd jenkins.debian.net

    sudo -u jenkins-adm git pull $GIT_REPO
fi
./update_jdn.sh
EOF
reset_clock=true
if [ "$1" = "all" ] ; then
	echo -n "Running j.d.n.git updates on $HOSTS now"
	# reset_clock can be false as update_jdn.sh sets the time
	reset_clock=false
	shift
	if [ ! -z "$1" ] ; then
		real_command="$@"
		echo -n "Running '$real_command' on $HOSTS now."
		real_command="$@ && echo '__reallyreally=ok__'"
	fi
elif [ "$1" = "upgrade" ] ; then
	real_command="export LANG=C && sudo apt-get update && sudo apt-get upgrade && sudo apt-get dist-upgrade && sudo apt-get clean"
	shift
elif [ "$1" = "upgradey" ] ; then
	real_command="export LANG=C && sudo apt-get update && sudo apt-get -y upgrade && sudo apt-get -y dist-upgrade && sudo apt-get clean"
	shift
elif [ "$1" = "rmstamp" ] ; then
	real_command="sudo rm -f /var/log/jenkins/*stamp && echo '__reallyreally=ok__'"
	reset_clock=false
	shift
elif [ "$1" = "check" ] ; then
	real_command="pgrep -l -a -f _build.sh"
	reset_clock=false
	shift
elif [ "$1" = "" ] || [ "$1" = "jenkins" ] ; then
        export HOSTS="root@jenkins.debian.net"
	shift
	for i in $@ ; do
		case $i in
			1|pb1)	 HOSTS="$HOSTS profitbricks-build1-amd64.debian.net" ;;
			2|pb2)	 HOSTS="$HOSTS profitbricks-build2-i386.debian.net" ;;
			3|pb3)	 HOSTS="$HOSTS profitbricks-build3-amd64.debian.net" ;;
			3+4|pb3+pb4|pb3+4) HOSTS="$HOSTS profitbricks-build3-amd64.debian.net profitbricks-build4-amd64.debian.net" ;;
			4|pb4)	 HOSTS="$HOSTS profitbricks-build4-amd64.debian.net" ;;
			5|pb5)	 HOSTS="$HOSTS profitbricks-build5-amd64.debian.net" ;;
			6|pb6)	 HOSTS="$HOSTS profitbricks-build6-i386.debian.net" ;;
			7|pb7)	 HOSTS="$HOSTS profitbricks-build7-amd64.debian.net" ;;
			9|pb9)	 HOSTS="$HOSTS profitbricks-build9-amd64.debian.net" ;;
			10|pb10) HOSTS="$HOSTS profitbricks-build10-amd64.debian.net" ;;
			11|pb11) HOSTS="$HOSTS profitbricks-build11-amd64.debian.net" ;;
			15|pb15) HOSTS="$HOSTS profitbricks-build15-amd64.debian.net" ;;
			c9)      HOSTS="$HOSTS codethink-sled9-arm64.debian.net" ;;
			c10)     HOSTS="$HOSTS codethink-sled10-arm64.debian.net" ;;
			c11)     HOSTS="$HOSTS codethink-sled11-arm64.debian.net" ;;
			c12)     HOSTS="$HOSTS codethink-sled12-arm64.debian.net" ;;
			c13)     HOSTS="$HOSTS codethink-sled13-arm64.debian.net" ;;
			c14)     HOSTS="$HOSTS codethink-sled14-arm64.debian.net" ;;
			c15)     HOSTS="$HOSTS codethink-sled15-arm64.debian.net" ;;
			c16)     HOSTS="$HOSTS codethink-sled16-arm64.debian.net" ;;
			armhf|amd64|i386|arm64)	 HOSTS="$HOSTS $(echo $ALL_HOSTS | sed 's# #\n#g' | grep $i)" ;;
			*) ;;
		esac
	done
	echo -n "Running j.d.n.git ('$real_command') updates on $HOSTS now"
fi
BG=""

get_arch_color() {
	case $1 in
		*amd64*)		BG=lightgreen ;;
		*i386*)			BG=lightblue ;;
		*arm64*)		BG=orange ;;
		*armhf*)		BG=lightyellow ;;
		*jenkins.debian.*)	BG=yellow ;;
		*)			BG=white ;;
	esac
}

LOG=$(mktemp -u)
for i in $HOSTS ; do
	echo -n "."
       	command="$real_command"
	if $reset_clock ; then
 		case $i in
			profitbricks-build4-amd64*|profitbricks-build5-amd64*|profitbricks-build6-i386*|profitbricks-build15-amd64*|profitbricks-build16-i386*)
				#  set correct future date
				command="sudo ntpdate -b de.pool.ntp.org && $real_command && sudo date --set=\"+398 days +6 hours + 23 minutes\" && echo '__$(echo $i|cut -d '.' -f1)=ok__'"
				;;
			codethink9*|codethink11*|codethink13*|codethink15*)
				#  set correct future date
				command="sudo ntpdate -b de.pool.ntp.org && $real_command && sudo date --set=\"+398 days +6 hours + 23 minutes\" && echo '__$(echo $i|cut -d '.' -f1)=ok__'"
				;;
			*)
				;;
		esac
	fi
	get_arch_color $i
	xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "ssh -o 'BatchMode = yes' -t $i '$command' | tee $LOG.$i ; sleep 2 ; touch $LOG.$i.done" &
done
sleep 3
COUNTER=0
for i in $HOSTS ; do
	while ! test -f $LOG.$i.done ; do
		let COUNTER+=1
		sleep 1
		echo -n "."
		if [ $COUNTER -eq 42 ] ; then
			echo 
			echo -n "$LOG.$i.done still doesnt exist, how strange…"
			COUNTER=0
			continue
		fi
	done
done
echo

PROBLEMS=""
for i in $HOSTS ; do
	HNAME1=$(echo $i | cut -d "@" -f2 | cut -d "." -f1|cut -d "-" -f1)	# pb nodes (h01ger)
	HNAME2=$(echo $i | cut -d "@" -f2 | cut -d "." -f1)			# non -armhf ones (vagrant)
	TAIL=$(tail -1 $LOG.$i 2>/dev/null)
	if [ "$i" = "root@jenkins.debian.net" ] ; then
		if ! ( [[ "$TAIL" =~ "__$HNAME1=ok__" ]] || [[ "$TAIL" =~ "__$HNAME2=ok__" ]] || [[ "$TAIL" =~ "__reallyreally=ok__" ]] || [[ "$TAIL" =~ "0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." ]] ) ; then
			echo "Problems on $i:"
		fi
		cat $LOG.$i
		rm $LOG.$i $LOG.$i.done > /dev/null
	elif [[ "$TAIL" =~ "__$HNAME1=ok__" ]] || [[ "$TAIL" =~ "__$HNAME2=ok__" ]] || [[ "$TAIL" =~ "__reallyreally=ok__" ]] || [[ "$TAIL" =~ "0 upgraded, 0 newly installed, 0 to remove and 0 not upgraded." ]] || [ ! -f $LOG.$i ] ; then
		rm $LOG.$i $LOG.$i.done > /dev/null
	else
		echo "Problems on $i: $TAIL"
		if [ -z "$PROBLEMS" ] ; then
			PROBLEMS=" $i"
		else
			PROBLEMS=" $i\n$PROBLEMS"
		fi
		get_arch_color $i
		xterm -class deploy-jenkins -bg $BG -fa 'DejaVuSansMono' -fs 10 -e "less +G $LOG.$i ; rm $LOG.$i $LOG.$i.done" &
	fi
done
echo

echo "$(echo $HOSTS | sed -s "s# #\n#g" | wc -l) hosts updated."
if [ ! -z "$PROBLEMS" ] ; then
	echo "Problems on:"
	echo -e "$PROBLEMS"
	echo
fi
END=$(date +'%s')
DURATION=$(( $END - $START ))
HOUR=$(echo "$DURATION/3600"|bc)
MIN=$(echo "($DURATION-$HOUR*3600)/60"|bc)
SEC=$(echo "$DURATION-$HOUR*3600-$MIN*60"|bc)
echo "$(date) - total duration: ${HOUR}h ${MIN}m ${SEC}s."
echo
echo "the end."