reproducible Debian: refactor build service so that it only needs once service instead of 166

Signed-off-by: Holger Levsen <holger@layer-acht.org>

2 files changed, 93 insertions, 53 deletions

diff --git a/bin/reproducible_build_service.sh b/bin/reproducible_build_service.sh
index 0d3e260b..4f801580 100755
--- a/bin/reproducible_build_service.sh
+++ b/bin/reproducible_build_service.sh
@@ -3,65 +3,62 @@
 # Copyright © 2017 Holger Levsen (holger@layer-acht.org)
 # released under the GPLv=2
 
-# normally defined by jenkins
-JENKINS_URL=https://jenkins.debian.net
 set -e
 set -x
-# sleep up to 2.3 seconds (additionally to the random sleep reproducible_build.sh does anyway)
-/bin/sleep $(echo "scale=1 ; $(shuf -i 1-23 -n 1)/10" | bc )
 
-DEBUG=false
-. /srv/jenkins/bin/common-functions.sh
-common_init "$@"
 
-# common code defining db access
-. /srv/jenkins/bin/reproducible_common.sh
+choose_node() {
+	case $1 in
+		arm64_1)	NODE1=codethink-sled12-arm64	NODE2=codethink-sled15-arm64 ;;
+		i386_1)		NODE1=profitbricks-build2-i386	NODE2=profitbricks-build6-i386 ;;
+		i386_2)		NODE1=profitbricks-build6-i386	NODE2=profitbricks-build2-i386 ;;
+		i386_3)		NODE1=profitbricks-build2-i386	NODE2=profitbricks-build16-i386 ;;
+		i386_4)		NODE1=profitbricks-build16-i386	NODE2=profitbricks-build2-i386 ;;
+		i386_5)		NODE1=profitbricks-build12-i386	NODE2=profitbricks-build6-i386 ;;
+		i386_6)		NODE1=profitbricks-build6-i386	NODE2=profitbricks-build12-i386 ;;
+		i386_7)		NODE1=profitbricks-build12-i386	NODE2=profitbricks-build16-i386 ;;
+		i386_8)		NODE1=profitbricks-build16-i386	NODE2=profitbricks-build12-i386 ;;
+		i386_9)		NODE1=profitbricks-build2-i386	NODE2=profitbricks-build6-i386 ;;
+		i386_10)	NODE1=profitbricks-build6-i386	NODE2=profitbricks-build2-i386 ;;
+		i386_11)	NODE1=profitbricks-build2-i386	NODE2=profitbricks-build16-i386 ;;
+		i386_12)	NODE1=profitbricks-build16-i386	NODE2=profitbricks-build2-i386 ;;
+		i386_13)	NODE1=profitbricks-build12-i386	NODE2=profitbricks-build6-i386 ;;
+		i386_14)	NODE1=profitbricks-build6-i386	NODE2=profitbricks-build12-i386 ;;
+		i386_15)	NODE1=profitbricks-build12-i386	NODE2=profitbricks-build16-i386 ;;
+		i386_16)	NODE1=profitbricks-build16-i386	NODE2=profitbricks-build12-i386 ;;
+		i386_17)	NODE1=profitbricks-build2-i386	NODE2=profitbricks-build6-i386 ;;
+		i386_18)	NODE1=profitbricks-build6-i386	NODE2=profitbricks-build2-i386 ;;
+		i386_19)	NODE1=profitbricks-build2-i386	NODE2=profitbricks-build16-i386 ;;
+		i386_20)	NODE1=profitbricks-build16-i386	NODE2=profitbricks-build2-i386 ;;
+		i386_21)	NODE1=profitbricks-build12-i386	NODE2=profitbricks-build6-i386 ;;
+		i386_22)	NODE1=profitbricks-build6-i386	NODE2=profitbricks-build12-i386 ;;
+		i386_23)	NODE1=profitbricks-build12-i386	NODE2=profitbricks-build16-i386 ;;
+		i386_24)	NODE1=profitbricks-build16-i386	NODE2=profitbricks-build12-i386 ;;
+		*)		echo "Sleeping 60min"
+				sleep 60m
+				exit 0
+				;;
+	esac
+}
 
-BUILD_BASE=/var/lib/jenkins/userContent/reproducible/debian/build_service/$1
-OLD_ID=$(ls -1rt $BUILD_BASE|grep -v latest|sort -n|tail -1)
-let BUILD_ID=OLD_ID+1
-mkdir -p $BUILD_BASE/$BUILD_ID
-rm -f $BUILD_BASE/latest
-ln -sf $BUILD_ID $BUILD_BASE/latest
+NODE1=""
+NODE2=""
+ARCH=i386
+# let's start with two workers only…
+for i in $(seq 1 2) ; do
+        # sleep up to 2.3 seconds (additionally to the random sleep reproducible_build.sh does anyway)
+        /bin/sleep $(echo "scale=1 ; $(shuf -i 1-23 -n 1)/10" | bc )
 
-export BUILD_URL=https://jenkins.debian.net/userContent/build_service/$1/
-export BUILD_ID=$BUILD_ID
-export JOB_NAME="reproducible_builder_$1"
-export
+	WORKER_NAME=${ARCH}_$i
+	choose_node $WORKER_NAME
+	BUILD_BASE=/var/lib/jenkins/userContent/reproducible/debian/build_service/$WORKER_NAME
+	mkdir -p $BUILD_BASE
+	echo "$(date --utc) - Starting $WORKER_NAME"
+	/srv/jenkins/bin/reproducible_build_service_worker.sh $WORKER_NAME $NODE1 $NODE2 >$BUILD_BASE/worker.log 2>&1 &
+done
 
-case $1 in
-	arm64_1)	NODE1=codethink-sled12-arm64	NODE2=codethink-sled15-arm64 ;;
-	i386_1)		NODE1=profitbricks-build2-i386	NODE2=profitbricks-build6-i386 ;;
-	i386_2)		NODE1=profitbricks-build6-i386	NODE2=profitbricks-build2-i386 ;;
-	i386_3)		NODE1=profitbricks-build2-i386	NODE2=profitbricks-build16-i386 ;;
-	i386_4)		NODE1=profitbricks-build16-i386	NODE2=profitbricks-build2-i386 ;;
-	i386_5)		NODE1=profitbricks-build12-i386	NODE2=profitbricks-build6-i386 ;;
-	i386_6)		NODE1=profitbricks-build6-i386	NODE2=profitbricks-build12-i386 ;;
-	i386_7)		NODE1=profitbricks-build12-i386	NODE2=profitbricks-build16-i386 ;;
-	i386_8)		NODE1=profitbricks-build16-i386	NODE2=profitbricks-build12-i386 ;;
-	i386_9)		NODE1=profitbricks-build2-i386	NODE2=profitbricks-build6-i386 ;;
-	i386_10)	NODE1=profitbricks-build6-i386	NODE2=profitbricks-build2-i386 ;;
-	i386_11)	NODE1=profitbricks-build2-i386	NODE2=profitbricks-build16-i386 ;;
-	i386_12)	NODE1=profitbricks-build16-i386	NODE2=profitbricks-build2-i386 ;;
-	i386_13)	NODE1=profitbricks-build12-i386	NODE2=profitbricks-build6-i386 ;;
-	i386_14)	NODE1=profitbricks-build6-i386	NODE2=profitbricks-build12-i386 ;;
-	i386_15)	NODE1=profitbricks-build12-i386	NODE2=profitbricks-build16-i386 ;;
-	i386_16)	NODE1=profitbricks-build16-i386	NODE2=profitbricks-build12-i386 ;;
-	i386_17)	NODE1=profitbricks-build2-i386	NODE2=profitbricks-build6-i386 ;;
-	i386_18)	NODE1=profitbricks-build6-i386	NODE2=profitbricks-build2-i386 ;;
-	i386_19)	NODE1=profitbricks-build2-i386	NODE2=profitbricks-build16-i386 ;;
-	i386_20)	NODE1=profitbricks-build16-i386	NODE2=profitbricks-build2-i386 ;;
-	i386_21)	NODE1=profitbricks-build12-i386	NODE2=profitbricks-build6-i386 ;;
-	i386_22)	NODE1=profitbricks-build6-i386	NODE2=profitbricks-build12-i386 ;;
-	i386_23)	NODE1=profitbricks-build12-i386	NODE2=profitbricks-build16-i386 ;;
-	i386_24)	NODE1=profitbricks-build16-i386	NODE2=profitbricks-build12-i386 ;;
-	*)		echo "Sleeping 60min" 
-			sleep 60m
-			exit 0
-			;;
-esac
-
-/srv/jenkins/bin/reproducible_build.sh $NODE1 $NODE2 >$BUILD_BASE/$BUILD_ID/console.log 2>&1
+# not sure if this is really needed…
+while true ; do sleep 5m ; done
 
 # TODO left:
 # * maintenance job needs to:
@@ -70,5 +67,4 @@ esac
 # * logs should auto display in browser like with jenkins… (long-polling, meta-refresh, something)
 #   - there's an NPH solution pointed out by Xtaran
 # * translate yaml into a script or such to create those service files (done for i386 for now)
-# * have one service per arch, starting X build jobs instead?
 
diff --git a/bin/reproducible_build_service_worker.sh b/bin/reproducible_build_service_worker.sh
new file mode 100755
index 00000000..b057b77a
--- /dev/null
+++ b/bin/reproducible_build_service_worker.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+# Copyright © 2017 Holger Levsen (holger@layer-acht.org)
+# released under the GPLv=2
+
+# normally defined by jenkins
+JENKINS_URL=https://jenkins.debian.net
+set -e
+set -x
+
+WORKER_NAME=$1
+NODE1=$2
+NODE2=$3
+
+DEBUG=false
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code defining db access
+. /srv/jenkins/bin/reproducible_common.sh
+
+while true ; do
+	# TODO
+	# - test here if the builder service is actually running…
+
+	# sleep up to 2.3 seconds (additionally to the random sleep reproducible_build.sh does anyway)
+	/bin/sleep $(echo "scale=1 ; $(shuf -i 1-23 -n 1)/10" | bc )
+
+	BUILD_BASE=/var/lib/jenkins/userContent/reproducible/debian/build_service/$WORKER_NAME
+	OLD_ID=$(ls -1rt $BUILD_BASE|egrep -v "(latest|worker.log)" |sort -n|tail -1)
+	let BUILD_ID=OLD_ID+1
+	mkdir -p $BUILD_BASE/$BUILD_ID
+	rm -f $BUILD_BASE/latest
+	ln -sf $BUILD_ID $BUILD_BASE/latest
+
+	export BUILD_URL=https://jenkins.debian.net/userContent/build_service/$WORKER_NAME/
+	export BUILD_ID=$BUILD_ID
+	export JOB_NAME="reproducible_builder_$WORKER_NAME"
+	export
+
+	/srv/jenkins/bin/reproducible_build.sh $NODE1 $NODE2 >$BUILD_BASE/$BUILD_ID/console.log 2>&1
+done
+
+