reproducible fedora: first shot at building rpm based distros.

5 files changed, 251 insertions, 4 deletions

diff --git a/TODO b/TODO
index 80f708b1..4b1d4f51 100644
--- a/TODO
+++ b/TODO
@@ -282,12 +282,15 @@ This is about Debian, below are more todo entries for other projects…
 ==== reproducible Fedora
 
 * build script
-** call the script reproducible_rpms.sh as it can also build OpenSuSE packages
-** then use yumdownloader to download rpms: 'yumdownloader --source sudo'
-** use mock to build:  'mock -r fedora-23-x86_64 --resultdir=. sudo-1.8.14p3-1.fc23.src.rpm'
+** how to get a list of packages?
+** no variations introduced yet:
+*** use '-j$NUM_CPU' and 'NEW_NUM_CPU=$(echo $NUM_CPU-1|bc)'
+*** modify TZ, LANG, LC_ALL, umask
+* other bits:
+** use modified rpmbuild package
 * hosts/pb-build3/etc/yum/repos.d/* is really not sooo good but works…
-* how to get a list of packages?
 * build rawhide instead of fedora-23?
+** when adding rawhide make sure this is not needed: 'yumdownloader --enablerepo=fedora-23-src --source sudo'
 
 * more notes:
 ** http://blog.packagecloud.io/eng/2015/05/11/building-rpm-packages-with-mock/
diff --git a/bin/jenkins_node_wrapper.sh b/bin/jenkins_node_wrapper.sh
index 22aa3e60..ccd1645a 100755
--- a/bin/jenkins_node_wrapper.sh
+++ b/bin/jenkins_node_wrapper.sh
@@ -109,6 +109,8 @@ elif [[ "$*" =~ reproducible_setup_mock_fedora23_x86_64 ]] ; then
 	exec /srv/jenkins/bin/reproducible_setup_mock.sh fedora-23 x86_64 ; croak "Exec failed";
 elif [ "$1" = "/srv/jenkins/bin/reproducible_build_archlinux_pkg.sh" ] && ( [ "$2" = "1" ] || [ "$2" = "2" ] ) ; then
 	exec /srv/jenkins/bin/reproducible_build_archlinux_pkg.sh "$2" "$3" "$4" "$5" ; croak "Exec failed";
+elif [ "$1" = "/srv/jenkins/bin/reproducible_build_rpm.sh" ] && ( [ "$2" = "1" ] || [ "$2" = "2" ] ) ; then
+	exec /srv/jenkins/bin/reproducible_build_rpm.sh "$2" "$3" "$4" "$5" "$6" ; croak "Exec failed";
 elif [ "$*" = "some_jenkins_job_name" ] ; then
 	exec echo run any commands here ; croak "Exec failed";
 fi
diff --git a/bin/reproducible_build_rpm.sh b/bin/reproducible_build_rpm.sh
new file mode 100755
index 00000000..37c74e6f
--- /dev/null
+++ b/bin/reproducible_build_rpm.sh
@@ -0,0 +1,228 @@
+#!/bin/bash
+
+# Copyright 2015 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv=2
+
+DEBUG=false
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+# common code
+. /srv/jenkins/bin/reproducible_common.sh
+
+set -e
+
+cleanup_all() {
+	cd
+	# delete makepkg build dir
+	if [ ! -z $SRCPACKAGE ] && [ -d /tmp/$SRCPACKAGE-$(basename $TMPDIR) ] ; then
+		rm -r /tmp/$SRCPACKAGE-$(basename $TMPDIR)
+	fi
+	# delete main work dir (only on master)
+	if [ "$MODE" = "master" ] ; then
+		rm $TMPDIR -r
+		echo "$(date -u) - $TMPDIR deleted."
+	fi
+	rm -f $DUMMY > /dev/null || true
+}
+
+handle_remote_error() {
+	MESSAGE="${BUILD_URL}console got remote error $1"
+	echo "$(date -u ) - $MESSAGE" | tee -a /var/log/jenkins/reproducible-remote-error.log
+	echo "Sleeping 5m before aborting the job."
+	sleep 5m
+	exec /srv/jenkins/bin/abort.sh
+	exit 0
+}
+
+
+download_package() {
+	echo "$(date -u ) - downloading ${SRCPACKAGE} for $RELEASE now."
+	yumdownloader --source ${SRCPACKAGE}
+	SRC_RPM="$(ls $SRCPACKAGE*.src.rpm)"
+}
+
+choose_package() {
+	echo "$(date -u ) - choosing package to be build."
+	local MIN_AGE=6
+	for PKG in sudo ; do
+		# build package if it has never build or at least $MIN_AGE days ago
+		if [ ! -d $BASE/rpm/$RELEASE/$ARCH/$PKG ] || [ ! -z $(find $BASE/rpm/$RELEASE/$ARCH/ -name $PKG -mtime +$MIN_AGE) ] ; then
+			SRCPACKAGE=$PKG
+			echo "$(date -u ) - building package $PKG from '$RELEASE' on '$ARCH' now..."
+				# very simple locking…
+				mkdir -p $BASE/rpm/$RELEASE/$ARCH/$PKG
+				touch $BASE/rpm/$RELEASE/$ARCH/$PKG
+				# break out of the loop and then out of this function too,
+				# to build this package…
+				break
+			fi
+		done
+	done
+	if [ -z $SRCPACKAGE ] ; then
+		echo "$(date -u ) - no package found to be build, sleeping 6h."
+		for i in $(seq 1 12) ; do
+			sleep 30m
+			echo "$(date -u ) - still sleeping..."
+		done
+		echo "$(date -u ) - exiting cleanly now."
+		exit 0
+	fi
+}
+
+first_build() {
+	echo "============================================================================="
+	echo "Building for $RELEASE ($ARCH) on $(hostname -f) now."
+	echo "Source package: ${SRCPACKAGE}"
+	echo "Date:           $(date -u)"
+	echo "============================================================================="
+	set -x
+	download_package
+	local BUILDDIR="/tmp/$SRCPACKAGE-$(basename $TMPDIR)"
+	local LOG=$TMPDIR/b1/$SRCPACKAGE/build1.log
+	# nicely run mock with a timeout of 4h
+	timeout -k 4.1h 4h /usr/bin/ionice -c 3 /usr/bin/nice \
+		mock -r $RELEASE-$ARCH --resultdir=. $SRC_RPM 2>&1 | tee -a $LOG
+	PRESULT=${PIPESTATUS[0]}
+	if [ $PRESULT -eq 124 ] ; then
+		echo "$(date -u) - mock was killed by timeout after 4h." | tee -a $LOG
+	fi
+	if ! "$DEBUG" ; then set +x ; fi
+}
+
+second_build() {
+	echo "============================================================================="
+	echo "Re-Building for $RELEASE ($ARCH) on $(hostname -f) now."
+	echo "Source package: ${SRCPACKAGE}"
+	echo "Date:           $(date -u)"
+	echo "============================================================================="
+	set -x
+	download_package
+	local BUILDDIR="/tmp/$SRCPACKAGE-$(basename $TMPDIR)"
+	local LOG=$TMPDIR/b2/$SRCPACKAGE/build2.log
+	# NEW_NUM_CPU=$(echo $NUM_CPU-1|bc)
+	# nicely run mock with a timeout of 4h
+	timeout -k 4.1h 4h /usr/bin/ionice -c 3 /usr/bin/nice \
+		mock -r $RELEASE-$ARCH --resultdir=. $SRC_RPM 2>&1 | tee -a $LOG
+	PRESULT=${PIPESTATUS[0]}
+	if [ $PRESULT -eq 124 ] ; then
+		echo "$(date -u) - mock was killed by timeout after 4h." | tee -a $LOG
+	fi
+	if ! "$DEBUG" ; then set +x ; fi
+}
+
+remote_build() {
+	local BUILDNR=$1
+	local NODE=$RPM_BUILD_NODE
+	local FQDN=$NODE.debian.net
+	local PORT=22
+	set +e
+	ssh -p $PORT $FQDN /bin/true
+	RESULT=$?
+	# abort job if host is down
+	if [ $RESULT -ne 0 ] ; then
+		SLEEPTIME=$(echo "$BUILDNR*$BUILDNR*5"|bc)
+		echo "$(date -u) - $NODE seems to be down, sleeping ${SLEEPTIME}min before aborting this job."
+		sleep ${SLEEPTIME}m
+		exec /srv/jenkins/bin/abort.sh
+	fi
+	ssh -p $PORT $FQDN /srv/jenkins/bin/reproducible_build_rpm.sh $BUILDNR $RELEASE $ARCH ${SRCPACKAGE} ${TMPDIR}
+	RESULT=$?
+	if [ $RESULT -ne 0 ] ; then
+		ssh -p $PORT $FQDN "rm -r $TMPDIR" || true
+		handle_remote_error "with exit code $RESULT from $NODE for build #$BUILDNR for ${SRCPACKAGE} from $RELEASE ($ARCH)"
+	fi
+	rsync -e "ssh -p $PORT" -r $FQDN:$TMPDIR/b$BUILDNR $TMPDIR/
+	RESULT=$?
+	if [ $RESULT -ne 0 ] ; then
+		echo "$(date -u ) - rsync from $NODE failed, sleeping 2m before re-trying..."
+		sleep 2m
+		rsync -e "ssh -p $PORT" -r $FQDN:$TMPDIR/b$BUILDNR $TMPDIR/
+		RESULT=$?
+		if [ $RESULT -ne 0 ] ; then
+			handle_remote_error "when rsyncing remote build #$BUILDNR results from $NODE"
+		fi
+	fi
+	ls -R $TMPDIR
+	ssh -p $PORT $FQDN "rm -r $TMPDIR"
+	set -e
+}
+
+#
+# below is what controls the world
+#
+
+TMPDIR=$(mktemp --tmpdir=/srv/reproducible-results -d)  # where everything actually happens
+trap cleanup_all INT TERM EXIT
+cd $TMPDIR
+
+DATE=$(date -u +'%Y-%m-%d %H:%M')
+START=$(date +'%s')
+BUILDER="${JOB_NAME#reproducible_builder_}/${BUILD_ID}"
+DUMMY=$(mktemp -t rpm-dummy-XXXXXXXX)
+
+#
+# determine mode
+#
+if [ "$1" = "1" ] || [ "$1" = "2" ] ; then
+	MODE="$1"
+	RELEASE="$2"
+	ARCH="$3"
+	SRCPACKAGE="$4"
+	TMPDIR="$5"
+	[ -d $TMPDIR ] || mkdir -p $TMPDIR
+	cd $TMPDIR
+	mkdir -p b$MODE/$SRCPACKAGE
+	if [ "$MODE" = "1" ] ; then
+		first_build
+	else
+		second_build
+	fi
+	# preserve results and delete build directory
+	mv -v /tmp/$SRCPACKAGE-$(basename $TMPDIR)/$SRCPACKAGE/*.pkg.tar.xz $TMPDIR/b$MODE/$SRCPACKAGE/ || ls /tmp/$SRCPACKAGE-$(basename $TMPDIR)/$SRCPACKAGE/
+	rm -r /tmp/$SRCPACKAGE-$(basename $TMPDIR)/
+	echo "$(date -u) - build #$MODE for $SRCPACKAGE on $HOSTNAME done."
+	exit 0
+fi
+MODE="master"
+
+#
+# main - only used in master-mode
+#
+delay_start # randomize start times
+# first, we need to choose a packagey…
+RELEASE="$1"
+ARCH="$2"
+SRCPACKAGE=""	# package name
+SRC_RPM=""	# src rpm file name
+choose_package
+# build package twice
+mkdir b1 b2
+remote_build 1
+# only do the 2nd build if the 1st produced results
+if [ ! -z "$(ls $TMPDIR/b1/$SRCPACKAGE/*.rpm 2>/dev/null|| true)" ] ; then
+	remote_build 2
+	# run diffoscope on the results
+	TIMEOUT="30m"
+	DIFFOSCOPE="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --version 2>&1)"
+	echo "$(date -u) - Running $DIFFOSCOPE now..."
+	cd $TMPDIR/b1/$SRCPACKAGE
+	for ARTIFACT in *.rpm ; do
+		[ -f $ARTIFACT ] || continue
+		call_diffoscope $SRCPACKAGE $ARTIFACT
+		# publish page
+		if [ -f $TMPDIR/$SRCPACKAGE/$ARTIFACT.html ] ; then
+			cp $TMPDIR/$SRCPACKAGE/$ARTIFACT.html $BASE/rpm/$RELEASE/$ARCH/$SRCPACKAGE/
+		fi
+	done
+fi
+# publish logs
+cd $TMPDIR/b1/$SRCPACKAGE
+cp build1.log $BASE/rpm/$RELEASE/$ARCH/$SRCPACKAGE/
+[ ! -f $TMPDIR/b2/$SRCPACKAGE/build2.log ] || cp $TMPDIR/b2/$SRCPACKAGE/build2.log $BASE/rpm/$RELEASE/$ARCH/$SRCPACKAGE/
+echo "$(date -u) - $REPRODUCIBLE_URL/rpm/$RELEASE/$ARCH/$SRCPACKAGE/ updated."
+
+cd
+cleanup_all
+trap - INT TERM EXIT
+
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index a6e4a154..43349a3b 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -56,6 +56,9 @@ USERTAGS="toolchain infrastructure timestamps fileordering buildpath username ho
 ARCHLINUX_BUILD_NODE=profitbricks-build3-amd64
 ARCHLINUX_REPOS="core extra"
 
+# common settings for testing rpm based distros
+RPM_BUILD_NODE=profitbricks-build3-amd64
+
 # number of cores to be used
 NUM_CPU=$(grep -c '^processor' /proc/cpuinfo)
 
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index e7ed8b0b..b59602da 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -774,6 +774,10 @@
     name: '{name}_builder_archlinux_2'
 
 - job-template:
+    defaults: reproducible_builder_trigger
+    name: '{name}_builder_fedora23_x86_64_1'
+
+- job-template:
     defaults: reproducible
     name: '{name}_html_archlinux'
 
@@ -1555,6 +1559,13 @@
             my_shell: '/srv/jenkins/bin/reproducible_build_archlinux_pkg.sh'
             my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
             my_node: ''
+        - '{name}_builder_fedora23_x86_64_1':
+            my_description: 'Try to reproducibly build a Fedora 23 package.'
+            my_timed: '* * * * *'
+            my_trigger: ''
+            my_shell: '/srv/jenkins/bin/reproducible_build_rpm.sh fedora23 x86_64'
+            my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org'
+            my_node: ''
         - '{name}_html_archlinux':
             my_description: 'Build a simple webpage for Arch Linux reproducibility tests'
             my_timed: ''