reproducible: run debbindiff from within a(nother) sid schroot (created by another job)

3 files changed, 39 insertions, 44 deletions

diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index 0876de9f..ac78ea52 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -26,13 +26,14 @@ unschedule_from_db() {
 	# unmark build as properly finished
 	sqlite3 -init $INIT ${PACKAGES_DB} "DELETE FROM sources_scheduled WHERE name = '$SRCPACKAGE';"
 	# update html page for package
-	set -x
+	set +x
 	process_packages $SRCPACKAGE
-	echo "Successfully updated the database and updated the html file for the package."
-	echo "Enjoy $JENKINS_URL/userContent/rb-pkg/$SRCPACKAGE.html"
+	echo
+	echo "Successfully updated the database and updated $JENKINS_URL/userContent/rb-pkg/$SRCPACKAGE.html"
+	echo
 }
 
-TMPDIR=$(mktemp --tmpdir=$PWD -d)
+TMPDIR=$(mktemp --tmpdir=/srv/reproducible-results -d)
 trap cleanup_all INT TERM EXIT
 cd $TMPDIR
 RESULT=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT name,date_scheduled FROM sources_scheduled WHERE date_build_started = '' ORDER BY date_scheduled LIMIT 1")
@@ -99,8 +100,8 @@ else
 				break
 			fi
 		done
-		set -x
 		if ! $SUITABLE ; then
+			set -x
 			sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"not for us\", \"$DATE\")"
 			set +x
 			echo "Package ${SRCPACKAGE} (${VERSION}) shall only be build on \"$(echo "${ARCHITECTURES}" | xargs echo )\" and thus was skipped." | tee -a ${RBUILDLOG}
@@ -108,8 +109,10 @@ else
 			exit 0
 		fi
 		set +e
+		set -x
 		NUM_CPU=$(cat /proc/cpuinfo |grep ^processor|wc -l)
-		( timeout 15m nice ionice -c 3 sudo DEB_BUILD_OPTIONS="parallel=$NUM_CPU" pbuilder --build --debbuildopts "-b" --basetgz /var/cache/pbuilder/base-reproducible.tgz --distribution sid ${SRCPACKAGE}_*.dsc ) 2>&1 | tee -a ${RBUILDLOG}
+		( timeout 12h nice ionice -c 3 sudo DEB_BUILD_OPTIONS="parallel=$NUM_CPU" pbuilder --build --debbuildopts "-b" --basetgz /var/cache/pbuilder/base-reproducible.tgz --distribution sid ${SRCPACKAGE}_*.dsc ) 2>&1 | tee -a ${RBUILDLOG}
+		set +x
 		if [ -f /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}_amd64.changes ] ; then
 			mkdir b1 b2
 			dcmd cp /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}_amd64.changes b1
@@ -117,12 +120,12 @@ else
 			# so first delete files from .dsc, then from .changes file
 			sudo dcmd rm /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}.dsc
 			sudo dcmd rm /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}_amd64.changes
-			set +x
 			echo "============================================================================="
 			echo "Re-building ${SRCPACKAGE} now."
 			echo "============================================================================="
 			set -x
 			timeout 12h nice ionice -c 3 sudo DEB_BUILD_OPTIONS="parallel=$NUM_CPU" pbuilder --build --debbuildopts "-b" --basetgz /var/cache/pbuilder/base-reproducible.tgz --distribution sid ${SRCPACKAGE}_${EVERSION}.dsc
+			set +x
 			dcmd cp /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}_amd64.changes b2
 			# and again (see comment 5 lines above)
 			sudo dcmd rm /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}.dsc
@@ -131,25 +134,30 @@ else
 			LOGFILE=$(ls ${SRCPACKAGE}_${EVERSION}.dsc)
 			LOGFILE=$(echo ${LOGFILE%.dsc}.debbindiff.html)
 			BUILDINFO=${SRCPACKAGE}_${EVERSION}_amd64.buildinfo
-			( timeout 15m /var/lib/jenkins/debbindiff.git/debbindiff.py --html ./${LOGFILE} b1/${SRCPACKAGE}_${EVERSION}_amd64.changes b2/${SRCPACKAGE}_${EVERSION}_amd64.changes ) 2>&1 >> ${RBUILDLOG}
+			( timeout 15m schroot --directory /tmp -c source:jenkins-reproducible-sid debbindiff -- --html $TMPDIR/${LOGFILE} $TMPDIR/b1/${SRCPACKAGE}_${EVERSION}_amd64.changes $TMPDIR/b2/${SRCPACKAGE}_${EVERSION}_amd64.changes ) 2>&1 >> ${RBUILDLOG}
 			RESULT=$?
+			set +x
 			set -e
 			echo | tee -a ${RBUILDLOG}
 			if [ $RESULT -eq 124 ] ; then
-				echo "$(date) - debbindiff.py was killed after running into timeouot..." | tee -a ${RBUILDLOG}
+				echo "$(date) - debbindiff was killed after running into timeouot... maybe there is still $JENKINS_URL/userContent/dbd/${LOGFILE}" | tee -a ${RBUILDLOG}
 			elif [ $RESULT -eq 1 ] ; then
-				echo "$(date) - debbindiff.py found issues, please investigate $JENKINS_URL/userContent/dbd/${LOGFILE}" | tee -a ${RBUILDLOG}
+				DEBBINDIFFOUT="debbindiff found issues, please investigate $JENKINS_URL/userContent/dbd/${LOGFILE}"
 			fi
 			if [ ! -f ./${LOGFILE} ] && [ -f b1/${BUILDINFO} ] ; then
-				cp b1/${BUILDINFO} /var/lib/jenkins/userContent/buildinfo/
+				cp b1/${BUILDINFO} /var/lib/jenkins/userContent/buildinfo/ 2>&1 > /dev/null
 				figlet ${SRCPACKAGE}
-				echo "debbindiff.py found no differences in the changes files, and a .buildinfo file also exist." | tee -a ${RBUILDLOG}
+				echo
+				echo "debbindiff found no differences in the changes files, and a .buildinfo file also exist." | tee -a ${RBUILDLOG}
 				echo "${SRCPACKAGE} built successfully and reproducibly." | tee -a ${RBUILDLOG}
 				sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"reproducible\",  \"$DATE\")"
 				unschedule_from_db
 			else
-				cp b1/${BUILDINFO} /var/lib/jenkins/userContent/buildinfo/ || true
+				echo | tee -a ${RBUILDLOG}
+				echo -n "$(date) - ${SRCPACKAGE} failed to build reproducibly " | tee -a ${RBUILDLOG}
+				cp b1/${BUILDINFO} /var/lib/jenkins/userContent/buildinfo/ 2>&1 > /dev/null || true
 				if [ -f ./${LOGFILE} ] ; then
+					echo -n "$DEBBINDIFFOUT" | tee -a ${RBUILDLOG}
 					# FIXME: work around debbindiff not having external CSS support (#764470)
 					# should really be fixed in debbindiff and just moved....
 					if grep -q "Generated by debbindiff 3" ./${LOGFILE} ; then
@@ -157,27 +165,26 @@ else
 					else
 						mv ./${LOGFILE} /var/lib/jenkins/userContent/dbd/
 					fi
+				else
+					echo -n ", debbindiff produced no output (which is strange)"
 				fi
-				sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"unreproducible\", \"$DATE\")"
-				unschedule_from_db
-				set +x
-				echo -n "${SRCPACKAGE} failed to build reproducibly" | tee -a ${RBUILDLOG}
 				if [ ! -f b1/${BUILDINFO} ] ; then
-					echo "and a .buildinfo file is missing too." | tee -a ${RBUILDLOG}
+					echo " and a .buildinfo file is missing." | tee -a ${RBUILDLOG}
 				else
 					echo "." | tee -a ${RBUILDLOG}
 				fi
+				sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"unreproducible\", \"$DATE\")"
+				unschedule_from_db
 			fi
 		else
-			sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"FTBFS\", \"$DATE\")"
-			unschedule_from_db
 			set +x
 			echo "${SRCPACKAGE} failed to build from source."
+			sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"FTBFS\", \"$DATE\")"
+			unschedule_from_db
 		fi
 	fi
 
 fi
-set -x
 cd ..
 cleanup_all
 trap - INT TERM EXIT
diff --git a/bin/reproducible_setup.sh b/bin/reproducible_setup.sh
index c89be793..f5cab3d8 100755
--- a/bin/reproducible_setup.sh
+++ b/bin/reproducible_setup.sh
@@ -23,23 +23,10 @@ for PKG in linux cups zurl openclipart eigen3 xmds2 ; do
 	fi
 done
 
-# FIXME: needed as long as there is no backport for debbindiff
-# 	 or as long as we dont run it within pbuilder...
-# fetch git repo for the debbindiff command used in reproducible_build.sh
-WORKSPACE=$PWD
-cd /var/lib/jenkins
-if [ -d debbindiff.git ] ; then
-	cd debbindiff.git
-	git pull
-else
-	git clone git://git.debian.org/git/reproducible/debbindiff.git debbindiff.git
-fi
-cd $WORKSPACE
-
+#
+# create script to configure a pbuilder chroot
+#
 create_setup_tmpfile() {
-	#
-	# script to configure a pbuilder chroot
-	#
 	cat > ${TMPFILE} <<- EOF
 #
 # this script is run within the pbuilder environment to further customize it
@@ -84,10 +71,10 @@ echo
 EOF
 }
 
-setup_pbuilder() {
 #
 # setup pbuilder for reproducible builds
 #
+setup_pbuilder() {
 	echo "$(date) - creating /var/cache/pbuilder/${1}.tgz now..."
 	TMPFILE=$(mktemp)
 	create_setup_tmpfile
diff --git a/bin/schroot-create.sh b/bin/schroot-create.sh
index f59010b1..5f31e3cd 100755
--- a/bin/schroot-create.sh
+++ b/bin/schroot-create.sh
@@ -53,6 +53,11 @@ bootstrap() {
 	echo "deb-src $MIRROR $DISTRO main"        | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list > /dev/null
 	echo "${BACKPORTS}"                        | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list >/dev/null
 	echo "${BACKPORTSSRC}"                     | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list >/dev/null
+
+	sudo chroot $CHROOT_TARGET apt-get update
+	if [ -n "$1" ] ; then
+		sudo chroot $CHROOT_TARGET apt-get install -y --no-install-recommends "$@"
+	fi
 }
 
 cleanup() {
@@ -61,7 +66,7 @@ cleanup() {
 	fi
 }
 trap cleanup INT TERM EXIT
-bootstrap
+bootstrap $@
 
 trap - INT TERM EXIT
 
@@ -91,8 +96,4 @@ sudo tee /etc/schroot/chroot.d/jenkins-"$TARGET" <<-__END__
 	union-type=aufs
 	__END__
 
-schroot --directory /root -c "source:jenkins-$TARGET" -u root -- apt-get update
-if [ -n "$1" ]
-then
-	schroot --directory /root -c "source:jenkins-$TARGET" -u root -- apt-get install -y --no-install-recommends "$@"
-fi
+