reproducible archlinux: refactor to support multiple repositories

2 files changed, 57 insertions, 52 deletions

diff --git a/bin/reproducible_build_archlinux_pkg.sh b/bin/reproducible_build_archlinux_pkg.sh
index 64c49f4f..17cddcd9 100755
--- a/bin/reproducible_build_archlinux_pkg.sh
+++ b/bin/reproducible_build_archlinux_pkg.sh
@@ -44,23 +44,24 @@ choose_package() {
 	# every 2 days we check for new archlinux packages
 	touch -d "$(date -d '2 days ago' '+%Y-%m-%d') 00:00 UTC" $DUMMY
 	if [ ! -f $ARCHLINUX_PKGS ] || [ $DUMMY -nt $ARCHLINUX_PKGS ] ; then
-		echo "$(date -u ) - updating list of available packages."
+		REPOSITORY="core"
+		echo "$(date -u ) - updating list of available packages in repository '$REPOSITORY'."
 		local SESSION="archlinux-scheduler-$RANDOM"
 		schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
-		schroot --run-session -c $SESSION --directory /var/abs/core -- ls -1|sort -R|xargs echo > $ARCHLINUX_PKGS
+		schroot --run-session -c $SESSION --directory /var/abs/$REPOSITORY -- ls -1|sort -R|xargs echo > $ARCHLINUX_PKGS
 		schroot --end-session -c $SESSION
 	fi
 	rm $DUMMY > /dev/null
-	echo "$(date -u ) - these packages are known to us:"
+	echo "$(date -u ) - these packages in repository '$REPOSITORY' are known to us:"
 	cat $ARCHLINUX_PKGS
 	for PKG in $(cat $ARCHLINUX_PKGS) ; do
 		# build package if it has never build or at least a week ago
-		if [ ! -d $BASE/archlinux/$PKG ] || [ ! -z $(find $BASE/archlinux/ -name $PKG -mtime +6) ] ; then
+		if [ ! -d $BASE/archlinux/$REPOSITORY/$PKG ] || [ ! -z $(find $BASE/archlinux/$REPOSITORY/ -name $PKG -mtime +6) ] ; then
 			SRCPACKAGE=$PKG
-			echo "$(date -u ) - building package $PKG now..."
+			echo "$(date -u ) - building package $PKG from '$REPOSITORY' now..."
 			# very simple locking…
-			mkdir -p $BASE/archlinux/$PKG
-			touch $BASE/archlinux/$PKG
+			mkdir -p $BASE/archlinux/$REPOSITORY/$PKG
+			touch $BASE/archlinux/$REPOSITORY/$PKG
 			break
 		fi
 	done
@@ -77,7 +78,7 @@ choose_package() {
 
 first_build() {
 	echo "============================================================================="
-	echo "Building ${SRCPACKAGE} for Arch Linux on $(hostname -f) now."
+	echo "Building ${SRCPACKAGE} from repository '$REPOSITORY' for Arch Linux on $(hostname -f) now."
 	echo "Date:     $(date)"
 	echo "Date UTC: $(date -u)"
 	echo "============================================================================="
@@ -88,7 +89,7 @@ first_build() {
 	schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
 	echo "MAKEFLAGS=-j$NUM_CPU" | schroot --run-session -c $SESSION --directory /tmp -u root -- tee -a /etc/makepkg.conf
 	schroot --run-session -c $SESSION --directory /tmp -- mkdir $BUILDDIR
-	schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/core/$SRCPACKAGE $BUILDDIR/
+	schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/$REPOSITORY/$SRCPACKAGE $BUILDDIR/
 	# just set timezone in the 1st build
 	echo 'export TZ="/usr/share/zoneinfo/Etc/GMT+12"' | schroot --run-session -c $SESSION --directory /tmp -- tee -a /var/lib/jenkins/.bashrc
 	# nicely run makepkg with a timeout of 4h
@@ -104,7 +105,7 @@ first_build() {
 
 second_build() {
 	echo "============================================================================="
-	echo "Re-Building ${SRCPACKAGE} for Arch Linux on $(hostname -f) now."
+	echo "Re-Building ${SRCPACKAGE} from repository '$REPOSITORY' for Arch Linux on $(hostname -f) now."
 	echo "Date:     $(date)"
 	echo "Date UTC: $(date -u)"
 	echo "============================================================================="
@@ -116,7 +117,7 @@ second_build() {
 	schroot --begin-session --session-name=$SESSION -c jenkins-reproducible-archlinux
 	echo "MAKEFLAGS=-j$NEW_NUM_CPU" | schroot --run-session -c $SESSION --directory /tmp -u root -- tee -a /etc/makepkg.conf
 	schroot --run-session -c $SESSION --directory /tmp -- mkdir $BUILDDIR
-	schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/core/$SRCPACKAGE $BUILDDIR/
+	schroot --run-session -c $SESSION --directory /tmp -- cp -r /var/abs/$REPOSITORY/$SRCPACKAGE $BUILDDIR/
 	# add more variations in the 2nd build: TZ, LANG, LC_ALL, umask
 	schroot --run-session -c $SESSION --directory /tmp -- tee -a /var/lib/jenkins/.bashrc <<-__END__
 	export TZ="/usr/share/zoneinfo/Etc/GMT-14"
@@ -213,13 +214,14 @@ fi
 # main - only used in master-mode
 #
 delay_start # randomize start times
-# first, we need to choose a package…
+# first, we need to choose a package from a repository…
+REPOSITORY=""
 SRCPACKAGE=""
 choose_package
 # build package twice
 mkdir b1 b2
 remote_build 1
-# only do the 2nd build if the 1st produced some results
+# only do the 2nd build if the 1st produced results
 if [ ! -z "$(ls $TMPDIR/b1/$SRCPACKAGE/*.pkg.tar.xz 2>/dev/null|| true)" ] ; then
 	remote_build 2
 	# run diffoscope on the results
@@ -232,15 +234,15 @@ if [ ! -z "$(ls $TMPDIR/b1/$SRCPACKAGE/*.pkg.tar.xz 2>/dev/null|| true)" ] ; the
 		call_diffoscope $SRCPACKAGE $ARTIFACT
 		# publish page
 		if [ -f $TMPDIR/$SRCPACKAGE/$ARTIFACT.html ] ; then
-			cp $TMPDIR/$SRCPACKAGE/$ARTIFACT.html $BASE/archlinux/$SRCPACKAGE/
+			cp $TMPDIR/$SRCPACKAGE/$ARTIFACT.html $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/
 		fi
 	done
 fi
 # publish logs
 cd $TMPDIR/b1/$SRCPACKAGE
-cp build1.log $BASE/archlinux/$SRCPACKAGE/
-[ ! -f $TMPDIR/b2/$SRCPACKAGE/build2.log ] || cp $TMPDIR/b2/$SRCPACKAGE/build2.log $BASE/archlinux/$SRCPACKAGE/
-echo "$(date -u) - $REPRODUCIBLE_URL/archlinux/$SRCPACKAGE/ updated."
+cp build1.log $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/
+[ ! -f $TMPDIR/b2/$SRCPACKAGE/build2.log ] || cp $TMPDIR/b2/$SRCPACKAGE/build2.log $BASE/archlinux/$REPOSITORY/$SRCPACKAGE/
+echo "$(date -u) - $REPRODUCIBLE_URL/archlinux/$REPOSITORY/$SRCPACKAGE/ updated."
 
 cd
 cleanup_all
diff --git a/bin/reproducible_html_archlinux.sh b/bin/reproducible_html_archlinux.sh
index aa948f58..0e92e0e9 100755
--- a/bin/reproducible_html_archlinux.sh
+++ b/bin/reproducible_html_archlinux.sh
@@ -37,45 +37,48 @@ cat > $PAGE <<- EOF
 EOF
 write_page_intro 'Arch Linux'
 write_explaination_table 'Arch Linux'
-write_page "    <table><tr><th>source package</th><th>test result</th><th>test date</th><th>1st build log</th><th>2nd build log</th></tr>"
-for PKG in $(find $ARCHBASE/* -maxdepth 1 -type d -exec basename {} \;) ; do
-	write_page "     <tr>"
-	write_page "      <td>$PKG</td>"
-	write_page "      <td>"
-	if [ -z "$(cd $ARCHBASE/$PKG/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
-		if [ ! -z "$(grep '==> ERROR: Could not resolve all dependencies' $ARCHBASE/$PKG/build1.log)" ] ; then
-			write_page "       <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> could not resolve dependencies"
-		elif [ ! -z "$(egrep '==> ERROR: .pacman. failed to install missing dependencies.' $ARCHBASE/$PKG/build1.log)" ] ; then
-			write_page "       <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> failed to install dependencies"
-		elif [ ! -z "$(egrep '==> ERROR: A failure occurred in (build|package)' $ARCHBASE/$PKG/build1.log)" ] ; then
-			write_page "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build from source"
-		elif [ ! -z "$(egrep '==> ERROR: A failure occurred in check' $ARCHBASE/$PKG/build1.log)" ] ; then
-			write_page "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build from source, while running tests"
-		elif [ ! -z "$(egrep '==> ERROR: Failure while downloading' $ARCHBASE/$PKG/build1.log)" ] ; then
-			write_page "       <img src=\"/userContent/static/weather-severe-alert.png\" alt=\"404 icon\" /> failed to download source"
-		elif [ ! -z "$(egrep '==> ERROR: One or more files did not pass the validity check' $ARCHBASE/$PKG/build1.log)" ] ; then
-			write_page "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to verify source"
-		elif [ ! -z "$(egrep 'makepkg was killed by timeout after 4h' $ARCHBASE/$PKG/build1.log)" ] ; then
-			write_page "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build, killed by timeout after 4h"
+write_page "    <table><tr><th>repository</th><th>source package</th><th>test result</th><th>test date</th><th>1st build log</th><th>2nd build log</th></tr>"
+for REPOSITORY in "core" ; do
+	for PKG in $(find $ARCHBASE/$REPOSITORY/* -maxdepth 1 -type d -exec basename {} \;) ; do
+		write_page "     <tr>"
+		write_page "      <td>$REPOSITORY</td>"
+		write_page "      <td>$PKG</td>"
+		write_page "      <td>"
+		if [ -z "$(cd $ARCHBASE/$REPOSITORY/$PKG/ ; ls *.pkg.tar.xz.html 2>/dev/null)" ] ; then
+			if [ ! -z "$(grep '==> ERROR: Could not resolve all dependencies' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+				write_page "       <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> could not resolve dependencies"
+			elif [ ! -z "$(egrep '==> ERROR: .pacman. failed to install missing dependencies.' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+				write_page "       <img src=\"/userContent/static/weather-snow.png\" alt=\"depwait icon\" /> failed to install dependencies"
+			elif [ ! -z "$(egrep '==> ERROR: A failure occurred in (build|package)' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+				write_page "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build from source"
+			elif [ ! -z "$(egrep '==> ERROR: A failure occurred in check' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+				write_page "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build from source, while running tests"
+			elif [ ! -z "$(egrep '==> ERROR: Failure while downloading' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+				write_page "       <img src=\"/userContent/static/weather-severe-alert.png\" alt=\"404 icon\" /> failed to download source"
+			elif [ ! -z "$(egrep '==> ERROR: One or more files did not pass the validity check' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+				write_page "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to verify source"
+			elif [ ! -z "$(egrep 'makepkg was killed by timeout after 4h' $ARCHBASE/$REPOSITORY/$PKG/build1.log)" ] ; then
+				write_page "       <img src=\"/userContent/static/weather-storm.png\" alt=\"ftbfs icon\" /> failed to build, killed by timeout after 4h"
+			else
+				write_page "       probably failed to build from source, please investigate"
+				# or is it reproducible???
+			fi
 		else
-			write_page "       probably failed to build from source, please investigate"
-			# or is it reproducible???
+			for ARTIFACT in $(cd $ARCHBASE/$REPOSITORY/$PKG/ ; ls *.pkg.tar.xz.html) ; do
+				write_page "       <img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> <a href=\"/archlinux/$REPOSITORY/$PKG/$ARTIFACT\">${ARTIFACT:0:-5}</a> is unreproducible<br />"
+			done
 		fi
-	else
-		for ARTIFACT in $(cd $ARCHBASE/$PKG/ ; ls *.pkg.tar.xz.html) ; do
-			write_page "       <img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> <a href=\"/archlinux/$PKG/$ARTIFACT\">${ARTIFACT:0:-5}</a> is unreproducible<br />"
+		write_page "      </td>"
+		write_page "      <td>$(LANG=C TZ=UTC ls --full-time $ARCHBASE/$REPOSITORY/$PKG/build1.log | cut -d ' ' -f6 )</td>"
+		for LOG in build1.log build2.log ; do
+			if [ -f $ARCHBASE/$REPOSITORY/$PKG/$LOG ] ; then
+				write_page "      <td><a href=\"/archlinux/$REPOSITORY/$PKG/$LOG\">$LOG</a></td>"
+			else
+				write_page "      <td>&nbsp;</td>"
+			fi
 		done
-	fi
-	write_page "      </td>"
-	write_page "      <td>$(LANG=C TZ=UTC ls --full-time $ARCHBASE/$PKG/build1.log | cut -d ' ' -f6 )</td>"
-	for LOG in build1.log build2.log ; do
-		if [ -f $ARCHBASE/$PKG/$LOG ] ; then
-			write_page "      <td><a href=\"/archlinux/$PKG/$LOG\">$LOG</a></td>"
-		else
-			write_page "      <td>&nbsp;</td>"
-		fi
+		write_page "     </tr>"
 	done
-	write_page "     </tr>"
 done
 write_page "    </table>"
 write_page "</div></div>"