From 3845fa8418060dce2370bfbdd7e10fcca7c02b28 Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Mon, 3 Aug 2015 02:00:33 +0200 Subject: reproducible: split repo comparison into two tables: used and obsolete --- TODO | 4 +- bin/reproducible_html_repository_comparison.sh | 65 +++++++++++++++----------- 2 files changed, 41 insertions(+), 28 deletions(-) diff --git a/TODO b/TODO index c986c637..f457372a 100644 --- a/TODO +++ b/TODO @@ -179,6 +179,7 @@ properties: ** fix https://jenkins.debian.net/munin/debian.net/jenkins.debian.net/jenkins_builds.html which is broken since jessie upgrade ** exclude depwait from no_notes ** include depwait in graphs +** repo-comparison: check for binaries without source * lesser prio ** more graphs: @@ -186,7 +187,6 @@ properties: *** graph packages in testing+unstable which need to be fixed ** reproducible_create_meta_pkg_sets uses schroot created by dpkg_setup_schroot_jessie job (outside of reproducible job space...) ** "fork" etc/schroot/default into etc/schroot/reproducible -** repo-comparison: check for binaries without source ** move "untested" field in stats table too? (as in csv output...) ** new page: packages which are orphaned but have a reproducible usertagged patch ** a reproducible_log_grep_by_sql.(py|sh) would be nice, to only grep in packages with a certain status (build in the last X days) @@ -310,7 +310,7 @@ properties: * use http://10.0.0.15:8000/ as proxy * next: run pbuilder create jobs via remote job scheduling as described below * then: repo overview with binary packages -* then: armhf scheduling (only for testing for now) +* then: armhf scheduling (only for testing for now, introduce "arch-factor", eg. amd64=10 and armhf=3, to schedule less on some archs) * then: armhf building - run the job on jenkins.d.n and build1 on one host and build2 on another and then run debbindiff on jenkins.d.n… * then: run maintenance job via remote job scheduling * monitor their temperatures via munin, probably using munin-async via ssh… diff --git a/bin/reproducible_html_repository_comparison.sh b/bin/reproducible_html_repository_comparison.sh index ab4abd21..3b8571cc 100755 --- a/bin/reproducible_html_repository_comparison.sh +++ b/bin/reproducible_html_repository_comparison.sh @@ -17,11 +17,17 @@ PAGE=index_${VIEW}.html SOURCES=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) PACKAGES=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) TMPFILE=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) +TABLE_TODO=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) +TABLE_DONE=$(mktemp --tmpdir=$TEMPDIR repo-comp-XXXXXXXXX) MODIFIED_IN_SID=0 MODIFIED_IN_EXP=0 BINNMUS_NEEDED=0 +write_row() { + echo "$1" >> $ROW +} + echo "$(date) - starting to write $PAGE page." write_page_header $VIEW "Comparison between the reproducible builds apt repository and regular Debian suites" write_page "

These source packages are different from unstable in our apt repository on alioth. They are available for testing using these sources.lists entries:

"
@@ -127,8 +133,10 @@ for PKG in $SOURCEPKGS ; do
 			CBINARIES="$CBINARIES
$i"
 		done
 		BET="$BET"
+		ROW=$TABLE_TODO
 	else
 		BET=" "
+		ROW=$TABLE_DONE
 	fi
 	if [ ! -z "$CRUFT" ] ; then
 		CRUFT="$(echo $CRUFT|sed 's# #
#g')"
@@ -136,8 +144,8 @@ for PKG in $SOURCEPKGS ; do
 	#
 	# write output
 	#
-	write_page "
src:$PKG
"
-	write_page " "
+	write_row "
src:$PKG
"
+	write_row " "
 	case $PKG in
 		strip-nondeterminism|debbindiff)
 			URL="http://anonscm.debian.org/cgit/reproducible/$PKG.git" ;;
@@ -146,41 +154,41 @@ for PKG in $SOURCEPKGS ; do
 	esac
 	curl $URL > $TMPFILE
 	if [ "$(grep "'error'>No repositories found" $TMPFILE 2>/dev/null)" ] ; then
-		write_page "$URL
(no git repository found)"
+		write_row "$URL
(no git repository found)"
 	elif [ "$(grep "'error'>Invalid branch" $TMPFILE 2>/dev/null)" ] ; then
 		URL="http://anonscm.debian.org/cgit/reproducible/$PKG.git/?h=merged/reproducible_builds"
 		curl $URL > $TMPFILE
 		if [ "$(grep "'error'>Invalid branch" $TMPFILE 2>/dev/null)" ] ; then
 			if ! $OBSOLETE_IN_SID ; then
-				write_page "$PKG.git
non-standard branch"
+				write_row "$PKG.git
non-standard branch"
 			else
-				write_page "$PKG.git
non-standard branch (but that is ok, our package aint't used in unstable)"
+				write_row "$PKG.git
non-standard branch (but that is ok, our package aint't used in unstable)"
 			fi
 		else
-			write_page "$PKG.git"
-			write_page "
(merged"
+			write_row "$PKG.git"
+			write_row "
(merged"
 			if $OBSOLETE_IN_TESTING ; then
-				write_page "and available in testing and unstable)"
+				write_row "and available in testing and unstable)"
 			elif $OBSOLETE_IN_SID ; then
-				write_page "and available in unstable)"
+				write_row "and available in unstable)"
 			elif $OBSOLETE_IN_EXP ; then
-				write_page "and available in experimental)"
+				write_row "and available in experimental)"
 			fi
 		fi
 	else
-		write_page "$PKG.git"
+		write_row "$PKG.git"
 		if [ "$PKG" != "strip-nondeterminism" ] && [ "$PKG" != "debbindiff" ] ; then
 			if $OBSOLETE_IN_TESTING && $OBSOLETE_IN_SID && $OBSOLETE_IN_EXP ; then
-				write_page "
(unused?"
-				write_page "
Then the branch should probably renamed."
+				write_row "
(unused?"
+				write_row "
Then the branch should probably renamed."
 			elif $OBSOLETE_IN_SID && $OBSOLETE_IN_EXP ; then
-				write_page "
(only used in testing, fixed in sid,"
-				write_page "
branch should probably renamed)"
+				write_row "
(only used in testing, fixed in sid,"
+				write_row "
branch should probably renamed)"
 			elif $OBSOLETE_IN_EXP ; then
-				write_page "
(only used in testing and unstable, fixed in experimental)"
+				write_row "
(only used in testing and unstable, fixed in experimental)"
 			fi
 		elif ( [ "$PKG" = "strip-nondeterminism" ] || [ "$PKG" = "debbindiff" ] ) && $OBSOLETE_IN_SID ; then
-			write_page "
(this repo is always used)"
+			write_row "
(this repo is always used)"
 		fi
 	fi
 	if ! $OBSOLETE_IN_SID ; then
@@ -189,20 +197,24 @@ for PKG in $SOURCEPKGS ; do
 	if ! $OBSOLETE_IN_EXP ; then
 		let "MODIFIED_IN_EXP+=1"
 	fi
-	write_page " "
-	write_page " PTS"
+	write_row " "
+	write_row " PTS"
 	URL="https://bugs.debian.org/cgi-bin/pkgreport.cgi?src=$PKG&users=reproducible-builds@lists.alioth.debian.org&archive=both"
 	for TAG in $USERTAGS ; do
 		URL="$URL&tag=$TAG"
 	 done
-	write_page " bugs"
-	write_page " $CRUFT"
-	write_page " $BET $CBINARIES"
-	write_page " $CTEST"
-	write_page " $CSID"
-	write_page " $CEXP"
-	write_page ""
+	write_row " bugs"
+	write_row " $CRUFT"
+	write_row " $BET $CBINARIES"
+	write_row " $CTEST"
+	write_row " $CSID"
+	write_row " $CEXP"
+	write_row ""
 done
+cat $TABLE_TODO >> $PAGE
+write_page "
"
+write_page "
"
+cat $TABLE_DONE >> $PAGE
 write_page "
package (obsolete in our repo)git repoPTS linkusertagged bugold versions in our repo
(needed for reproducing old builds)		version in our repo
(available binary packages per architecture)		version in 'testing'version in 'unstable'version in 'experimental'
"
 write_page_footer
 publish_page
@@ -212,4 +224,5 @@ echo "$BINNMUS_NEEDED" > /srv/reproducible-results/binnmus_needed.txt
 
 # cleanup
 rm $SOURCES $PACKAGES $TMPFILE
+rm $TABLE_TODO $TABLE_DONE
 
-- 
cgit v1.2.3-70-g09d2