reproducible fedora rpms: build a lot more packages (700 instead of 150) and choose them via some searchterms

3 files changed, 14 insertions, 6 deletions

diff --git a/bin/reproducible_build_rpm.sh b/bin/reproducible_build_rpm.sh
index 3b66fc89..936c2844 100755
--- a/bin/reproducible_build_rpm.sh
+++ b/bin/reproducible_build_rpm.sh
@@ -59,12 +59,19 @@ download_package() {
 }
 
 choose_package() {
-	echo "$(date -u ) - choosing package to be build."
 	local MIN_AGE=7
 	touch -d "$(date -u -d "$MIN_AGE days ago" '+%Y-%m-%d %H:%M') UTC" $DUMMY
-	# instead of hardcoding the list of packages we can also use something like this to get a list of all packages:
-	# yumdownloader --urls --source --releasever=23 '*'
-	for PKG in $(echo audit-libs basesystem bash binutils bzip2 bzip2-libs ca-certificates chkconfig coreutils cpio cpp cracklib cracklib-dicts crypto-policies curl cyrus-sasl-lib diffutils dwz elfutils elfutils-default-yama-scope elfutils-libelf elfutils-libs emacs-filesystem expat fedora-release fedora-repos file file-libs filesystem findutils gawk gc gcc gcc-c++ gdb gdbm ghc-srpm-macros glib2 glibc glibc-common glibc-devel glibc-headers gmp gnat-srpm-macros gnupg2 gnutls go-srpm-macros grep groff-base guile gzip info isl kernel-headers keyutils-libs krb5-libs libacl libarchive libassuan libatomic_ops libattr libbabeltrace libblkid libcap libcap-ng libcom_err libcurl libdb libdb-utils libfdisk libffi libgcc libgcrypt libgomp libgpg-error libidn libipt libksba libmetalink libmount libmpc libnghttp2 libpwquality libseccomp libsecret libselinux libsemanage libsepol libsmartcols libssh2 libstdc++ libstdc++-devel libtasn1 libtool-ltdl libunistring libuser libutempter libuuid libverto libxml2 lua lzo make mpfr ncurses ncurses-base ncurses-libs nettle npth nspr nss nss-softokn nss-softokn-freebl nss-sysinit nss-tools nss-util ocaml-srpm-macros openldap openssl-libs p11-kit p11-kit-trust pam patch pcre perl perl-Carp perl-constant perl-Encode perl-Exporter perl-File-Path perl-File-Temp perl-generators perl-Getopt-Long perl-HTTP-Tiny perl-libs perl-macros perl-MIME-Base64 perl-parent perl-PathTools perl-Pod-Escapes perl-podlators perl-Pod-Perldoc perl-Pod-Simple perl-Pod-Usage perl-Scalar-List-Utils perl-Socket perl-srpm-macros perl-Storable perl-Term-ANSIColor perl-Term-Cap perl-Text-ParseWords perl-Text-Tabs+Wrap perl-threads perl-threads-shared perl-Time-HiRes perl-Time-Local perl-Unicode-Normalize pinentry pkgconfig popt python3 python3-libs python3-pip python3-setuptools readline redhat-rpm-config rpm rpm-build rpm-build-libs rpm-libs rpm-plugin-selinux sed setup shadow-utils shared-mime-info sqlite systemd-libs tar tzdata unzip ustr util-linux which xz xz-libs zip zlib | sed -s "s# #\n#g" | sort -R | xargs echo) ; do
+	if [ ! -f ${RPM_PKGS}_$RELEASE ] || [ $DUMMY -nt ${RPM_PKGS}_$RELEASE ] ; then
+		echo "$(date -u ) - updating list of available packages for $RELEASE"
+		SEARCHTERMS="apache2 bash fedora gcc gnome gnu ipa kde linux mock openssl redhat rpm ssh system-config systemd xorg yum"
+		echo "$(date -u ) - instead of building everything only packages matching these searchterms are build: $SEARCHTERMS"
+		local i=""
+		( for i in $SEARCHTERMS ; do repoquery --qf "%{name}" "*$i*" ; done ) | sort -u > ${RPM_PKGS}_$RELEASE
+		cat ${RPM_PKGS}_$RELEASE
+	fi
+	echo "$(date -u ) - choosing package to be build."
+	local PKG=""
+	for PKG in $(cat ${RPM_PKGS}_$RELEASE) ; do
 		# build package if it has never build or at least $MIN_AGE days ago
 		if [ ! -d $BASE/rpms/$RELEASE/$ARCH/$PKG ] || [ $DUMMY -nt $BASE/rpms/$RELEASE/$ARCH/$PKG ] ; then
 			SRCPACKAGE=$PKG
diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh
index b17cf3b4..3aee08cc 100755
--- a/bin/reproducible_common.sh
+++ b/bin/reproducible_common.sh
@@ -59,6 +59,7 @@ ARCHLINUX_PKGS=/srv/reproducible-results/.archlinux_pkgs
 
 # common settings for testing rpm based distros
 RPM_BUILD_NODE=profitbricks-build3-amd64
+RPM_PKGS=/srv/reproducible-results/.rpm_pkgs
 
 # number of cores to be used
 NUM_CPU=$(grep -c '^processor' /proc/cpuinfo)
@@ -316,7 +317,7 @@ write_page_intro() {
 	elif [ "$1" = "fedora-23" ] ; then
 		local PROJECTNAME="Fedora 23"
 		write_page "        <p><em>Reproducible $PROJECTNAME</em> is an effort to apply this to $PROJECTNAME. Thus $PROJECTNAME packages are build twice, with a few varitations added and then the resulting packages from the two builds are compared using <a href=\"https://tracker.debian.org/diffoscope\">diffoscope</a>. Please note that the toolchain is not varied at all as the rebuild happens on exactly the same system. More variations are expected to be seen in the wild.</p>"
-		write_page "        <p>Please note that this set up is as new as December 12th, so quite some things are still lacking, eg. https://github.com/kholia/ReproducibleBuilds is not followed at all yet and there are no variations introduced for the 2nd build. Also only 151 source packages from the <em>@buildsys-build</em> package group are currently being tested. OTOH this setup is mature enough that it requires very few trivial changes to build all 17080 source packages in $PROJECTNAME, if it were sensible. Which it isn't right now, but should be soon.</p>"
+		write_page "        <p>Please note that this set up is as new as December 12th, so quite some things are still lacking, eg. https://github.com/kholia/ReproducibleBuilds is not followed at all yet and there are no variations introduced for the 2nd build. Also only about 700 source packages are currently being tested. OTOH this setup is mature enough that it requires very few trivial changes to build all 17080 source packages in $PROJECTNAME, if it were sensible. Which it isn't right now, but should be soon.</p>"
 	fi
 	if [ "$1" != "Arch Linux" ] && [ "$1" != "fedora-23" ] ; then
 		write_page "       <p>There is a weekly run <a href=\"https://jenkins.debian.net/view/reproducible/job/reproducible_$PROJECTNAME/\">jenkins job</a> to test the <code>$BRANCH</code> branch of <a href=\"$PROJECTURL\">$PROJECTNAME.git</a>. The jenkins job is running <a href=\"http://anonscm.debian.org/cgit/qa/jenkins.debian.net.git/tree/bin/reproducible_$PROJECTNAME.sh\">reproducible_$PROJECTNAME.sh</a>$BUILD_ENVIRONMENT and this script is solely responsible for creating this page. Feel invited to join <code>#debian-reproducible</code> (on irc.oftc.net) to request job runs whenever sensible. Patches and other <a href=\"mailto:reproducible-builds@lists.alioth.debian.org\">feedback</a> are very much appreciated - if you want to help, please start by looking at the <a href=\"https://jenkins.debian.net/userContent/todo.html#_reproducible_$(echo $1|tr '[:upper:]' '[:lower:]')\">ToDo list for $1</a>, you might find something easy to contribute."
diff --git a/bin/reproducible_html_rpms.sh b/bin/reproducible_html_rpms.sh
index cc3bb3b5..f826bd2a 100755
--- a/bin/reproducible_html_rpms.sh
+++ b/bin/reproducible_html_rpms.sh
@@ -28,7 +28,7 @@ HTML_TARGET=""
 HTML_RPM_STATS=$(mktemp)
 
 echo "$(date -u) - starting to analyse build results for '$RELEASE' ($ARCH)."
-TOTAL=151 # FIXME: calculate this like $(cat ${ARCHLINUX_PKGS}_$RELEASE | sed -s "s# #\n#g" | wc -l)
+TOTAL=$(cat ${RPM_PKGS}_$RELEASE | sed -s "s# #\n#g" | wc -l)
 TESTED=0
 NR_FTBFS=0
 NR_FTBR=0