reproducible: refactor, so that this script can be used on other hosts, also on those not running Debian

1 files changed, 240 insertions, 227 deletions

diff --git a/update_jdn.sh b/update_jdn.sh
index 02e37c2a..110b9cd1 100755
--- a/update_jdn.sh
+++ b/update_jdn.sh
@@ -7,6 +7,7 @@ BASEDIR=/root/jenkins.debian.net
 PVNAME=/dev/vdb      # LVM physical volume for jobs
 VGNAME=jenkins01     # LVM volume group
 STAMP=/var/log/jenkins/update-jenkins.stamp
+TMPFILE=$(mktemp)
 
 explain() {
 	echo
@@ -33,19 +34,21 @@ done
 
 mkdir -p /srv/workspace
 
-if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then
-	echo "tmpfs		/srv/workspace	tmpfs	defaults,size=100g	0	0" >> /etc/fstab
-fi
+if [ "$HOSTNAME" = "jenkins" ] ; then
+	if ! grep -q '^tmpfs\s\+/srv/workspace\s' /etc/fstab; then
+		echo "tmpfs		/srv/workspace	tmpfs	defaults,size=100g	0	0" >> /etc/fstab
+	fi
 
-if ! mountpoint -q /srv/workspace; then
-	if test -z "$(ls -A /srv/workspace)"; then
-		mount /srv/workspace
-	else
-		explain "mountpoint /srv/workspace is non-empty"
+	if ! mountpoint -q /srv/workspace; then
+		if test -z "$(ls -A /srv/workspace)"; then
+			mount /srv/workspace
+		else
+			explain "mountpoint /srv/workspace is non-empty"
+		fi
 	fi
 fi
 
-# make sure needed directories exists
+# make sure needed directories exists - some directories will not be needed on all hosts...
 for directory in /schroots /srv/reproducible-results /srv/d-i /srv/live-build ; do
 	if [ ! -d $directory ] ; then
 		sudo mkdir $directory
@@ -68,136 +71,136 @@ if ! test -h /chroots; then
 	fi
 fi
 
-if ! test -h /var/cache/pbuilder/build; then
-	rmdir /var/cache/pbuilder/build || rm -f /var/cache/pbuilder/build
-	if test -e /var/cache/pbuilder/build; then
-		explain "could not clear /var/cache/pbuilder/build"
-	else
-		ln -s /srv/workspace/pbuilder /var/cache/pbuilder/build
+# only on Debian systems
+if [ -f /etc/debian_version ] ; then
+	if ! test -h /var/cache/pbuilder/build; then
+		rmdir /var/cache/pbuilder/build || rm -f /var/cache/pbuilder/build
+		if test -e /var/cache/pbuilder/build; then
+			explain "could not clear /var/cache/pbuilder/build"
+		else
+			ln -s /srv/workspace/pbuilder /var/cache/pbuilder/build
+		fi
 	fi
-fi
-
-#
-# install packages we need
-#
-if [ ./$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then
-	sudo apt-get install \
-			apache2 \
-			apt-file \
-			apt-listchanges \
-			bash-completion \
-			bc \
-			binfmt-support \
-			bison \
-			build-essential \
-			calamaris \
-			cmake \
-			cron-apt \
-			csvtool \
-			cucumber \
-			curl \
-			debootstrap \
-			devscripts \
-			dnsmasq-base \
-			dose-extra \
-			dstat \
-			etckeeper \
-			figlet \
-			flex \
-			gawk \
-			ghc \
-			gocr \
-			graphviz \
-			haveged \
-			iasl \
-			imagemagick \
-			ip2host \
-			less \
-			libapache2-mod-macro \
-			libav-tools \
-			libcap2-bin \
-			libfile-touch-perl \
-			libguestfs-tools \
-			libjson-rpc-perl \
-			libsikuli-script-java \
-			libsoap-lite-perl \
-			libvirt0 \
-			libvirt-bin \
-			libvirt-dev \
-			libvpx1 \
-			libxslt1-dev \
-			linux-image-amd64 \
-			mock \
-			molly-guard \
-			moreutils \
-			mr \
-			mtr-tiny \
-			munin \
-			munin-plugins-extra \
-			ntp \
-			openbios-ppc \
-			openbios-sparc \
-			openjdk-7-jre \
-			ovmf \
-			pigz \
-			postgresql-client-9.4 \
-			poxml \
-			procmail \
-			python3-debian \
-			python3-psycopg2 \
-			python3-yaml \
-			python-arpy \
-			python-hachoir-metadata \
-			python-imaging \
-			python-lzma \
-			python-pip \
-			python-rpy2 \
-			python-setuptools \
-			python-twisted \
-			python-yaml \
-			qemu \
-			qemu-kvm \
-			qemu-system-x86 \
-			qemu-user-static \
-			radvd \
-			ruby-json \
-			ruby-libvirt \
-			ruby-packetfu \
-			ruby-rjb \
-			ruby-rspec \
-			schroot \
-			screen \
-			seabios \
-			shorewall \
-			shorewall6 \
-			sqlite3 \
-			squid3 \
-			subversion \
-			subversion-tools \
-			sudo \
-			syslinux \
-			tcpdump \
-			unclutter \
-			unzip \
-			vim \
-			virt-viewer \
-			vncsnapshot \
-			vnstat \
-			x11-apps \
-			x11vnc \
-			xtightvncviewer \
-			xvfb \
-			zutils \
-			sysvinit-core
-
-	sudo apt-get install -t jessie-backports \
-			pbuilder
-	#		botch
-
-	explain "Packages installed."
 
-else
-	explain "No new packages to be installed."
+	#
+	# install packages we need
+	#
+	if [ ./$0 -nt $STAMP ] || [ ! -f $STAMP ] ; then
+		sudo apt-get install \
+				apache2 \
+				apt-file \
+				apt-listchanges \
+				bash-completion \
+				bc \
+				binfmt-support \
+				bison \
+				build-essential \
+				calamaris \
+				cmake \
+				cron-apt \
+				csvtool \
+				cucumber \
+				curl \
+				debootstrap \
+				devscripts \
+				dnsmasq-base \
+				dose-extra \
+				dstat \
+				etckeeper \
+				figlet \
+				flex \
+				gawk \
+				ghc \
+				gocr \
+				graphviz \
+				haveged \
+				iasl \
+				imagemagick \
+				ip2host \
+				less \
+				libapache2-mod-macro \
+				libav-tools \
+				libcap2-bin \
+				libfile-touch-perl \
+				libguestfs-tools \
+				libjson-rpc-perl \
+				libsikuli-script-java \
+				libsoap-lite-perl \
+				libvirt0 \
+				libvirt-bin \
+				libvirt-dev \
+				libvpx1 \
+				libxslt1-dev \
+				linux-image-amd64 \
+				mock \
+				molly-guard \
+				moreutils \
+				mr \
+				mtr-tiny \
+				munin \
+				munin-plugins-extra \
+				ntp \
+				openbios-ppc \
+				openbios-sparc \
+				openjdk-7-jre \
+				ovmf \
+				pigz \
+				postgresql-client-9.4 \
+				poxml \
+				procmail \
+				python3-debian \
+				python3-psycopg2 \
+				python3-yaml \
+				python-arpy \
+				python-hachoir-metadata \
+				python-imaging \
+				python-lzma \
+				python-pip \
+				python-rpy2 \
+				python-setuptools \
+				python-twisted \
+				python-yaml \
+				qemu \
+				qemu-kvm \
+				qemu-system-x86 \
+				qemu-user-static \
+				radvd \
+				ruby-json \
+				ruby-libvirt \
+				ruby-packetfu \
+				ruby-rjb \
+				ruby-rspec \
+				schroot \
+				screen \
+				seabios \
+				shorewall \
+				shorewall6 \
+				sqlite3 \
+				squid3 \
+				subversion \
+				subversion-tools \
+				sudo \
+				syslinux \
+				tcpdump \
+				unclutter \
+				unzip \
+				vim \
+				virt-viewer \
+				vncsnapshot \
+				vnstat \
+				x11-apps \
+				x11vnc \
+				xtightvncviewer \
+				xvfb \
+				zutils \
+				sysvinit-core
+		sudo apt-get install -t jessie-backports \
+				pbuilder
+		#		botch
+		explain "Packages installed."
+	else
+		explain "No new packages to be installed."
+	fi
 fi
 
 #
@@ -209,23 +212,27 @@ sudo cp --preserve=mode,timestamps -r hosts/jenkins/etc/* /etc
 #
 # more configuration than a simple cp can do
 #
-if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then
-	sudo a2enmod proxy
-	sudo a2enmod proxy_http
-	sudo a2enmod rewrite
-	sudo a2enmod ssl
-	sudo a2enmod headers
-	sudo a2enmod macro
-	sudo a2enmod filter
-fi
 sudo chown root.root /etc/sudoers.d/jenkins ; sudo chmod 700 /etc/sudoers.d/jenkins
 sudo chown root.root /etc/sudoers.d/jenkins-adm ; sudo chmod 700 /etc/sudoers.d/jenkins-adm
-sudo a2ensite -q jenkins.debian.net
-sudo a2enconf -q munin
-sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.conf
-# for reproducible.d.n url rewriting:
-[ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent
-sudo service apache2 reload
+
+if [ "$HOSTNAME" = "jenkins" ] ; then
+	if [ ! -e /etc/apache2/mods-enabled/proxy.load ] ; then
+		sudo a2enmod proxy
+		sudo a2enmod proxy_http
+		sudo a2enmod rewrite
+		sudo a2enmod ssl
+		sudo a2enmod headers
+		sudo a2enmod macro
+		sudo a2enmod filter
+	fi
+	sudo a2ensite -q jenkins.debian.net
+	sudo a2enconf -q munin
+	sudo chown jenkins-adm.jenkins-adm /etc/apache2/sites-enabled/jenkins.conf
+	# for reproducible.d.n url rewriting:
+	[ -L /var/www/userContent ] || sudo ln -sf /var/lib/jenkins/userContent /var/www/userContent
+	sudo service apache2 reload
+fi
+
 cd /etc/munin/plugins ; sudo rm -f postfix_* open_inodes df_inode interrupts irqstats threads proc_pri vmstat if_err_eth0 fw_forwarded_local fw_packets forks open_files users 2>/dev/null
 [ -L apache_accesses ] || for i in apache_accesses apache_volume ; do ln -s /usr/share/munin/plugins/$i $i ; done
 explain "Packages configured."
@@ -246,48 +253,49 @@ chown -R jenkins:jenkins /var/lib/jenkins/.ssh
 chmod 700 /var/lib/jenkins/.ssh
 chmod 600 /var/lib/jenkins/.ssh/authorized_keys
 explain "Jenkins updated."
-cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/
-TMPFILE=$(mktemp)
-git log | grep ^Author| cut -d " " -f2-|sort -u > $TMPFILE
-echo "----" >> $TMPFILE
-cat THANKS.head > /var/lib/jenkins/userContent/THANKS
-# samuel and lunar committed with several commiters, only display one
-grep -v "samuel.thibault@ens-lyon.org" $TMPFILE | grep -v Lunar >> /var/lib/jenkins/userContent/THANKS
-rm $TMPFILE
-cp -pr userContent /var/lib/jenkins/
-cd /var/lib/jenkins/userContent/
-ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css"
-[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README
-[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO
-[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL
-[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING
-diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS
-mv THANKS .THANKS
-rm TODO README INSTALL CONTRIBUTING
-chown -R jenkins.jenkins /var/lib/jenkins/userContent
-explain "Updated user content for Jenkins."
 
-#
-# run jenkins-job-builder to update jobs if needed
-#     (using sudo because /etc/jenkins_jobs is root:root 700)
-#
-cd /srv/jenkins/job-cfg
-for metaconfig in *.yaml.py ; do
-# there are both python2 and python3 scripts here
-	./$metaconfig > $TMPFILE
-	if ! $(diff ${metaconfig%.py} $TMPFILE > /dev/null) ; then
-		cp $TMPFILE ${metaconfig%.py}
-	fi
-done
-for config in *.yaml ; do
-	if [ $config -nt $STAMP ] || [ ! -f $STAMP ] ; then
-		sudo jenkins-jobs update $config
-	else
-		echo "$config has not changed, nothing to do."
-	fi
-done
-explain "Jenkins jobs updated."
-rm -f $TMPFILE
+if [ "$HOSTNAME" = "jenkins" ] ; then
+	cp -pr README INSTALL TODO CONTRIBUTING d-i-preseed-cfgs /var/lib/jenkins/userContent/
+	git log | grep ^Author| cut -d " " -f2-|sort -u > $TMPFILE
+	echo "----" >> $TMPFILE
+	cat THANKS.head > /var/lib/jenkins/userContent/THANKS
+	# samuel and lunar committed with several commiters, only display one
+	grep -v "samuel.thibault@ens-lyon.org" $TMPFILE | grep -v Lunar >> /var/lib/jenkins/userContent/THANKS
+	rm $TMPFILE
+	cp -pr userContent /var/lib/jenkins/
+	cd /var/lib/jenkins/userContent/
+	ASCIIDOC_PARAMS="-a numbered -a data-uri -a iconsdir=/etc/asciidoc/images/icons -a scriptsdir=/etc/asciidoc/javascripts -b html5 -a toc -a toclevels=4 -a icons -a stylesheet=$(pwd)/theme/debian-asciidoc.css"
+	[ about.html -nt README ] || asciidoc $ASCIIDOC_PARAMS -o about.html README
+	[ todo.html -nt TODO ] || asciidoc $ASCIIDOC_PARAMS -o todo.html TODO
+	[ setup.html -nt INSTALL ] || asciidoc $ASCIIDOC_PARAMS -o setup.html INSTALL
+	[ contributing.html -nt CONTRIBUTING ] || asciidoc $ASCIIDOC_PARAMS -o contributing.html CONTRIBUTING
+	diff THANKS .THANKS >/dev/null || asciidoc $ASCIIDOC_PARAMS -o thanks.html THANKS
+	mv THANKS .THANKS
+	rm TODO README INSTALL CONTRIBUTING
+	chown -R jenkins.jenkins /var/lib/jenkins/userContent
+	explain "Updated user content for Jenkins."
+
+	#
+	# run jenkins-job-builder to update jobs if needed
+	#     (using sudo because /etc/jenkins_jobs is root:root 700)
+	#
+	cd /srv/jenkins/job-cfg
+	for metaconfig in *.yaml.py ; do
+	# there are both python2 and python3 scripts here
+		./$metaconfig > $TMPFILE
+		if ! $(diff ${metaconfig%.py} $TMPFILE > /dev/null) ; then
+			cp $TMPFILE ${metaconfig%.py}
+		fi
+	done
+	for config in *.yaml ; do
+		if [ $config -nt $STAMP ] || [ ! -f $STAMP ] ; then
+			sudo jenkins-jobs update $config
+		else
+			echo "$config has not changed, nothing to do."
+		fi
+	done
+	explain "Jenkins jobs updated."
+fi
 
 #
 # configure git for jenkins
@@ -297,42 +305,46 @@ if [ "$(sudo su - jenkins -c 'git config --get user.email')" != "jenkins@jenkins
 	sudo su - jenkins -c "git config --global user.name Jenkins"
 fi
 
-#
-# configure pbuilder for jenkins user
-#
-sudo chown jenkins /var/cache/pbuilder/result
-
-#
-# creating LVM volume group for jobs
-#
-if [ "$PVNAME" = "" ]; then
-    figlet -f banner Error
-    explain "Set \$PVNAME to physical volume pathname."
-    exit 1
-else
-    if ! sudo pvs $PVNAME >/dev/null 2>&1; then
-        sudo pvcreate $PVNAME
-    fi
-    if ! sudo vgs $VGNAME >/dev/null 2>&1; then
-        sudo vgcreate $VGNAME $PVNAME
-    fi
+if [ -f /etc/debian_version ] ; then
+	#
+	# configure pbuilder for jenkins user
+	#
+	sudo chown jenkins /var/cache/pbuilder/result
 fi
 
-#
-# generate the kgb-client configurations
-#
-cd $BASEDIR
-KGB_SECRETS="/srv/jenkins/kgb/secrets.yml"
-if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then
-    # the last condition is to assure the files are owned by the right user/team
-    if [ "$KGB_SECRETS" -nt $STAMP ] || [ ! -f $STAMP ] ; then
-        sudo -u jenkins-adm "./deploy_kgb.py"
-    else
-        explain "kgb-client configuration unchanged, nothing to do."
-    fi
-else
-    echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated"
-    echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm."
+if [ "$HOSTNAME" = "jenkins" ] ; then
+	#
+	# creating LVM volume group for jobs
+	#
+	if [ "$PVNAME" = "" ]; then
+	    figlet -f banner Error
+	    explain "Set \$PVNAME to physical volume pathname."
+	    exit 1
+	else
+	    if ! sudo pvs $PVNAME >/dev/null 2>&1; then
+	        sudo pvcreate $PVNAME
+	    fi
+	    if ! sudo vgs $VGNAME >/dev/null 2>&1; then
+	        sudo vgcreate $VGNAME $PVNAME
+	    fi
+	fi
+	
+	#
+	# generate the kgb-client configurations
+	#
+	cd $BASEDIR
+	KGB_SECRETS="/srv/jenkins/kgb/secrets.yml"
+	if [ -f "$KGB_SECRETS" ] && [ $(stat -c "%a:%U:%G" "$KGB_SECRETS") = "640:jenkins-adm:jenkins-adm" ] ; then
+	    # the last condition is to assure the files are owned by the right user/team
+	    if [ "$KGB_SECRETS" -nt $STAMP ] || [ ! -f $STAMP ] ; then
+	        sudo -u jenkins-adm "./deploy_kgb.py"
+	    else
+	        explain "kgb-client configuration unchanged, nothing to do."
+	    fi
+	else
+	    echo "Warning: $KGB_SECRETS either does not exist or has bad permissions. Please fix. KGB configs not generated"
+	    echo "We expect the secrets file to be mode 640 and owned by jenkins-adm:jenkins-adm."
+	fi
 fi
 
 #
@@ -346,4 +358,5 @@ rgrep FIXME $BASEDIR/* | grep -v "rgrep FIXME" | grep -v echo
 # finally
 #
 touch $STAMP	# so on the next run, only configs newer than this file will be updated
+rm -f $TMPFILE
 explain "$(hostname -f) successfully updated."