diff options
| -rw-r--r-- | TODO | 11 | ||||
| -rwxr-xr-x | bin/jenkins_node_wrapper.sh | 2 | ||||
| -rwxr-xr-x | bin/reproducible_build_rpm.sh | 228 | ||||
| -rwxr-xr-x | bin/reproducible_common.sh | 3 | ||||
| -rw-r--r-- | job-cfg/reproducible.yaml | 11 |
5 files changed, 251 insertions, 4 deletions
@@ -282,12 +282,15 @@ This is about Debian, below are more todo entries for other projects… ==== reproducible Fedora * build script -** call the script reproducible_rpms.sh as it can also build OpenSuSE packages -** then use yumdownloader to download rpms: 'yumdownloader --source sudo' -** use mock to build: 'mock -r fedora-23-x86_64 --resultdir=. sudo-1.8.14p3-1.fc23.src.rpm' +** how to get a list of packages? +** no variations introduced yet: +*** use '-j$NUM_CPU' and 'NEW_NUM_CPU=$(echo $NUM_CPU-1|bc)' +*** modify TZ, LANG, LC_ALL, umask +* other bits: +** use modified rpmbuild package * hosts/pb-build3/etc/yum/repos.d/* is really not sooo good but works… -* how to get a list of packages? * build rawhide instead of fedora-23? +** when adding rawhide make sure this is not needed: 'yumdownloader --enablerepo=fedora-23-src --source sudo' * more notes: ** http://blog.packagecloud.io/eng/2015/05/11/building-rpm-packages-with-mock/ diff --git a/bin/jenkins_node_wrapper.sh b/bin/jenkins_node_wrapper.sh index 22aa3e60..ccd1645a 100755 --- a/bin/jenkins_node_wrapper.sh +++ b/bin/jenkins_node_wrapper.sh @@ -109,6 +109,8 @@ elif [[ "$*" =~ reproducible_setup_mock_fedora23_x86_64 ]] ; then exec /srv/jenkins/bin/reproducible_setup_mock.sh fedora-23 x86_64 ; croak "Exec failed"; elif [ "$1" = "/srv/jenkins/bin/reproducible_build_archlinux_pkg.sh" ] && ( [ "$2" = "1" ] || [ "$2" = "2" ] ) ; then exec /srv/jenkins/bin/reproducible_build_archlinux_pkg.sh "$2" "$3" "$4" "$5" ; croak "Exec failed"; +elif [ "$1" = "/srv/jenkins/bin/reproducible_build_rpm.sh" ] && ( [ "$2" = "1" ] || [ "$2" = "2" ] ) ; then + exec /srv/jenkins/bin/reproducible_build_rpm.sh "$2" "$3" "$4" "$5" "$6" ; croak "Exec failed"; elif [ "$*" = "some_jenkins_job_name" ] ; then exec echo run any commands here ; croak "Exec failed"; fi diff --git a/bin/reproducible_build_rpm.sh b/bin/reproducible_build_rpm.sh new file mode 100755 index 00000000..37c74e6f --- /dev/null +++ b/bin/reproducible_build_rpm.sh @@ -0,0 +1,228 @@ +#!/bin/bash + +# Copyright 2015 Holger Levsen <holger@layer-acht.org> +# released under the GPLv=2 + +DEBUG=false +. /srv/jenkins/bin/common-functions.sh +common_init "$@" + +# common code +. /srv/jenkins/bin/reproducible_common.sh + +set -e + +cleanup_all() { + cd + # delete makepkg build dir + if [ ! -z $SRCPACKAGE ] && [ -d /tmp/$SRCPACKAGE-$(basename $TMPDIR) ] ; then + rm -r /tmp/$SRCPACKAGE-$(basename $TMPDIR) + fi + # delete main work dir (only on master) + if [ "$MODE" = "master" ] ; then + rm $TMPDIR -r + echo "$(date -u) - $TMPDIR deleted." + fi + rm -f $DUMMY > /dev/null || true +} + +handle_remote_error() { + MESSAGE="${BUILD_URL}console got remote error $1" + echo "$(date -u ) - $MESSAGE" | tee -a /var/log/jenkins/reproducible-remote-error.log + echo "Sleeping 5m before aborting the job." + sleep 5m + exec /srv/jenkins/bin/abort.sh + exit 0 +} + + +download_package() { + echo "$(date -u ) - downloading ${SRCPACKAGE} for $RELEASE now." + yumdownloader --source ${SRCPACKAGE} + SRC_RPM="$(ls $SRCPACKAGE*.src.rpm)" +} + +choose_package() { + echo "$(date -u ) - choosing package to be build." + local MIN_AGE=6 + for PKG in sudo ; do + # build package if it has never build or at least $MIN_AGE days ago + if [ ! -d $BASE/rpm/$RELEASE/$ARCH/$PKG ] || [ ! -z $(find $BASE/rpm/$RELEASE/$ARCH/ -name $PKG -mtime +$MIN_AGE) ] ; then + SRCPACKAGE=$PKG + echo "$(date -u ) - building package $PKG from '$RELEASE' on '$ARCH' now..." + # very simple locking… + mkdir -p $BASE/rpm/$RELEASE/$ARCH/$PKG + touch $BASE/rpm/$RELEASE/$ARCH/$PKG + # break out of the loop and then out of this function too, + # to build this package… + break + fi + done + done + if [ -z $SRCPACKAGE ] ; then + echo "$(date -u ) - no package found to be build, sleeping 6h." + for i in $(seq 1 12) ; do + sleep 30m + echo "$(date -u ) - still sleeping..." + done + echo "$(date -u ) - exiting cleanly now." + exit 0 + fi +} + +first_build() { + echo "=============================================================================" + echo "Building for $RELEASE ($ARCH) on $(hostname -f) now." + echo "Source package: ${SRCPACKAGE}" + echo "Date: $(date -u)" + echo "=============================================================================" + set -x + download_package + local BUILDDIR="/tmp/$SRCPACKAGE-$(basename $TMPDIR)" + local LOG=$TMPDIR/b1/$SRCPACKAGE/build1.log + # nicely run mock with a timeout of 4h + timeout -k 4.1h 4h /usr/bin/ionice -c 3 /usr/bin/nice \ + mock -r $RELEASE-$ARCH --resultdir=. $SRC_RPM 2>&1 | tee -a $LOG + PRESULT=${PIPESTATUS[0]} + if [ $PRESULT -eq 124 ] ; then + echo "$(date -u) - mock was killed by timeout after 4h." | tee -a $LOG + fi + if ! "$DEBUG" ; then set +x ; fi +} + +second_build() { + echo "=============================================================================" + echo "Re-Building for $RELEASE ($ARCH) on $(hostname -f) now." + echo "Source package: ${SRCPACKAGE}" + echo "Date: $(date -u)" + echo "=============================================================================" + set -x + download_package + local BUILDDIR="/tmp/$SRCPACKAGE-$(basename $TMPDIR)" + local LOG=$TMPDIR/b2/$SRCPACKAGE/build2.log + # NEW_NUM_CPU=$(echo $NUM_CPU-1|bc) + # nicely run mock with a timeout of 4h + timeout -k 4.1h 4h /usr/bin/ionice -c 3 /usr/bin/nice \ + mock -r $RELEASE-$ARCH --resultdir=. $SRC_RPM 2>&1 | tee -a $LOG + PRESULT=${PIPESTATUS[0]} + if [ $PRESULT -eq 124 ] ; then + echo "$(date -u) - mock was killed by timeout after 4h." | tee -a $LOG + fi + if ! "$DEBUG" ; then set +x ; fi +} + +remote_build() { + local BUILDNR=$1 + local NODE=$RPM_BUILD_NODE + local FQDN=$NODE.debian.net + local PORT=22 + set +e + ssh -p $PORT $FQDN /bin/true + RESULT=$? + # abort job if host is down + if [ $RESULT -ne 0 ] ; then + SLEEPTIME=$(echo "$BUILDNR*$BUILDNR*5"|bc) + echo "$(date -u) - $NODE seems to be down, sleeping ${SLEEPTIME}min before aborting this job." + sleep ${SLEEPTIME}m + exec /srv/jenkins/bin/abort.sh + fi + ssh -p $PORT $FQDN /srv/jenkins/bin/reproducible_build_rpm.sh $BUILDNR $RELEASE $ARCH ${SRCPACKAGE} ${TMPDIR} + RESULT=$? + if [ $RESULT -ne 0 ] ; then + ssh -p $PORT $FQDN "rm -r $TMPDIR" || true + handle_remote_error "with exit code $RESULT from $NODE for build #$BUILDNR for ${SRCPACKAGE} from $RELEASE ($ARCH)" + fi + rsync -e "ssh -p $PORT" -r $FQDN:$TMPDIR/b$BUILDNR $TMPDIR/ + RESULT=$? + if [ $RESULT -ne 0 ] ; then + echo "$(date -u ) - rsync from $NODE failed, sleeping 2m before re-trying..." + sleep 2m + rsync -e "ssh -p $PORT" -r $FQDN:$TMPDIR/b$BUILDNR $TMPDIR/ + RESULT=$? + if [ $RESULT -ne 0 ] ; then + handle_remote_error "when rsyncing remote build #$BUILDNR results from $NODE" + fi + fi + ls -R $TMPDIR + ssh -p $PORT $FQDN "rm -r $TMPDIR" + set -e +} + +# +# below is what controls the world +# + +TMPDIR=$(mktemp --tmpdir=/srv/reproducible-results -d) # where everything actually happens +trap cleanup_all INT TERM EXIT +cd $TMPDIR + +DATE=$(date -u +'%Y-%m-%d %H:%M') +START=$(date +'%s') +BUILDER="${JOB_NAME#reproducible_builder_}/${BUILD_ID}" +DUMMY=$(mktemp -t rpm-dummy-XXXXXXXX) + +# +# determine mode +# +if [ "$1" = "1" ] || [ "$1" = "2" ] ; then + MODE="$1" + RELEASE="$2" + ARCH="$3" + SRCPACKAGE="$4" + TMPDIR="$5" + [ -d $TMPDIR ] || mkdir -p $TMPDIR + cd $TMPDIR + mkdir -p b$MODE/$SRCPACKAGE + if [ "$MODE" = "1" ] ; then + first_build + else + second_build + fi + # preserve results and delete build directory + mv -v /tmp/$SRCPACKAGE-$(basename $TMPDIR)/$SRCPACKAGE/*.pkg.tar.xz $TMPDIR/b$MODE/$SRCPACKAGE/ || ls /tmp/$SRCPACKAGE-$(basename $TMPDIR)/$SRCPACKAGE/ + rm -r /tmp/$SRCPACKAGE-$(basename $TMPDIR)/ + echo "$(date -u) - build #$MODE for $SRCPACKAGE on $HOSTNAME done." + exit 0 +fi +MODE="master" + +# +# main - only used in master-mode +# +delay_start # randomize start times +# first, we need to choose a packagey… +RELEASE="$1" +ARCH="$2" +SRCPACKAGE="" # package name +SRC_RPM="" # src rpm file name +choose_package +# build package twice +mkdir b1 b2 +remote_build 1 +# only do the 2nd build if the 1st produced results +if [ ! -z "$(ls $TMPDIR/b1/$SRCPACKAGE/*.rpm 2>/dev/null|| true)" ] ; then + remote_build 2 + # run diffoscope on the results + TIMEOUT="30m" + DIFFOSCOPE="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --version 2>&1)" + echo "$(date -u) - Running $DIFFOSCOPE now..." + cd $TMPDIR/b1/$SRCPACKAGE + for ARTIFACT in *.rpm ; do + [ -f $ARTIFACT ] || continue + call_diffoscope $SRCPACKAGE $ARTIFACT + # publish page + if [ -f $TMPDIR/$SRCPACKAGE/$ARTIFACT.html ] ; then + cp $TMPDIR/$SRCPACKAGE/$ARTIFACT.html $BASE/rpm/$RELEASE/$ARCH/$SRCPACKAGE/ + fi + done +fi +# publish logs +cd $TMPDIR/b1/$SRCPACKAGE +cp build1.log $BASE/rpm/$RELEASE/$ARCH/$SRCPACKAGE/ +[ ! -f $TMPDIR/b2/$SRCPACKAGE/build2.log ] || cp $TMPDIR/b2/$SRCPACKAGE/build2.log $BASE/rpm/$RELEASE/$ARCH/$SRCPACKAGE/ +echo "$(date -u) - $REPRODUCIBLE_URL/rpm/$RELEASE/$ARCH/$SRCPACKAGE/ updated." + +cd +cleanup_all +trap - INT TERM EXIT + diff --git a/bin/reproducible_common.sh b/bin/reproducible_common.sh index a6e4a154..43349a3b 100755 --- a/bin/reproducible_common.sh +++ b/bin/reproducible_common.sh @@ -56,6 +56,9 @@ USERTAGS="toolchain infrastructure timestamps fileordering buildpath username ho ARCHLINUX_BUILD_NODE=profitbricks-build3-amd64 ARCHLINUX_REPOS="core extra" +# common settings for testing rpm based distros +RPM_BUILD_NODE=profitbricks-build3-amd64 + # number of cores to be used NUM_CPU=$(grep -c '^processor' /proc/cpuinfo) diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index e7ed8b0b..b59602da 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -774,6 +774,10 @@ name: '{name}_builder_archlinux_2' - job-template: + defaults: reproducible_builder_trigger + name: '{name}_builder_fedora23_x86_64_1' + +- job-template: defaults: reproducible name: '{name}_html_archlinux' @@ -1555,6 +1559,13 @@ my_shell: '/srv/jenkins/bin/reproducible_build_archlinux_pkg.sh' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' my_node: '' + - '{name}_builder_fedora23_x86_64_1': + my_description: 'Try to reproducibly build a Fedora 23 package.' + my_timed: '* * * * *' + my_trigger: '' + my_shell: '/srv/jenkins/bin/reproducible_build_rpm.sh fedora23 x86_64' + my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' + my_node: '' - '{name}_html_archlinux': my_description: 'Build a simple webpage for Arch Linux reproducibility tests' my_timed: '' |