diff options
| -rw-r--r-- | TODO | 3 | ||||
| -rwxr-xr-x | bin/reproducible_build.sh | 52 |
2 files changed, 11 insertions, 44 deletions
@@ -173,8 +173,7 @@ properties: * higher prio: ** reproducible_build.sh changes needed: *** fix the cause for the frequent 404s or workaround it well -*** fix: "DIFFOSCOPE='E: Failed to change to directory ‘/tmp’: Permission denied" - maybe by making sure the cause is gone… https://jenkins.debian.net/job/reproducible_builder_amd64_14/909/ is an example for that -*** race condition with itself? https://jenkins.debian.net/job/reproducible_builder_armhf_6/716/console says Warning, package php5 in unstable on armhf is probably already building elsewhere, exiting. Please check https://jenkins.debian.net/job/reproducible_builder_armhf_6/716/ which does not compute. +*** fix: "DIFFOSCOPE='E: Failed to change to directory /tmp: Permission denied' - maybe by making sure the cause is gone… https://jenkins.debian.net/job/reproducible_builder_amd64_14/909/ is an example for that *** deal properly with out of diskspace problems, https://jenkins.debian.net/job/reproducible_builder_amd64_8/2575/ is an archived build which hit it **** '~jenkins/spacetest' has a list of affected logs in testing/amd64, though at least the syslinux one is a false positive *** diffoscope needs to be run on the target arch... diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh index 6428baca..c05e1eab 100755 --- a/bin/reproducible_build.sh +++ b/bin/reproducible_build.sh @@ -29,19 +29,8 @@ create_results_dirs() { handle_race_condition() { echo | tee -a $BUILDLOG - local msg="Warning, package ${SRCPACKAGE} in ${SUITE} on ${ARCH} is probably already building elsewhere, exiting.\n" - msg="${msg}Please check $BUILD_URL and https://reproducible.debian.net/index_${ARCH}_scheduled.html for a different build of $SUITE/$ARCH/${SRCPACKAGE}.\n" - case $1 in - "db") - msg="${msg}The race condition was caught while marking the build attempt in the database.\n" - ;; - "init") - msg="${msg}The race condition was caught while writing the lockfile.\n" - ;; - "lockfile") - msg="${msg}The race condition was caught while checking the lockfile for pid correctness.\n" - ;; - esac + local RESULT=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT builder FROM schedule WHERE package_id = '$SRCPKGID'") + local msg="Warning, package ${SRCPACKAGE} in ${SUITE} on ${ARCH} is probably already building at $RESULT, while this is $BUILD_URL.\n" printf "$msg" | tee -a $BUILDLOG printf "$(date -u) - $msg" >> /var/log/jenkins/reproducible-race-conditions.log echo "$(date -u) - Terminating this build quickly and nicely..." | tee -a $RBUILDLOG @@ -52,18 +41,10 @@ handle_race_condition() { # cleanup cd rm -r $TMPDIR || true - if ! $BAD_LOCKFILE ; then rm -f $LOCKFILE ; fi exec /srv/jenkins/bin/abort.sh exit 0 } -check_for_race_conditions() { - if [ $$ -ne $(cat "$LOCKFILE") ] ; then - BAD_LOCKFILE=true - handle_race_condition lockfile - fi -} - save_artifacts() { local random=$(head /dev/urandom | tr -cd '[:alnum:]'| head -c5) local ARTIFACTS="artifacts/r00t-me/${SRCPACKAGE}_${SUITE}_tmp-${random}" @@ -108,7 +89,6 @@ cleanup_all() { cd rm -r $TMPDIR || true fi - if ! $BAD_LOCKFILE ; then rm -f $LOCKFILE ; fi } update_db_and_html() { @@ -415,8 +395,6 @@ init_package_build() { local ANNOUNCE="Artifacts will be preserved." fi create_results_dirs - # used to catch race conditions when the same package is build by two parallel jobs - LOCKFILE="/tmp/reproducible-lockfile-${SUITE}-${ARCH}-${SRCPACKAGE}" echo "=============================================================================" echo "Initialising reproducibly build of ${SRCPACKAGE} in ${SUITE} on ${ARCH} on $(hostname -f) now. $ANNOUNCE" echo "=============================================================================" @@ -432,20 +410,17 @@ init_package_build() { echo >> /var/lib/jenkins/stale_builds.txt fi rm -f $BAD_BUILDS - # mark build attempt + # mark build attempt… local RESULT=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT date_build_started FROM schedule WHERE package_id = '$SRCPKGID'") if [ -z "$RESULT" ] ; then - sqlite3 -init $INIT ${PACKAGES_DB} "UPDATE schedule SET date_build_started='$DATE', builder='$BUILDER' WHERE package_id = '$SRCPKGID'" - else - echo "query resulted in $RESULT" - BAD_LOCKFILE=true - handle_race_condition db - fi - if [ ! -f "$LOCKFILE" ] ; then - echo $$ > "$LOCKFILE" + # check this worked… + sqlite3 -init $INIT ${PACKAGES_DB} "UPDATE schedule SET date_build_started='$DATE', builder='$BUILDER' WHERE package_id = '$SRCPKGID' AND date_build_started='' AND builder=''" + RESULT=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT date_build_started FROM schedule WHERE package_id = '$SRCPKGID' AND date_build_started='$DATE' AND builder='$BUILDER'") + if [ -z "$RESULT" ] ; then + handle_race_condition + fi else - BAD_LOCKFILE=true - handle_race_condition init + handle_race_condition fi echo "$(date -u ) - starting to build ${SRCPACKAGE}/${SUITE}/${ARCH} on $(hostname -f) on '$DATE'" | tee ${RBUILDLOG} echo "The jenkins build log is/was available at ${BUILD_URL}console" | tee -a ${RBUILDLOG} @@ -662,7 +637,6 @@ build_rebuild() { exit 0 elif [ -f b1/${SRCPACKAGE}_${EVERSION}_${ARCH}.changes ] ; then # the first build did not FTBFS, try rebuild it. - check_for_race_conditions remote_build 2 $NODE2 $PORT2 if [ -f b2/${SRCPACKAGE}_${EVERSION}_${ARCH}.changes ] ; then # both builds were fine, i.e., they did not FTBFS. @@ -685,7 +659,6 @@ cd $TMPDIR DATE=$(date -u +'%Y-%m-%d %H:%M') START=$(date +'%s') RBUILDLOG=$(mktemp --tmpdir=$TMPDIR) -BAD_LOCKFILE=false BUILDER="${JOB_NAME#reproducible_builder_}/${BUILD_ID}" ARCH="$(dpkg --print-architecture)" @@ -703,8 +676,6 @@ elif [ "$1" = "1" ] || [ "$1" = "2" ] ; then TMPDIR="$4" [ -d $TMPDIR ] || mkdir -p $TMPDIR cd $TMPDIR - # used to catch race conditions when the same package is being built by two parallel jobs - LOCKFILE="/tmp/reproducible-lockfile-${SUITE}-${ARCH}-${SRCPACKAGE}" get_source_package mkdir b$MODE if [ "$MODE" = "1" ] ; then @@ -746,9 +717,7 @@ get_source_package cat ${SRCPACKAGE}_${EVERSION}.dsc | tee -a ${RBUILDLOG} -check_for_race_conditions check_suitability -check_for_race_conditions build_rebuild # defines FTBFS redefines RBUILDLOG if [ $FTBFS -eq 0 ] ; then check_buildinfo @@ -762,7 +731,6 @@ elif [ $FTBFS -eq 0 ] ; then call_diffoscope_on_changes_files # defines DIFFOSCOPE, update_db_and_html defines STATUS fi -check_for_race_conditions cd .. cleanup_all trap - INT TERM EXIT |