diff options
| -rw-r--r-- | TODO | 1 | ||||
| -rwxr-xr-x | bin/reproducible_build.sh | 9 | ||||
| -rwxr-xr-x | bin/reproducible_html_live_status.py | 2 | ||||
| -rwxr-xr-x | bin/reproducible_maintenance.sh | 10 |
4 files changed, 20 insertions, 2 deletions
@@ -181,7 +181,6 @@ properties: ** new page: packages which are orphaned but have a reproducible usertagged patch ** cleanup old testing + diffscope schroots on armhf nodes, diffoscope schroots on pb-amd64 nodes and old pbuilder setups on jenkins ** reproducible_build.sh changes needed: -*** should look for other builds "running" as this build job in the db on start and cleanup these from the db and prepare bits for a daily mail ... *** fix: "DIFFOSCOPE='E: Failed to change to directory ‘/tmp’: Permission denied" - maybe by making sure the cause is gone… *** race condition with itself? https://jenkins.debian.net/job/reproducible_builder_armhf_6/716/console says Warning, package php5 in unstable on armhf is probably already building elsewhere, exiting. Please check https://jenkins.debian.net/job/reproducible_builder_armhf_6/716/ which does not compute. *** diffoscope needs to be run on the target arch... diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh index 6919b768..2d6bce0a 100755 --- a/bin/reproducible_build.sh +++ b/bin/reproducible_build.sh @@ -415,6 +415,15 @@ init_package_build() { echo "=============================================================================" echo "Initialising reproducibly build of ${SRCPACKAGE} in ${SUITE} on ${ARCH} on $(hostname -f) now. $ANNOUNCE" echo "=============================================================================" + # remove previous build attempts which didnt finish correctly + BUILDER_PREFIX="${JOB_NAME#reproducible_builder_}/" + BAD_BUILDS=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT package_id, date_build_started, builder FROM schedule WHERE builder LIKE '${BUILDER_PREFIX}%'") + if [ ! -z "$BAD_BUILDS" ] ; then + # stale_builds.txt is mailed once a day by reproducible_maintenance.sh + echo "$(date -u) - stale builds detected, cleaning up database from these entries:" | tee -a /var/lib/jenkins/stale_builds.txt + echo -n $BAD_BUILDS | tee -a /var/lib/jenkins/stale_builds.txt + sqlite3 -init $INIT ${PACKAGES_DB} "UPDATE schedule SET date_build_started='' WHERE builder LIKE '${BUILDER_PREFIX}%'" + fi # mark build attempt if [ -z "$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT date_build_started FROM schedule WHERE package_id = '$SRCPKGID'")" ] ; then sqlite3 -init $INIT ${PACKAGES_DB} "UPDATE schedule SET date_build_started='$DATE', builder='$BUILDER' WHERE package_id = '$SRCPKGID'" diff --git a/bin/reproducible_html_live_status.py b/bin/reproducible_html_live_status.py index 334a31b5..1d55bb39 100755 --- a/bin/reproducible_html_live_status.py +++ b/bin/reproducible_html_live_status.py @@ -78,7 +78,7 @@ def generate_live_status_table(arch): counter += 1 # the numbers 17 and 9 should really be derived from /var/lib/jenkins/jobs/reproducible_builder_${arch}_* instead of being hard-coded here... if ( arch == 'amd64' and counter == 17 ) or ( arch == 'armhf' and counter == 9 ): - html += '<tr><td colspan="10">There are more builds marked as currently building in the database than there are ' + arch + ' build jobs. This does not compute. Please cleanup and please automate cleanup.</td></tr>' + html += '<tr><td colspan="10">There are more builds marked as currently building in the database than there are ' + arch + ' build jobs. This does not compute, please investigate and fix the cause.</td></tr>' suite = row[1] arch = row[2] pkg = row[3] diff --git a/bin/reproducible_maintenance.sh b/bin/reproducible_maintenance.sh index c577a659..0fd38938 100755 --- a/bin/reproducible_maintenance.sh +++ b/bin/reproducible_maintenance.sh @@ -336,6 +336,16 @@ if [ ! -z "$BADPERMS" ] ; then echo fi +# once a day, send mail about stale builds +if [ "$HOSTNAME" = "jenkins" ] && [ $(date -u +%H) -eq 0 ] ; then + if [ -s /var/lib/jenkins/stale_builds.txt ] ; then + TMPFILE=$(mktemp --tmpdir=$TEMPDIR maintenance-XXXXXXXXXXXX) + mv /var/lib/jenkins/stale_builds.txt $TMPFILE + cat $TMPFILE | mail -s "stale builds found" qa-jenkins-scm@lists.alioth.debian.org + rm -f $TMPFILE + fi +fi + if ! $DIRTY ; then echo "$(date -u ) - Everything seems to be fine." echo |