diff options
| author | Holger Levsen <holger@layer-acht.org> | 2015-10-03 11:43:46 +0200 |
|---|---|---|
| committer | Holger Levsen <holger@layer-acht.org> | 2015-10-03 11:43:46 +0200 |
| commit | 6026ac4931af6453c8f07717d6513a398325c1bb (patch) | |
| tree | 628369842752891f480a5a1620815a12137b457b | |
| parent | 65c85c4a007b076f1ea9bcca0f6f9d89cb84e75f (diff) | |
| download | jenkins.debian.net-6026ac4931af6453c8f07717d6513a398325c1bb.tar.xz | |
reproducible: detect enospace during builds, if so, send notifications, unregister this build, pause the builder and abort the job
| -rw-r--r-- | TODO | 3 | ||||
| -rwxr-xr-x | bin/reproducible_build.sh | 17 |
2 files changed, 17 insertions, 3 deletions
@@ -124,9 +124,6 @@ properties: === reproducible builds * make reproducible_build.sh rock solid again: -** deal properly with out of diskspace problems, https://jenkins.debian.net/job/reproducible_builder_amd64_8/2575/ is an archived build which hit it -*** '~jenkins/spacetest' has a list of affected logs in testing/amd64, though at least the syslinux one is a false positive -*** this is important cause it causes packages to falsely ftbfs ** fix: "DIFFOSCOPE='E: Failed to change to directory /tmp: Permission denied' - maybe by making sure the cause is gone… https://jenkins.debian.net/job/reproducible_builder_amd64_14/909/ is an example for that ** fix disorderfs setup to *always* unmount+cleanup, this causes the full disks atm ** diffoscope needs to be run on the target arch..., this should probably be doable with qemu-static on the host diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh index 648cf362..722e9d38 100755 --- a/bin/reproducible_build.sh +++ b/bin/reproducible_build.sh @@ -203,6 +203,12 @@ handle_ftbfs() { handle_depwait return fi + for NEEDLE in '^tar:.*Cannot write: No space left on device' 'fatal error: error writing to .* No space left on device' './configure: line .* printf: write error: No space left on device' 'cat: write error: No space left on device' ; do + if zgrep -e "$NEEDLE" "$BASE/logs/$SUITE/$ARCH/${SRCPACKAGE}_${EVERSION}.build${BUILD}.log.gz" ; then + handle_enospace + return + fi + done done calculate_build_duration update_db_and_html "FTBFS" @@ -266,6 +272,17 @@ handle_unhandled() { exit 0 } +handle_enospace() { + unregister_build + MESSAGE="$BUILD_URL ran into diskspace problems, please investigate. Will send this builder to sleep for 3h now." + echo "$MESSAGE" + echo "$MESSAGE" | mail -s "$BUILD_URL ran into diskspace problems" qa-jenkins-scm@lists.alioth.debian.org + irc_msg "$MESSAGE" + sleep 3h + exec /srv/jenkins/bin/abort.sh + exit 0 +} + dbd_timeout() { local msg="$DIFFOSCOPE was killed after running into timeout after $1" if [ ! -s ./${DBDREPORT} ] ; then |