From b00c1387d90d3833a892d22c609f9ad4d04590da Mon Sep 17 00:00:00 2001 From: Holger Levsen Date: Mon, 29 Sep 2014 21:29:57 +0200 Subject: reproducible: new job, to build packages which failed and where randomly selected only --- TODO | 7 +++---- bin/reproducible_build.sh | 43 ++++++++++++++++++++++++++++--------------- job-cfg/reproducible.yaml | 9 +++++++++ 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/TODO b/TODO index dbda91d1..89a4d7d3 100644 --- a/TODO +++ b/TODO @@ -20,7 +20,7 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian * move /srv/jenkins.debian.net-scm-sync.git somewhere public? * keep git+svn urls, and use commit hooks. also better for test setups of this setup as well as overall stability. -* fully automate backup and backup /var/lib/jenkins/jobs /var/lib/munin /var/log /root/ too +* fully automate backup and backup /var/lib/jenkins/jobs /var/lib/munin /var/log /root/ /var/lib/jenkins/reproducible.db too * put kgb-client.conf in git and sed passwords from filesystem into it... * push jenkins munin plugins upstream @@ -90,9 +90,8 @@ See link:https://jenkins.debian.net/userContent/about.html["about jenkins.debian === reproducible -* done: improve random jobs to only select previously untested packages -* plus random_any (=the existing random job) then? -* have a job which runs failed packages for which a newer version exists +* have a job which runs failed packages for which a newer version exists, oldest first. doing this in sql is impossible with bash, while easy in python... +* packages from jobs should insert themselves into job_sources and then issue a warning if found in two jobs.. === Test Debian live diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh index bcaafd15..740a20d1 100755 --- a/bin/reproducible_build.sh +++ b/bin/reproducible_build.sh @@ -37,6 +37,10 @@ if [ ! -f ${PACKAGES_DB} ] ; then (suite TEXT NOT NULL, amount INTEGER NOT NULL, PRIMARY KEY (suite))' + sqlite3 ${PACKAGES_DB} ' + CREATE TABLE job_sources + (name TEXT NOT NULL, + job TEXT NOT NULL)' fi # 30 seconds timeout when trying to get a lock INIT=/var/lib/jenkins/reproducible.init @@ -53,21 +57,30 @@ if [[ $1 =~ ^-?[0-9]+$ ]] ; then TMPFILE=$(mktemp) curl http://ftp.de.debian.org/debian/dists/sid/main/source/Sources.xz > $TMPFILE AMOUNT=$1 - REAL_AMOUNT=0 - GUESSES=$(echo "${AMOUNT}*3" | bc) - PACKAGES="" - CANDIDATES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | egrep -v "^linux$"| sort -R | head -$GUESSES | xargs echo) - for PKG in $CANDIDATES ; do - if [ $REAL_AMOUNT -eq $AMOUNT ] ; then - continue - fi - RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"") - if [ "$RESULT" = "" ] ; then - PACKAGES="${PACKAGES} $PKG" - let "REAL_AMOUNT=REAL_AMOUNT+1" - fi - done - AMOUNT=$REAL_AMOUNT + if [ $AMOUNT -gt 0 ] ; then + REAL_AMOUNT=0 + GUESSES=$(echo "${AMOUNT}*3" | bc) + PACKAGES="" + CANDIDATES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | egrep -v "^linux$"| sort -R | head -$GUESSES | xargs echo) + for PKG in $CANDIDATES ; do + if [ $REAL_AMOUNT -eq $AMOUNT ] ; then + continue + fi + RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"") + if [ "$RESULT" = "" ] ; then + PACKAGES="${PACKAGES} $PKG" + let "REAL_AMOUNT=REAL_AMOUNT+1" + fi + done + AMOUNT=$REAL_AMOUNT + for PKG in $PACKAGES ; do + sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO job_sources VALUES ('$PKG','random')" + done + else + # this is kind of a hack: if $1 is 0, then schedule 33 failed packages which were nadomly picked + AMOUNT=33 + PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT source_packages.name FROM source_packages,job_sources WHERE (( source_packages.status = 'unreproducible' OR source_packages.status = 'FTBFS') AND source_packages.name = job_sources.name AND job_sources.job = 'random') ORDER BY source_packages.build_date LIMIT $AMOUNT" | xargs -r echo) + fi # update amount of available packages (for doing statistics later) P_IN_SOURCES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | wc -l) sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_stats VALUES (\"sid\", \"${P_IN_SOURCES}\")" diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index 4bca7a2c..8f70dcd4 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -128,6 +128,10 @@ defaults: reproducible name: '{name}_build_random_packages' +- job-template: + defaults: reproducible + name: '{name}_build_random_failures' + - job-template: defaults: reproducible name: '{name}_build_core' @@ -202,6 +206,11 @@ my_timed: '23 3,7,11,15,19,23 * * *' my_shell: '/srv/jenkins/bin/reproducible_build.sh' my_packages: '66' + - '{name}_build_random_failures': + my_description: 'Try to reproducible build 33 packages which failed before and which come from the random job:' + my_timed: '42 15 * * *' + my_shell: '/srv/jenkins/bin/reproducible_build.sh' + my_packages: '0' - '{name}_build_test_reproducibility': my_description: 'Reproducible build of a few packages, some we know will succeed and some we know which "should" fail: ' my_timed: '' -- cgit v1.2.3-70-g09d2