diff options
| -rwxr-xr-x | bin/reproducible_build.sh | 79 | ||||
| -rw-r--r-- | job-cfg/reproducible.yaml | 14 |
2 files changed, 44 insertions, 49 deletions
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh index 4ffb6004..f20ab919 100755 --- a/bin/reproducible_build.sh +++ b/bin/reproducible_build.sh @@ -23,53 +23,45 @@ grep deb-src /etc/apt/sources.list | grep sid sudo apt-get update || ( sleep $(( $RANDOM % 70 + 30 )) ; sudo apt-get update || true ) # update sources table in db -TMPFILE=$(mktemp) -curl http://ftp.de.debian.org/debian/dists/sid/main/source/Sources.xz > $TMPFILE -CSVFILE=$(mktemp) -(xzcat $TMPFILE | egrep "(^Package:|^Version:)" | sed -s "s#^Version: ##g; s#Package: ##g; s#\n# #g"| while read PKG ; do read VERSION ; echo "$PKG,$VERSION" ; done) > $CSVFILE -sqlite3 -csv -init $INIT ${PACKAGES_DB} "DELETE from sources" -echo ".import $CSVFILE sources" | sqlite3 -csv -init $INIT ${PACKAGES_DB} -rm $CSVFILE # $TMPFILE is still being used - -set +x -# if $1 is an integer, build $1 random packages -if [[ $1 =~ ^-?[0-9]+$ ]] ; then - AMOUNT=$1 - if [ $AMOUNT -gt 0 ] ; then - REAL_AMOUNT=0 - GUESSES=$(echo "${AMOUNT}*3" | bc) - PACKAGES="" - CANDIDATES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | egrep -v "^(linux|cups|zurl)$" | sort -R | head -$GUESSES | xargs echo) - for PKG in $CANDIDATES ; do - if [ $REAL_AMOUNT -eq $AMOUNT ] ; then - continue - fi - RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"") - if [ "$RESULT" = "" ] ; then - PACKAGES="${PACKAGES} $PKG" - let "REAL_AMOUNT=REAL_AMOUNT+1" - fi - done - AMOUNT=$REAL_AMOUNT - else - # this is kind of a hack: if $1 is 0, then schedule 33 failed packages which were randomly picked and where a new version is available - AMOUNT=33 - PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT DISTINCT source_packages.name FROM source_packages,sources WHERE sources.version IN (SELECT version FROM sources WHERE name=source_packages.name ORDER by sources.version DESC LIMIT 1) AND (( source_packages.status = 'unreproducible' OR source_packages.status = 'FTBFS') AND source_packages.name = sources.name AND source_packages.version < sources.version) ORDER BY source_packages.build_date LIMIT $AMOUNT" | xargs -r echo) - echo "Info: Only unreproducible and FTBFS packages with a new version available are selected from this job." - AMOUNT=0 - for PKG in $PACKAGES ; do - let "AMOUNT=AMOUNT+1" - done - fi +update_sources_table() { + TMPFILE=$(mktemp) + curl http://ftp.de.debian.org/debian/dists/sid/main/source/Sources.xz > $TMPFILE + CSVFILE=$(mktemp) + (xzcat $TMPFILE | egrep "(^Package:|^Version:)" | sed -s "s#^Version: ##g; s#Package: ##g; s#\n# #g"| while read PKG ; do read VERSION ; echo "$PKG,$VERSION" ; done) > $CSVFILE + sqlite3 -csv -init $INIT ${PACKAGES_DB} "DELETE from sources" + echo ".import $CSVFILE sources" | sqlite3 -csv -init $INIT ${PACKAGES_DB} # update amount of available packages (for doing statistics later) P_IN_SOURCES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | sort -u | wc -l) sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_stats VALUES (\"sid\", \"${P_IN_SOURCES}\")" - rm $TMPFILE + rm $CSVFILE # $TMPFILE is still being used +} + +set +x +if [ $1 = "unknown" ] ; then + update_sources_table + AMOUNT=$2 + REAL_AMOUNT=0 + GUESSES=$(echo "${AMOUNT}*3" | bc) + PACKAGES="" + CANDIDATES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | egrep -v "^(linux|cups|zurl)$" | sort -R | head -$GUESSES | xargs echo) + for PKG in $CANDIDATES ; do + if [ $REAL_AMOUNT -eq $AMOUNT ] ; then + continue + fi + RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"") + if [ "$RESULT" = "" ] ; then + PACKAGES="${PACKAGES} $PKG" + fi + done +elif [ $1 = "known" ] ; then + update_sources_table + AMOUNT=$2 + PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT DISTINCT source_packages.name FROM source_packages,sources WHERE sources.version IN (SELECT version FROM sources WHERE name=source_packages.name ORDER by sources.version DESC LIMIT 1) AND (( source_packages.status = 'unreproducible' OR source_packages.status = 'FTBFS') AND source_packages.name = sources.name AND source_packages.version < sources.version) ORDER BY source_packages.build_date LIMIT $AMOUNT" | xargs -r echo) else # CANDIDATES is defined in that file . /srv/jenkins/bin/reproducible-candidates.sh PACKAGES="" - AMOUNT="10" + AMOUNT=$2 REAL_AMOUNT=0 for i in $(seq 0 ${#CANDIDATES[@]}) ; do if [ $REAL_AMOUNT -eq $AMOUNT ] ; then @@ -82,13 +74,16 @@ else let "REAL_AMOUNT=REAL_AMOUNT+1" fi done - AMOUNT=$REAL_AMOUNT - fi +AMOUNT=0 +for PKG in $PACKAGES ; do + let "AMOUNT=AMOUNT+1" +done echo "=============================================================================" echo "The following $AMOUNT source packages will be build: ${PACKAGES}" echo "=============================================================================" echo +rm -f $TMPFILE cleanup_all() { rm -r $TMPDIR diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index 4a436b6d..8d390463 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -33,7 +33,7 @@ artifactDaysToKeep: -1 artifactNumToKeep: -1 builders: - - shell: '{my_shell} {my_param}' + - shell: '{my_shell} {my_params}' publishers: - email: recipients: 'jenkins+debian-reproducible holger@layer-acht.org' @@ -140,25 +140,25 @@ my_description: 'Setup pbuilder for reproducible builds as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example' my_timed: '42 0 * * *' my_shell: '/srv/jenkins/bin/reproducible_setup.sh' - my_param: '' + my_params: '' - '{name}_stats': my_description: 'Reproducible builds statistics' my_timed: '0 0 * * *' my_shell: '/srv/jenkins/bin/reproducible_stats.sh' - my_param: '' + my_params: '' - '{name}_build_random_packages': my_description: 'Reproducible build this many random packages each day:' my_timed: '0,15,30,45 * * * *' my_shell: '/srv/jenkins/bin/reproducible_build.sh' - my_param: '10' + my_params: 'unknown 10' - '{name}_build_random_failures': - my_description: 'Try to reproducible build 33 packages which failed before and which come from the random job:' + my_description: 'Try to reproducible build packages which were build before and for which a new version exists:' my_timed: '42 2,14 * * *' my_shell: '/srv/jenkins/bin/reproducible_build.sh' - my_param: '0' + my_params: 'known 25' - '{name}_build_candidates': my_description: 'Reproducible build packages from a list of pre-defined candidates: ' my_timed: '40,15,30,45 3,9,15,21 * * *' my_shell: '/srv/jenkins/bin/reproducible_build.sh' - my_param: '' + my_params: 'candidates 20' |