summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorHolger Levsen <holger@layer-acht.org>2014-10-05 04:54:55 +0200
committerHolger Levsen <holger@layer-acht.org>2014-10-05 05:06:12 +0200
commit53d47cc169e005cafdb2d84e8347b5f1603579ca (patch)
treed8aec617c6f0f0532e816237883679e3978f483a
parentaa6959fccd4e53b2c85689680d4686f7daf005f8 (diff)
downloadjenkins.debian.net-53d47cc169e005cafdb2d84e8347b5f1603579ca.tar.xz
reproducible: refactor, sanitize param handling of _build.sh
-rwxr-xr-xbin/reproducible_build.sh79
-rw-r--r--job-cfg/reproducible.yaml14
2 files changed, 44 insertions, 49 deletions
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index 4ffb6004..f20ab919 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -23,53 +23,45 @@ grep deb-src /etc/apt/sources.list | grep sid
sudo apt-get update || ( sleep $(( $RANDOM % 70 + 30 )) ; sudo apt-get update || true )
# update sources table in db
-TMPFILE=$(mktemp)
-curl http://ftp.de.debian.org/debian/dists/sid/main/source/Sources.xz > $TMPFILE
-CSVFILE=$(mktemp)
-(xzcat $TMPFILE | egrep "(^Package:|^Version:)" | sed -s "s#^Version: ##g; s#Package: ##g; s#\n# #g"| while read PKG ; do read VERSION ; echo "$PKG,$VERSION" ; done) > $CSVFILE
-sqlite3 -csv -init $INIT ${PACKAGES_DB} "DELETE from sources"
-echo ".import $CSVFILE sources" | sqlite3 -csv -init $INIT ${PACKAGES_DB}
-rm $CSVFILE # $TMPFILE is still being used
-
-set +x
-# if $1 is an integer, build $1 random packages
-if [[ $1 =~ ^-?[0-9]+$ ]] ; then
- AMOUNT=$1
- if [ $AMOUNT -gt 0 ] ; then
- REAL_AMOUNT=0
- GUESSES=$(echo "${AMOUNT}*3" | bc)
- PACKAGES=""
- CANDIDATES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | egrep -v "^(linux|cups|zurl)$" | sort -R | head -$GUESSES | xargs echo)
- for PKG in $CANDIDATES ; do
- if [ $REAL_AMOUNT -eq $AMOUNT ] ; then
- continue
- fi
- RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"")
- if [ "$RESULT" = "" ] ; then
- PACKAGES="${PACKAGES} $PKG"
- let "REAL_AMOUNT=REAL_AMOUNT+1"
- fi
- done
- AMOUNT=$REAL_AMOUNT
- else
- # this is kind of a hack: if $1 is 0, then schedule 33 failed packages which were randomly picked and where a new version is available
- AMOUNT=33
- PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT DISTINCT source_packages.name FROM source_packages,sources WHERE sources.version IN (SELECT version FROM sources WHERE name=source_packages.name ORDER by sources.version DESC LIMIT 1) AND (( source_packages.status = 'unreproducible' OR source_packages.status = 'FTBFS') AND source_packages.name = sources.name AND source_packages.version < sources.version) ORDER BY source_packages.build_date LIMIT $AMOUNT" | xargs -r echo)
- echo "Info: Only unreproducible and FTBFS packages with a new version available are selected from this job."
- AMOUNT=0
- for PKG in $PACKAGES ; do
- let "AMOUNT=AMOUNT+1"
- done
- fi
+update_sources_table() {
+ TMPFILE=$(mktemp)
+ curl http://ftp.de.debian.org/debian/dists/sid/main/source/Sources.xz > $TMPFILE
+ CSVFILE=$(mktemp)
+ (xzcat $TMPFILE | egrep "(^Package:|^Version:)" | sed -s "s#^Version: ##g; s#Package: ##g; s#\n# #g"| while read PKG ; do read VERSION ; echo "$PKG,$VERSION" ; done) > $CSVFILE
+ sqlite3 -csv -init $INIT ${PACKAGES_DB} "DELETE from sources"
+ echo ".import $CSVFILE sources" | sqlite3 -csv -init $INIT ${PACKAGES_DB}
# update amount of available packages (for doing statistics later)
P_IN_SOURCES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | sort -u | wc -l)
sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_stats VALUES (\"sid\", \"${P_IN_SOURCES}\")"
- rm $TMPFILE
+ rm $CSVFILE # $TMPFILE is still being used
+}
+
+set +x
+if [ $1 = "unknown" ] ; then
+ update_sources_table
+ AMOUNT=$2
+ REAL_AMOUNT=0
+ GUESSES=$(echo "${AMOUNT}*3" | bc)
+ PACKAGES=""
+ CANDIDATES=$(xzcat $TMPFILE | grep "^Package" | grep -v "^Package-List:" | cut -d " " -f2 | egrep -v "^(linux|cups|zurl)$" | sort -R | head -$GUESSES | xargs echo)
+ for PKG in $CANDIDATES ; do
+ if [ $REAL_AMOUNT -eq $AMOUNT ] ; then
+ continue
+ fi
+ RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"")
+ if [ "$RESULT" = "" ] ; then
+ PACKAGES="${PACKAGES} $PKG"
+ fi
+ done
+elif [ $1 = "known" ] ; then
+ update_sources_table
+ AMOUNT=$2
+ PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT DISTINCT source_packages.name FROM source_packages,sources WHERE sources.version IN (SELECT version FROM sources WHERE name=source_packages.name ORDER by sources.version DESC LIMIT 1) AND (( source_packages.status = 'unreproducible' OR source_packages.status = 'FTBFS') AND source_packages.name = sources.name AND source_packages.version < sources.version) ORDER BY source_packages.build_date LIMIT $AMOUNT" | xargs -r echo)
else
# CANDIDATES is defined in that file
. /srv/jenkins/bin/reproducible-candidates.sh
PACKAGES=""
- AMOUNT="10"
+ AMOUNT=$2
REAL_AMOUNT=0
for i in $(seq 0 ${#CANDIDATES[@]}) ; do
if [ $REAL_AMOUNT -eq $AMOUNT ] ; then
@@ -82,13 +74,16 @@ else
let "REAL_AMOUNT=REAL_AMOUNT+1"
fi
done
- AMOUNT=$REAL_AMOUNT
-
fi
+AMOUNT=0
+for PKG in $PACKAGES ; do
+ let "AMOUNT=AMOUNT+1"
+done
echo "============================================================================="
echo "The following $AMOUNT source packages will be build: ${PACKAGES}"
echo "============================================================================="
echo
+rm -f $TMPFILE
cleanup_all() {
rm -r $TMPDIR
diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml
index 4a436b6d..8d390463 100644
--- a/job-cfg/reproducible.yaml
+++ b/job-cfg/reproducible.yaml
@@ -33,7 +33,7 @@
artifactDaysToKeep: -1
artifactNumToKeep: -1
builders:
- - shell: '{my_shell} {my_param}'
+ - shell: '{my_shell} {my_params}'
publishers:
- email:
recipients: 'jenkins+debian-reproducible holger@layer-acht.org'
@@ -140,25 +140,25 @@
my_description: 'Setup pbuilder for reproducible builds as described in https://wiki.debian.org/ReproducibleBuilds#Usage_example'
my_timed: '42 0 * * *'
my_shell: '/srv/jenkins/bin/reproducible_setup.sh'
- my_param: ''
+ my_params: ''
- '{name}_stats':
my_description: 'Reproducible builds statistics'
my_timed: '0 0 * * *'
my_shell: '/srv/jenkins/bin/reproducible_stats.sh'
- my_param: ''
+ my_params: ''
- '{name}_build_random_packages':
my_description: 'Reproducible build this many random packages each day:'
my_timed: '0,15,30,45 * * * *'
my_shell: '/srv/jenkins/bin/reproducible_build.sh'
- my_param: '10'
+ my_params: 'unknown 10'
- '{name}_build_random_failures':
- my_description: 'Try to reproducible build 33 packages which failed before and which come from the random job:'
+ my_description: 'Try to reproducible build packages which were build before and for which a new version exists:'
my_timed: '42 2,14 * * *'
my_shell: '/srv/jenkins/bin/reproducible_build.sh'
- my_param: '0'
+ my_params: 'known 25'
- '{name}_build_candidates':
my_description: 'Reproducible build packages from a list of pre-defined candidates: '
my_timed: '40,15,30,45 3,9,15,21 * * *'
my_shell: '/srv/jenkins/bin/reproducible_build.sh'
- my_param: ''
+ my_params: 'candidates 20'