summaryrefslogtreecommitdiffstats
path: root/bin
diff options
context:
space:
mode:
authorHolger Levsen <holger@layer-acht.org>2014-10-13 13:13:35 +0200
committerHolger Levsen <holger@layer-acht.org>2014-10-13 13:17:40 +0200
commit2a4efe5f82496b2e158b58303dbb06f84112f0c4 (patch)
tree109060c891d651ddaf0e3ac7dc9f75423d229c97 /bin
parent4da1f75e2d0b261bc1551f7f4705bc16ab45d6c7 (diff)
downloadjenkins.debian.net-2a4efe5f82496b2e158b58303dbb06f84112f0c4.tar.xz
reproducible: replace builder jobs with scheduler and build jobs
Diffstat (limited to 'bin')
-rwxr-xr-xbin/reproducible_build.sh171
-rwxr-xr-xbin/reproducible_scheduler.sh208
2 files changed, 234 insertions, 145 deletions
diff --git a/bin/reproducible_build.sh b/bin/reproducible_build.sh
index d1c1f5b1..a546214d 100755
--- a/bin/reproducible_build.sh
+++ b/bin/reproducible_build.sh
@@ -24,96 +24,6 @@ fi
# create dirs for results
mkdir -p /var/lib/jenkins/userContent/dbd/ /var/lib/jenkins/userContent/buildinfo/ /var/lib/jenkins/userContent/rbuild/
-# this needs sid entries in sources.list:
-grep deb-src /etc/apt/sources.list | grep sid
-# try apt-get update twice, else fail gracefully, aka not.
-sudo apt-get update || ( sleep $(( $RANDOM % 70 + 30 )) ; sudo apt-get update || true )
-
-# update sources table in db
-update_sources_table() {
- TMPFILE=$(mktemp)
- curl $MIRROR/dists/sid/main/source/Sources.xz > $TMPFILE
- CSVFILE=$(mktemp)
- (xzcat $TMPFILE | egrep "(^Package:|^Version:)" | sed -s "s#^Version: ##g; s#Package: ##g; s#\n# #g"| while read PKG ; do read VERSION ; echo "$PKG,$VERSION" ; done) > $CSVFILE
- sqlite3 -csv -init $INIT ${PACKAGES_DB} "DELETE from sources"
- echo ".import $CSVFILE sources" | sqlite3 -csv -init $INIT ${PACKAGES_DB}
- rm $CSVFILE # $TMPFILE is still being used # FIXME: remove TMPFILE too...
- echo "$(date) Removing duplicate versions from sources db..."
- for PKG in $(sqlite3 ${PACKAGES_DB} 'SELECT name FROM sources GROUP BY name HAVING count(name) > 1') ; do
- BET=""
- for VERSION in $(sqlite3 ${PACKAGES_DB} "SELECT version FROM sources where name = \"$PKG\"") ; do
- if [ "$BET" = "" ] ; then
- BET=$VERSION
- continue
- elif dpkg --compare-versions "$BET" lt "$VERSION" ; then
- BET=$VERSION
- fi
- done
- sqlite3 -init $INIT ${PACKAGES_DB} "DELETE FROM sources WHERE name = '$PKG' AND version != '$BET'"
- done
- echo "$(date) Done removing duplicate versions from sources db..."
- # verify duplicate entries have been removed correctly from the db
- P_IN_TMPFILE=$(xzcat $TMPFILE | grep "^Package:" | cut -d " " -f2 | sort -u | wc -l)
- P_IN_SOURCES=$(sqlite3 ${PACKAGES_DB} 'SELECT count(name) FROM sources')
- if [ $P_IN_TMPFILE -ne $P_IN_SOURCES ] ; then
- echo "DEBUG: P_IN_SOURCES = $P_IN_SOURCES"
- echo "DEBUG: P_IN_TMPFILE = $P_IN_TMPFILE"
- exit 1
- fi
-}
-
-set +x
-if [ $1 = "unknown" ] ; then
- update_sources_table
- AMOUNT=$2
- REAL_AMOUNT=0
- GUESSES=$(echo "${AMOUNT}*3" | bc)
- PACKAGES=""
- # FIXME: blacklisted is a valid status in the db which should be used...
- CANDIDATES=$(xzcat $TMPFILE | grep "^Package:" | cut -d " " -f2 | egrep -v "^(linux|cups|zurl|openclipart|eigen3|xmds2)$" | sort -R | head -$GUESSES | xargs echo)
- for PKG in $CANDIDATES ; do
- if [ $REAL_AMOUNT -eq $AMOUNT ] ; then
- continue
- fi
- RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"")
- if [ "$RESULT" = "" ] ; then
- PACKAGES="${PACKAGES} $PKG"
- fi
- done
-elif [ $1 = "known" ] ; then
- update_sources_table
- AMOUNT=$2
- # FIXME: blacklisted is a valid status in the db which should be used...
- PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT DISTINCT source_packages.name FROM source_packages,sources WHERE sources.version IN (SELECT version FROM sources WHERE name=source_packages.name ORDER by sources.version DESC LIMIT 1) AND (( source_packages.status = 'unreproducible' OR source_packages.status = 'FTBFS') AND source_packages.name = sources.name AND source_packages.version != sources.version) ORDER BY source_packages.build_date LIMIT $AMOUNT" | egrep -v "^(linux|cups|zurl|openclipart|eigen3|xmds2)$" | xargs -r echo)
-else
- # CANDIDATES is defined in that file
- . /srv/jenkins/bin/reproducible_candidates.sh
- PACKAGES=""
- AMOUNT=$2
- REAL_AMOUNT=0
- for i in $(seq 0 ${#CANDIDATES[@]}) ; do
- if [ $REAL_AMOUNT -eq $AMOUNT ] ; then
- continue
- fi
- PKG=${CANDIDATES[$i]}
- # FIXME: blacklisted is a valid status in the db which should be used...
- RESULT=$(sqlite3 ${PACKAGES_DB} "SELECT name FROM source_packages WHERE name = \"${PKG}\"")
- if [ "$RESULT" = "" ] ; then
- PACKAGES="${PACKAGES} $PKG"
- let "REAL_AMOUNT=REAL_AMOUNT+1"
- fi
- done
-fi
-AMOUNT=0
-for PKG in $PACKAGES ; do
- let "AMOUNT=AMOUNT+1"
-done
-echo "============================================================================="
-echo "The following $AMOUNT source packages will be build: ${PACKAGES}"
-echo "============================================================================="
-echo
-rm -f $TMPFILE
-
cleanup_all() {
rm -r $TMPDIR
}
@@ -124,57 +34,46 @@ cleanup_userContent() {
rm -f /var/lib/jenkins/userContent/buildinfo/${SRCPACKAGE}_*.buildinfo > /dev/null 2>&1
}
-cleanup_prebuild() {
- rm b1 b2 -rf
- rm -f ${SRCPACKAGE}_* > /dev/null 2>&1
+unschedule_from_db() {
+ # unmark build as properly finished
+ sqlite3 -init $INIT ${PACKAGES_DB} "DELETE FROM sources_scheduled WHERE name = '$SRCPACKAGE';"
}
TMPDIR=$(mktemp --tmpdir=$PWD -d)
-NUM_CPU=$(cat /proc/cpuinfo |grep ^processor|wc -l)
-COUNT_TOTAL=0
-COUNT_GOOD=0
-COUNT_BAD=0
-COUNT_SKIPPED=0
-GOOD=""
-BAD=""
-SOURCELESS=""
-SKIPPED=""
trap cleanup_all INT TERM EXIT
cd $TMPDIR
-for SRCPACKAGE in ${PACKAGES} ; do
+RESULT=$(sqlite3 -init $INIT ${PACKAGES_DB} "SELECT name,date_scheduled FROM sources_scheduled WHERE date_build_started = '' ORDER BY date_scheduled LIMIT 1")
+if [ -z "$RESULT" ] ; then
+ echo "No packages scheduled, sleeping 30m."
+ sleep 30m
+else
set +x
+ SRCPACKAGE=$(echo $RESULT|cut -d "|" -f1)
+ SCHEDULED_DATE=$(echo $RESULT|cut -d "|" -f2)
echo "============================================================================="
echo "Trying to build ${SRCPACKAGE} reproducibly now."
echo "============================================================================="
set -x
- let "COUNT_TOTAL=COUNT_TOTAL+1"
- cleanup_prebuild
- set +e
DATE=$(date +'%Y-%m-%d %H:%M')
- VERSION=$(apt-cache showsrc ${SRCPACKAGE} | grep ^Version | cut -d " " -f2 | sort -r | head -1)
- # check if we tested this version already before...
- STATUS=$(sqlite3 ${PACKAGES_DB} "SELECT status FROM source_packages WHERE name = \"${SRCPACKAGE}\" AND version = \"${VERSION}\"")
- # skip if we know this version and status = reproducible or unreproducible or FTBFS
- if [ "$STATUS" = "reproducible" ] || [ "$STATUS" = "unreproducible" ] || [ "$STATUS" = "FTBFS" ] ; then
- echo "Package ${SRCPACKAGE} (${VERSION}) with status '$STATUS' skipped, no newer version available."
- let "COUNT_SKIPPED=COUNT_SKIPPED+1"
- SKIPPED="${SRCPACKAGE} ${SKIPPED}"
- continue
- fi
+ # mark build attempt
+ sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO sources_scheduled VALUES ('$SRCPACKAGE','$SCHEDULED_DATE','$DATE');"
+
RBUILDLOG=/var/lib/jenkins/userContent/rbuild/${SRCPACKAGE}_None.rbuild.log
echo "Starting to build ${SRCPACKAGE} on $DATE" | tee ${RBUILDLOG}
# host has only sid in deb-src in sources.list
+ set +e
apt-get source --download-only --only-source ${SRCPACKAGE} >> ${RBUILDLOG} 2>&1
RESULT=$?
if [ $RESULT != 0 ] ; then
echo "Warning: Download of ${SRCPACKAGE} sources failed." | tee -a ${RBUILDLOG}
ls -l ${SRCPACKAGE}* | tee -a ${RBUILDLOG}
- SOURCELESS="${SOURCELESS} ${SRCPACKAGE}"
sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"None\", \"404\", \"$DATE\")"
set +x
echo "Warning: Maybe there was a network problem, or ${SRCPACKAGE} is not a source package, or was removed or renamed. Please investigate." | tee -a ${RBUILDLOG}
- continue
+ unschedule_from_db
+ exit 0
else
+ set -e
VERSION=$(grep "^Version: " ${SRCPACKAGE}_*.dsc| grep -v "GnuPG v" | sort -r | head -1 | cut -d " " -f2-)
# EPOCH_FREE_VERSION was too long
EVERSION=$(echo $VERSION | cut -d ":" -f2)
@@ -201,11 +100,11 @@ for SRCPACKAGE in ${PACKAGES} ; do
sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"not for us\", \"$DATE\")"
set +x
echo "Package ${SRCPACKAGE} (${VERSION}) shall only be build on \"${ARCHITECTURES}\" and thus was skipped." | tee -a ${RBUILDLOG}
- let "COUNT_SKIPPED=COUNT_SKIPPED+1"
- SKIPPED="${SRCPACKAGE} ${SKIPPED}"
- dcmd rm ${SRCPACKAGE}_${EVERSION}.dsc
- continue
+ unschedule_from_db
+ exit 0
fi
+ set +e
+ NUM_CPU=$(cat /proc/cpuinfo |grep ^processor|wc -l)
( timeout 15m nice ionice -c 3 sudo DEB_BUILD_OPTIONS="parallel=$NUM_CPU" pbuilder --build --debbuildopts "-b" --basetgz /var/cache/pbuilder/base-reproducible.tgz --distribution sid ${SRCPACKAGE}_*.dsc ) 2>&1 | tee -a ${RBUILDLOG}
if [ -f /var/cache/pbuilder/result/${SRCPACKAGE}_${EVERSION}_amd64.changes ] ; then
mkdir b1 b2
@@ -242,8 +141,7 @@ for SRCPACKAGE in ${PACKAGES} ; do
echo
echo "${SRCPACKAGE} built successfully and reproducibly."
sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"reproducible\", \"$DATE\")"
- let "COUNT_GOOD=COUNT_GOOD+1"
- GOOD="${SRCPACKAGE} ${GOOD}"
+ unschedule_from_db
else
cp b1/${BUILDINFO} /var/lib/jenkins/userContent/buildinfo/ || true
if [ -f ./${LOGFILE} ] ; then
@@ -256,6 +154,7 @@ for SRCPACKAGE in ${PACKAGES} ; do
fi
fi
sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"unreproducible\", \"$DATE\")"
+ unschedule_from_db
set +x
echo -n "${SRCPACKAGE} failed to build reproducibly."
if [ ! -f b1/${BUILDINFO} ] ; then
@@ -263,36 +162,18 @@ for SRCPACKAGE in ${PACKAGES} ; do
else
echo
fi
- let "COUNT_BAD=COUNT_BAD+1"
- BAD="${SRCPACKAGE} ${BAD}"
fi
- set -x
- rm b1 b2 -rf
else
sqlite3 -init $INIT ${PACKAGES_DB} "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"FTBFS\", \"$DATE\")"
+ unschedule_from_db
set +x
echo "${SRCPACKAGE} failed to build from source."
fi
- set -x
- dcmd rm ${SRCPACKAGE}_${EVERSION}.dsc
- rm -f ${SRCPACKAGE}_* > /dev/null 2>&1
fi
- set +x
- echo "============================================================================="
- echo "$COUNT_TOTAL of $AMOUNT done. Previous package: ${SRCPACKAGE}"
- echo "============================================================================="
- set -x
-done
+fi
+set -x
cd ..
cleanup_all
trap - INT TERM EXIT
-set +x
-echo
-echo
-echo "$COUNT_TOTAL packages attempted to build in total."
-echo "$COUNT_GOOD packages successfully built reproducibly: ${GOOD}"
-echo "$COUNT_SKIPPED packages skipped (either because they were successfully built reproducibly in the past or because they are not Architecture: 'any' nor 'all' nor 'amd64'): ${SKIPPED}"
-echo "$COUNT_BAD packages failed to built reproducibly: ${BAD}"
-echo "The following source packages doesn't exist in sid: $SOURCELESS"
diff --git a/bin/reproducible_scheduler.sh b/bin/reproducible_scheduler.sh
new file mode 100755
index 00000000..5cbfaad9
--- /dev/null
+++ b/bin/reproducible_scheduler.sh
@@ -0,0 +1,208 @@
+#!/bin/bash
+
+# Copyright 2014 Holger Levsen <holger@layer-acht.org>
+# released under the GPLv=2
+
+. /srv/jenkins/bin/common-functions.sh
+common_init "$@"
+
+set +x
+
+#
+# define db
+#
+PACKAGES_DB=/var/lib/jenkins/reproducible.db
+INIT=/var/lib/jenkins/reproducible.init
+if [ ! -f $PACKAGES_DB ] ; then
+ echo "$PACKAGES_DB doesn't exist, no builds possible."
+ exit 1
+elif [ -f $PACKAGES_DB.lock ] ; then
+ for i in $(seq 0 100) ; do
+ sleep 15
+ [ -f $PACKAGES_DB.lock ] || break
+ done
+ echo "$PACKAGES_DB.lock still exist, exiting."
+ exit 1
+fi
+
+#
+# functions, see below for main
+#
+update_apt() {
+ # this needs sid entries in sources.list:
+ grep deb-src /etc/apt/sources.list | grep sid
+ # try apt-get update three times, else fail
+ sudo apt-get update || ( sleep $(( $RANDOM % 70 + 30 )) ; sudo apt-get update ) || ( sleep $(( $RANDOM % 70 + 30 )) ; sudo apt-get update || exit 1 )
+}
+
+# update sources table in db
+update_sources_table() {
+ touch ${PACKAGES_DB}.lock
+ TMPFILE=$(mktemp)
+ curl $MIRROR/dists/sid/main/source/Sources.xz > $TMPFILE
+ CSVFILE=$(mktemp)
+ (xzcat $TMPFILE | egrep "(^Package:|^Version:)" | sed -s "s#^Version: ##g; s#Package: ##g; s#\n# #g"| while read PKG ; do read VERSION ; echo "$PKG,$VERSION" ; done) > $CSVFILE
+ sqlite3 -csv -init $INIT ${PACKAGES_DB} "DELETE from sources"
+ echo ".import $CSVFILE sources" | sqlite3 -csv -init $INIT ${PACKAGES_DB}
+ # count unique packages for later comparison
+ P_IN_TMPFILE=$(xzcat $TMPFILE | grep "^Package:" | cut -d " " -f2 | sort -u | wc -l)
+ # cleanup files already
+ rm $CSVFILE $TMPFILE
+ # cleanup db
+ echo "============================================================================="
+ echo "$(date) Removing duplicate versions from sources db..."
+ for PKG in $(sqlite3 ${PACKAGES_DB} 'SELECT name FROM sources GROUP BY name HAVING count(name) > 1') ; do
+ BET=""
+ for VERSION in $(sqlite3 ${PACKAGES_DB} "SELECT version FROM sources where name = \"$PKG\"") ; do
+ if [ "$BET" = "" ] ; then
+ BET=$VERSION
+ continue
+ elif dpkg --compare-versions "$BET" lt "$VERSION" ; then
+ BET=$VERSION
+ fi
+ done
+ sqlite3 -init $INIT ${PACKAGES_DB} "DELETE FROM sources WHERE name = '$PKG' AND version != '$BET'"
+ done
+ echo "$(date) Done removing duplicate versions from sources db..."
+ echo "============================================================================="
+ rm ${PACKAGES_DB}.lock
+ # verify duplicate entries have been removed correctly from the db
+ P_IN_SOURCES=$(sqlite3 ${PACKAGES_DB} 'SELECT count(name) FROM sources')
+ if [ $P_IN_TMPFILE -ne $P_IN_SOURCES ] ; then
+ echo "DEBUG: P_IN_SOURCES = $P_IN_SOURCES"
+ echo "DEBUG: P_IN_TMPFILE = $P_IN_TMPFILE"
+ exit 1
+ fi
+}
+
+do_sql_query() {
+ PACKAGES=$(sqlite3 -init $INIT ${PACKAGES_DB} "$QUERY")
+ if [ ! -z "$PACKAGES" ] ; then
+ AMOUNT=$(echo "$PACKAGES" | wc -l)
+ PACKAGES="$(echo $PACKAGES)"
+ else
+ AMOUNT=0
+ fi
+ echo "Criteria: $1"
+ echo "Amount: $AMOUNT"
+ echo "Packages: $PACKAGES"
+ echo "============================================================================="
+}
+
+select_unknown_packages() {
+ QUERY="
+ SELECT DISTINCT sources.name FROM sources
+ WHERE sources.name NOT IN
+ (SELECT sources.name FROM sources,sources_scheduled
+ WHERE sources.name=sources_scheduled.name)
+ AND sources.name NOT IN
+ (SELECT sources.name FROM sources,source_packages
+ WHERE sources.name=source_packages.name)
+ ORDER BY random()
+ LIMIT $1"
+ do_sql_query "never tested before, randomly sorted"
+}
+
+select_new_versions() {
+ QUERY="
+ SELECT DISTINCT sources.name FROM sources,source_packages
+ WHERE sources.name NOT IN
+ (SELECT sources.name FROM sources,sources_scheduled
+ WHERE sources.name=sources_scheduled.name)
+ AND sources.name IN
+ (SELECT sources.name FROM sources,source_packages
+ WHERE sources.name=source_packages.name
+ AND sources.version!=source_packages.version
+ AND source_packages.status!='blacklisted')
+ AND sources.name=source_packages.name
+ ORDER BY source_packages.build_date
+ LIMIT $1"
+ do_sql_query "tested before, new version available, sorted by last test date"
+}
+
+select_old_versions() {
+ # old versions older than two weeks only
+ QUERY="
+ SELECT DISTINCT sources.name FROM sources,source_packages
+ WHERE sources.name NOT IN
+ (SELECT sources.name FROM sources,sources_scheduled
+ WHERE sources.name=sources_scheduled.name)
+ AND sources.name IN
+ (SELECT sources.name FROM sources,source_packages
+ WHERE sources.name=source_packages.name
+ AND sources.version=source_packages.version
+ AND source_packages.status!='blacklisted')
+ AND sources.name=source_packages.name
+ AND source_packages.build_date < datetime('now', '-4 day')
+ ORDER BY source_packages.build_date
+ LIMIT $1"
+ do_sql_query "tested at least two weeks ago, no new version available, sorted by last test date"
+}
+
+
+schedule_packages() {
+ DATE=$(date +'%Y-%m-%d %H:%M')
+ TMPFILE=$(mktemp)
+ for PKG in $CANDIDATES ; do
+ echo "INSERT INTO sources_scheduled VALUES ('$PKG','$DATE','');" >> $TMPFILE
+ done
+ cat $TMPFILE | sqlite3 -init $INIT ${PACKAGES_DB}
+ rm $TMPFILE
+echo "============================================================================="
+echo "The following $TOTAL source packages have been scheduled: $CANDIDATES"
+echo "============================================================================="
+echo
+}
+
+#
+# main
+#
+update_apt
+SCHEDULED=$(sqlite3 ${PACKAGES_DB} 'SELECT count(name) FROM sources_scheduled')
+if [ $SCHEDULED -gt 250 ] ; then
+ echo "$SCHEDULED packages scheduled, nothing to do."
+ exit 0
+else
+ echo "$SCHEDULED packages currently scheduled, scheduling some more..."
+fi
+update_sources_table
+
+echo "Requesting 200 unknown packages..."
+select_unknown_packages 200
+let "TOTAL=$SCHEDULED+$AMOUNT"
+echo "So in total now $TOTAL packages about to be scheduled."
+CANDIDATES=$PACKAGES
+MESSAGE="Scheduled $AMOUNT unknown packages"
+
+if [ $TOTAL -le 250 ] ; then
+ NEW=50
+elif [ $TOTAL -le 450 ] ; then
+ NEW=25
+fi
+echo "Requesting $NEW new packages..."
+select_new_versions $NEW
+let "TOTAL=$TOTAL+$AMOUNT"
+echo "So in total now $TOTAL packages about to be scheduled."
+CANDIDATES="$CANDIDATES $PACKAGES"
+MESSAGE="$MESSAGE, $AMOUNT packages with new versions"
+
+if [ $TOTAL -lt 250 ] ; then
+ OLD=100
+elif [ $TOTAL -le 50 ] ; then
+ OLD=5
+else
+ OLD=1
+fi
+echo "Requesting $OLD old packages..."
+select_old_versions $OLD
+let "TOTAL=$TOTAL+$AMOUNT"
+echo "So in total now $TOTAL packages about to be scheduled."
+CANDIDATES="$CANDIDATES $PACKAGES"
+MESSAGE="$MESSAGE and $AMOUNT packages with the same version again, for a total of $TOTAL scheduled packages."
+
+# finally
+schedule_packages
+echo
+echo "$MESSAGE"
+kgb-client --conf /srv/jenkins/kgb/debian-reproducible.conf --relay-msg "$MESSAGE"
+echo