diff options
| -rw-r--r-- | TODO | 1 | ||||
| -rwxr-xr-x | bin/reproducible_setup_schroot.sh | 158 | ||||
| -rwxr-xr-x | bin/schroot-create.sh | 43 | ||||
| -rw-r--r-- | job-cfg/reproducible.yaml | 4 |
4 files changed, 36 insertions, 170 deletions
@@ -181,7 +181,6 @@ egrep -R -l '(debbindiff had trouble comparing|maybe there is still )' /var/lib/ * misc ** more graphs: graph average build duration by day -** meld bin/reproducible_setup_schroot.sh into bin/schroot-create.sh and alias the former to the latter ** reproducible_create_meta_pkg_sets uses schroot created by dpkg_setup_schroot_jessie job (outside of reproducible job space...) ** "fork" etc/schroot/default into etc/schroot/reproducible ** repo-comparison: check for binaries without source diff --git a/bin/reproducible_setup_schroot.sh b/bin/reproducible_setup_schroot.sh deleted file mode 100755 index a6a04c4b..00000000 --- a/bin/reproducible_setup_schroot.sh +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash - -# Copyright 2012-2015 Holger Levsen <holger@layer-acht.org> -# Copyright 2013 Antonio Terceiro <terceiro@debian.org> -# Copyright 2014 Joachim Breitner <nomeata@debian.org> -# released under the GPLv=2 - -DEBUG=false -. /srv/jenkins/bin/common-functions.sh -common_init "$@" - -# common code defining db access -. /srv/jenkins/bin/reproducible_common.sh - -# bootstraps a new chroot for schroot, and then moves it into the right location - -# $1 = schroot name -# $2 = base distro/suite -# $3 $4 ... = extra packages to install - -if [ $# -lt 2 ]; then - echo "usage: $0 TARGET SUITE [backports] CMD [ARG1 ARG2 ...]" - exit 1 -fi -TARGET="$1" -shift -SUITE="$1" -shift - -if [ ! -d "$CHROOT_BASE" ]; then - echo "Directory $CHROOT_BASE does not exist, aborting." - exit 1 -fi - -export CHROOT_TARGET=$(mktemp -d -p $CHROOT_BASE/ schroot-install-$TARGET-XXXX) -if [ -z "$CHROOT_TARGET" ]; then - echo "Could not create a directory to create the chroot in, aborting." - exit 1 -fi - -bootstrap() { - mkdir -p "$CHROOT_TARGET/etc/dpkg/dpkg.cfg.d" - echo force-unsafe-io > "$CHROOT_TARGET/etc/dpkg/dpkg.cfg.d/02dpkg-unsafe-io" - - echo "Bootstraping $SUITE into $CHROOT_TARGET now." - sudo debootstrap $SUITE $CHROOT_TARGET $MIRROR - - echo -e '#!/bin/sh\nexit 101' | sudo tee $CHROOT_TARGET/usr/sbin/policy-rc.d >/dev/null - sudo chmod +x $CHROOT_TARGET/usr/sbin/policy-rc.d - if [ ! -z "$http_proxy" ] ; then - echo "Acquire::http::Proxy \"$http_proxy\";" | sudo tee $CHROOT_TARGET/etc/apt/apt.conf.d/80proxy >/dev/null - fi - echo "deb-src $MIRROR $SUITE main" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list > /dev/null - - # things break without /proc - sudo mount --bind /proc $CHROOT_TARGET/proc - - sudo chroot $CHROOT_TARGET apt-get update - if [ -n "$1" ] ; then - set -x - sudo chroot $CHROOT_TARGET apt-get update - # install debbindiff with all recommends... - sudo chroot $CHROOT_TARGET apt-get install -y "$@" - if ! $DEBUG ; then set +x ; fi - else - # - # FIXME: this else block is unused - remove or merge everything else with schroot-create? - # - # schroot is used to download sources, so add our repo too - echo 'deb-src http://reproducible.alioth.debian.org/debian/ ./' > /etc/apt/sources.list.d/reproducible.list - TMPFILE=$(mktemp) - cat >> $TMPFILE <<- EOF -echo "-----BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1.4.12 (GNU/Linux) - -mQINBFQsy/gBEADKGF55qQpXxpTn7E0Vvqho82/HFB/yT9N2wD8TkrejhJ1I6hfJ -zFXD9fSi8WnNpLc6IjcaepuvvO4cpIQ8620lIuONQZU84sof8nAO0LDoMp/QdN3j -VViXRXQtoUmTAzlOBNpyb8UctAoSzPVgO3jU1Ngr1LWi36hQPvQWSYPNmbsDkGVE -unB0p8DCN88Yq4z2lDdlHgFIy0IDNixuRp/vBouuvKnpe9zyOkijV83Een0XSUsZ -jmoksFzLzjChlS5fAL3FjtLO5XJGng46dibySWwYx2ragsrNUUSkqTTmU7bOVu9a -zlnQNGR09kJRM77UoET5iSXXroK7xQ26UJkhorW2lXE5nQ97QqX7igWp2u0G74RB -e6y3JqH9W8nV+BHuaCVmW0/j+V/l7T3XGAcbjZw1A4w5kj8YGzv3BpztXxqyHQsy -piewXLTBn8dvgDqd1DLXI5gGxC3KGGZbC7v0rQlu2N6OWg2QRbcVKqlE5HeZxmGV -vwGQs/vcChc3BuxJegw/bnP+y0Ys5tsVLw+kkxM5wbpqhWw+hgOlGHKpJLNpmBxn -T+o84iUWTzpvHgHiw6ShJK50AxSbNzDWdbo7p6e0EPHG4Gj41bwO4zVzmQrFz//D -txVBvoATTZYMLF5owdCO+rO6s/xuC3s04pk7GpmDmi/G51oiz7hIhxJyhQARAQAB -tC5EZWJpYW4gUmVwcm9kdWNpYmxlIEJ1aWxkcyBBcmNoaXZlIFNpZ25pbmcgS2V5 -iQI9BBMBCAAnBQJULMv4AhsDBQkFo5qABQsJCAcDBRUKCQgLBRYDAgEAAh4BAheA -AAoJEF23ymfqWaMfFsMP/3jthq65H9avuM469jHcugcd0C5b7/DS+cGQ5E4NQIGL -6tGsqv5k6Nb0MoMMEAQSmWeXRkbYYxmEkrREMNg8tPlh4NiJimH3neNfI+8fGiHY -89FH7QDrrzGfMF9oJQ9zjWZTOs3bjJ4AfS4fkQiQ6UfO7TeMyz5Cw7kz+rS1m1tu -+RgHxD+6A+XxkIZnw5we1MH0SAFoq4j3boR8QkFUNMZsy97xWYON4QLpYwKCbiwL -Q4y06YTw4A7lp+B2JKLc70yRcjbixeAFlZfbhmGITTNAl3j8+48hRLLkJ+s8eT1r -DS1UkYi2xBSNa6TVtNxbDUwVTzzxDe+b8tW2BfC7TBOX2oq6e6ebRa+ghZFVLNY1 -3y+FilXGNMB7IvZ378idHYTNaiJuYXNkrd8UGunwK4NCWdZk05L9GdKeQ6DN380Y -L4QkKpINXSKjneWV7IITMFhvRZCgOVAmoHaq6kaGsl/FwHBA9I8hHXuSyvke8UMP -dmvR8ggv5wiY9NDjW55h7M+UIqEaoXws1algIKB/TWm4/RnQcrxoXBX16wyidzcv -Mb0BawlXZui0MNUSnZtxHMxrjejdvZdqtskHl9srB1QThH0jasmUqbQPxCnxMbf1 -4LhIp6XlXJFF1btgfCexNmcPuqeOMMDQ+du6Hqj2Yl5GYo2McWvjpSgkt5VmQfIz -=X8YA ------END PGP PUBLIC KEY BLOCK-----" -EOF - cat $TMPFILE | sudo chroot $CHROOT_TARGET apt-key add - - rm $TMPFILE - sudo chroot $CHROOT_TARGET apt-get update - fi - # use debbindiff from unstable - if [ "$SUITE" = "testing" ] ; then - echo "deb $MIRROR unstable main" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list > /dev/null - sudo chroot $CHROOT_TARGET apt-get update - # install debbindiff from unstable without re-adding all recommends... - sudo chroot $CHROOT_TARGET apt-get install -y -t unstable --no-install-recommends debbindiff - fi - echo - sudo chroot $CHROOT_TARGET dpkg -l debbindiff - echo - - sudo umount -l $CHROOT_TARGET/proc -} - -cleanup() { - if [ -d $CHROOT_TARGET ]; then - sudo umount -l $CHROOT_TARGET/proc || true - sudo rm -rf --one-file-system $CHROOT_TARGET || fuser -mv $CHROOT_TARGET - fi -} -trap cleanup INT TERM EXIT -bootstrap $@ - -trap - INT TERM EXIT - -# pivot the new schroot in place -rand=$RANDOM -if [ -d $SCHROOT_BASE/"$TARGET" ] -then - sudo mv $SCHROOT_BASE/"$TARGET" $SCHROOT_BASE/"$TARGET"-"$rand" -fi - -sudo mv $CHROOT_TARGET $SCHROOT_BASE/"$TARGET" - -if [ -d $SCHROOT_BASE/"$TARGET"-"$rand" ] -then - sudo rm -rf --one-file-system $SCHROOT_BASE/"$TARGET"-"$rand" || ( echo "Warning: $SCHROOT_BASE/${TARGET}-$rand could not be fully removed." ; ls $SCHROOT_BASE/${TARGET}-$rand -la ) -fi - -# write the schroot config -echo "Writing configuration" -sudo tee /etc/schroot/chroot.d/jenkins-"$TARGET" <<-__END__ - [jenkins-$TARGET] - description=Jenkins schroot $TARGET - directory=$SCHROOT_BASE/$TARGET - type=directory - root-users=jenkins - source-root-users=jenkins - union-type=aufs - __END__ - -trap - INT TERM EXIT diff --git a/bin/schroot-create.sh b/bin/schroot-create.sh index 2293dff5..8607c182 100755 --- a/bin/schroot-create.sh +++ b/bin/schroot-create.sh @@ -3,6 +3,7 @@ # Copyright 2012-2015 Holger Levsen <holger@layer-acht.org> # Copyright 2013 Antonio Terceiro <terceiro@debian.org> # Copyright 2014 Joachim Breitner <nomeata@debian.org> +# Copyright 2015 MAttia Rizzolo <mattia@mapreri.org> # released under the GPLv=2 DEBUG=false @@ -12,13 +13,14 @@ common_init "$@" # bootstraps a new chroot for schroot, and then moves it into the right location # $1 = schroot name -# $2 = base suite +# $2 = base distro/suite # $3 $4 ... = extra packages to install if [ $# -lt 2 ]; then - echo "usage: $0 TARGET SUITE [backports] CMD [ARG1 ARG2 ...]" + echo "usage: $0 TARGET SUITE [backports] [reproducible] [ARG1 ARG2 ...]" exit 1 fi + TARGET="$1" shift SUITE="$1" @@ -36,9 +38,13 @@ if [ "$1" = "backports" ] ; then EXTRA_SOURCES[2]="deb $MIRROR ${SUITE}-backports main" EXTRA_SOURCES[3]="deb-src $MIRROR ${SUITE}-backports main" shift -elif [ "$1" = "reproducible" ] ; then +fi + +if [ "$1" = "reproducible" ] ; then EXTRA_SOURCES[4]="deb http://reproducible.alioth.debian.org/debian/ ./" EXTRA_SOURCES[5]="deb-src http://reproducible.alioth.debian.org/debian/ ./" + REPRODUCIBLE=true + shift fi if [ ! -d "$CHROOT_BASE" ]; then @@ -106,20 +112,40 @@ bootstrap() { [ -z "${EXTRA_SOURCES[$i]}" ] || echo "${EXTRA_SOURCES[$i]}" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list >/dev/null done - if [ "$1" = "reproducible" ] ; then + if $REPRODUCIBLE ; then TMPFILE=$(mktemp -u) add_repokey $CHROOT_TARGET/$TMPFILE sudo chroot $CHROOT_TARGET bash $TMPFILE rm $CHROOT_TARGET/$TMPFILE - shift fi + sudo chroot $CHROOT_TARGET apt-get update if [ -n "$1" ] ; then for d in proc dev dev/pts ; do sudo mount --bind /$d $CHROOT_TARGET/$d done + set -x + sudo chroot $CHROOT_TARGET apt-get update + # install debbindiff with all recommends... + if [ "$1" = "debbindiff" ] ; then + sudo chroot $CHROOT_TARGET apt-get install -y --install-recommends debbindiff + fi sudo chroot $CHROOT_TARGET apt-get install -y --no-install-recommends "$@" sudo + # always use debbindiff from unstable + if [ "$SUITE" = "testing" ] && [ "$1" = "debbindiff" ] ; then + echo "deb $MIRROR unstable main" | sudo tee -a $CHROOT_TARGET/etc/apt/sources.list > /dev/null + sudo chroot $CHROOT_TARGET apt-get update + # install debbindiff from unstable without re-adding all recommends... + sudo chroot $CHROOT_TARGET apt-get install -y -t unstable --no-install-recommends debbindiff + fi + if ! $DEBUG ; then set +x ; fi + # double check debbindiff version + if [ "$1" = "debbindiff" ] ; then + echo + sudo chroot $CHROOT_TARGET dpkg -l debbindiff + echo + fi # umount in reverse order for d in dev/pts dev proc ; do sudo umount -l $CHROOT_TARGET/$d @@ -133,7 +159,7 @@ bootstrap() { cleanup() { if [ -d $CHROOT_TARGET ]; then - sudo rm -rf --one-file-system $CHROOT_TARGET || fuser -mv $CHROOT_TARGET + sudo rm -rf --one-file-system $CHROOT_TARGET || ( echo "Warning: $CHROOT_TARGET could not be fully removed." ; fuser -mv $CHROOT_TARGET ; ls $CHROOT_TARGET -la ) fi } trap cleanup INT TERM EXIT @@ -150,9 +176,8 @@ fi sudo mv $CHROOT_TARGET $SCHROOT_BASE/"$TARGET" -if [ -d $SCHROOT_BASE/"$TARGET"-"$rand" ] -then - sudo rm -rf --one-file-system $SCHROOT_BASE/"$TARGET"-"$rand" +if [ -d $SCHROOT_BASE/"$TARGET"-"$rand" ] ; then + sudo rm -rf --one-file-system $SCHROOT_BASE/"$TARGET"-"$rand" || ( echo "Warning: $SCHROOT_BASE/${TARGET}-$rand could not be fully removed." ; fuser -mv $SCHROOT_BASE/${TARGET}-$rand ; ls $SCHROOT_BASE/${TARGET}-$rand -la ) fi # write the schroot config diff --git a/job-cfg/reproducible.yaml b/job-cfg/reproducible.yaml index 5c4f875b..1187724d 100644 --- a/job-cfg/reproducible.yaml +++ b/job-cfg/reproducible.yaml @@ -292,12 +292,12 @@ - '{name}_setup_schroot_unstable_debbindiff': my_description: 'Setup schroot for running debbindiff in a unstable environment (this is needed to be able to correctly investigate haskell binaries...)' my_timed: '23 0 * * *' - my_shell: '/srv/jenkins/bin/reproducible_setup_schroot.sh reproducible-unstable-debbindiff unstable debbindiff locales-all' + my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible-unstable-debbindiff unstable debbindiff locales-all' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' - '{name}_setup_schroot_testing_debbindiff': my_description: 'Setup schroot for running debbindiff in a testing environment (this is needed to be able to correctly investigate haskell binaries...)' my_timed: '23 0 * * *' - my_shell: '/srv/jenkins/bin/reproducible_setup_schroot.sh reproducible-testing-debbindiff testing debbindiff locales-all' + my_shell: '/srv/jenkins/bin/schroot-create.sh reproducible-testing-debbindiff testing debbindiff locales-all' my_recipients: 'qa-jenkins-scm@lists.alioth.debian.org' - '{name}_setup_schroot_unstable': my_description: 'Setup unstable schroot for fetching source packages for the builder jobs.' |