diff options
| -rwxr-xr-x | bin/reproducible_lede.sh | 12 | ||||
| -rwxr-xr-x | bin/reproducible_openwrt.sh | 20 | ||||
| -rw-r--r-- | bin/reproducible_openwrt_common.sh | 123 |
3 files changed, 87 insertions, 68 deletions
diff --git a/bin/reproducible_lede.sh b/bin/reproducible_lede.sh index 05aaa6da..534056c4 100755 --- a/bin/reproducible_lede.sh +++ b/bin/reproducible_lede.sh @@ -42,20 +42,10 @@ START=$(date +'%s') trap cleanup_tmpdirs INT TERM EXIT cd $TMPBUILDDIR -echo "=============================================================================" -echo "$(date -u) - Cloning LEDE git repository." -echo "=============================================================================" -git clone --depth 1 -b $OPENWRT_GIT_BRANCH $OPENWRT_GIT_REPO lede -cd lede -OPENWRT="$(git log -1)" -OPENWRT_VERSION=$(git describe --always) -echo "This is lede $OPENWRT_VERSION." -echo -git log -1 create_results_dirs lede -build_two_times lede ar71xx_generic_ARCHERC7 "CONFIG_TARGET_ar71xx_generic=y\nCONFIG_TARGET_ar71xx_generic_ARCHERC7=y\n" +build_two_times lede ar71xx_generic_ARCHERC7 "CONFIG_TARGET_ar71xx_generic=y\nCONFIG_TARGET_ar71xx_generic_ARCHERC7=y\n" profitbricks-build3-amd64 profitbricks-build4-amd64 # for now we only build one architecture until it's at most reproducible #build_two_times x86_64 "CONFIG_TARGET_x86=y\nCONFIG_TARGET_x86_64=y\n" diff --git a/bin/reproducible_openwrt.sh b/bin/reproducible_openwrt.sh index a5fab82c..d3435900 100755 --- a/bin/reproducible_openwrt.sh +++ b/bin/reproducible_openwrt.sh @@ -42,27 +42,11 @@ START=$(date +'%s') trap cleanup_tmpdirs INT TERM EXIT cd $TMPBUILDDIR -echo "=============================================================================" -echo "$(date -u) - Cloning OpenWrt git repository." -echo "=============================================================================" -git clone --depth 1 -b $OPENWRT_GIT_BRANCH $OPENWRT_GIT_REPO openwrt -cd openwrt -OPENWRT="$(git log -1)" -OPENWRT_VERSION=$(git describe --always) -echo "This is openwrt $OPENWRT_VERSION." -echo -git log -1 - -# External feeds disabled for now as they break building (especially with CONFIG_ALL=y). -#echo "=============================================================================" -#echo "$(date -u) - Updating package feeds." -#echo "=============================================================================" -#./scripts/feeds update -a -#./scripts/feeds install -a create_results_dirs openwrt -build_two_times openwrt ar71xx_generic_ARCHERC7 "CONFIG_TARGET_ar71xx_generic=y\nCONFIG_TARGET_ar71xx_generic_ARCHERC7=y\n" +build_two_times openwrt ar71xx_generic_ARCHERC7 "CONFIG_TARGET_ar71xx_generic=y\nCONFIG_TARGET_ar71xx_generic_ARCHERC7=y\n" profitbricks-build3-amd64 profitbricks-build4-amd64 + # for now we only build one architecture until it's at most reproducible #build_two_times x86_64 "CONFIG_TARGET_x86=y\nCONFIG_TARGET_x86_64=y\n" diff --git a/bin/reproducible_openwrt_common.sh b/bin/reproducible_openwrt_common.sh index 1fc84302..b3f4e314 100644 --- a/bin/reproducible_openwrt_common.sh +++ b/bin/reproducible_openwrt_common.sh @@ -5,6 +5,15 @@ # 2016 Alexander Couzens <lynxis@fe80.eu> # released under the GPLv=2 +# only called direct on a remote build node +openwrt_cleanup_tmpdirs() { + export TMPDIR=$1 + export TMPBUILDDIR=$TMPDIR/build + cleanup_tmpdirs +} + +# called as trap handler +# called on cleanup cleanup_tmpdirs() { cd # (very simple) check we are deleting the right stuff @@ -23,13 +32,11 @@ create_results_dirs() { mkdir -p $BASE/$project/dbd } -# project = openwrt or lede -# postfix = we use the postfix to save difference of the first and second build save_logs() { - local project="$1" - local postfix="$2" + local TYPE=$1 + local RUN=$2 - tar cJf "$BASE/${project}/dbd/logs_${postfix}.tar.xz" logs/ + tar cJf "$TMPDIR/$RUN/logs_${TYPE}.tar.xz" logs/ } # RUN - is b1 or b2. b1 for first run, b2 for second @@ -173,6 +180,13 @@ openwrt_compile() { ionice -c 3 $MAKE $OPTIONS package/index || true # don't let some packages fail the whole build } +openwrt_get_banner() { + TMPDIR=$1 + TYPE=$2 + cd $TMPDIR/build/$TYPE + cat $(find build_dir/ -name banner | grep etc/banner|head -1| xargs cat /dev/null) +} + openwrt_cleanup() { rm build_dir/target-* -rf rm staging_dir/target-* -rf @@ -180,54 +194,85 @@ openwrt_cleanup() { rm logs/* -rf } -# TARGET a target including subtarget. E.g. ar71xx_generic -# CONFIG - a simple basic .config as string. Use \n to seperate lines +# openwrt_build is run on a remote host # TYPE - openwrt or lede +# RUN - b1 or b2. b1 means first run, b2 second +# TARGET - a target including subtarget. E.g. ar71xx_generic +# CONFIG - a simple basic .config as string. Use \n to seperate lines +# TMPPATH - is a unique path generated with mktmp # lede has a different output directory than openwrt -build_two_times() { - TYPE=$1 - TARGET=$2 - CONFIG=$3 +openwrt_build() { + local TYPE=$1 + local RUN=$2 + local TARGET=$3 + local CONFIG=$4 + export TMPDIR=$5 + export TMPBUILDDIR=$TMPDIR/build/ + mkdir -p $TMPBUILDDIR - openwrt_config $CONFIG - openwrt_build_toolchain + # we have also to set the TMP - # FIRST BUILD - openwrt_apply_variations b1 - openwrt_compile "$TYPE" b1 "$TARGET" + cd $TMPBUILDDIR - # get banner - cat $(find build_dir/ -name banner | grep etc/banner|head -1) > $BANNER_HTML + # checkout the repo + echo "=============================================================================" + echo "$(date -u) - Cloning $TYPE git repository." + echo "=============================================================================" + git clone --depth 1 -b $OPENWRT_GIT_BRANCH $OPENWRT_GIT_REPO $TYPE + cd $TYPE - # save results in b1 - [ "$TYPE" = "lede" ] && save_lede_results b1 - [ "$TYPE" = "openwrt" ] && save_openwrt_results b1 + # set tz, date, core, .. + openwrt_apply_variations $RUN + + # configure openwrt + openwrt_config $CONFIG + openwrt_build_toolchain + # build images and packages + openwrt_compile $TYPE $RUN $TARGET + + # save the results + [ "$TYPE" = "lede" ] && save_lede_results $RUN + [ "$TYPE" = "openwrt" ] && save_openwrt_results $RUN # copy logs - save_logs $TYPE b1 + save_logs $TYPE $RUN # clean up between builds openwrt_cleanup +} - # SECOND BUILD - openwrt_apply_variations b2 - openwrt_compile "$TYPE" b2 "$TARGET" +# build openwrt/lede on two different hosts +# TARGET a target including subtarget. E.g. ar71xx_generic +# CONFIG - a simple basic .config as string. Use \n to seperate lines +# TYPE - openwrt or lede +# lede has a different output directory than openwrt +build_two_times() { + TYPE=$1 + TARGET=$2 + CONFIG=$3 + HOST_B1=$4 + HOST_B2=$5 - # save results in b2 - [ "$TYPE" = "lede" ] && save_lede_results b2 - [ "$TYPE" = "openwrt" ] && save_openwrt_results b2 + ## HOST_B1 + RUN=b1 + TMPDIR_B1=$(ssh $HOST_B1 mktemp --tmpdir=/srv/workspace/chroots/ -d -t rbuild-lede-build-XXXXXXXX) + # TODO check tmpdir exist - # copy logs - save_logs $TYPE b2 + SCRIPT="$0 slave" + ssh $HOST_B1 $SCRIPT openwrt_build $TYPE $RUN $TARGET $CONFIG $TMPDIR_B1 - # reset environment to default values again - export LANG="en_GB.UTF-8" - unset LC_ALL - export TZ="/usr/share/zoneinfo/UTC" - export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:" - umask 0022 + # rsync back + # copy logs and images + rsync -a $HOST_B1:$TMPDIR_B1/$RUN/ $TMPDIR/$RUN/ - # clean up again - openwrt_cleanup -} + ssh $HOST_B1 $SCRIPT openwrt_get_banner $TMPDIR_B1 $TYPE > $BANNER_HTML + ssh $HOST_B1 $SCRIPT openwrt_cleanup_tmpdirs $TMPDIR_B1 + ## HOST_B2 + RUN=b2 + TMPDIR_B2=$(ssh $HOST_A mktemp --tmpdir=/srv/workspace/chroots/ -d -t rbuild-lede-build-XXXXXXXX) + ssh $HOST_B2 $SCRIPT openwrt_build $TYPE $RUN $TARGET $CONFIG $TMPDIR_B2 + + rsync -a $HOST_B2:$TMPDIR_B2/$RUN/ $TMPDIR/$RUN/ + ssh $HOST_B2 $SCRIPT openwrt_cleanup_tmpdirs $TMPDIR_B2 +} |