blob: 4ec211a065776c6c43b41f8bc7421b7a213f8eb2 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
|
#!/bin/bash
# Copyright 2014-2016 Holger Levsen <holger@layer-acht.org>
# released under the GPLv=2
DEBUG=false
. /srv/jenkins/bin/common-functions.sh
common_init "$@"
# common code defining db access
. /srv/jenkins/bin/reproducible_common.sh
set -e
# build for these architectures
MACHINES="sparc64 amd64"
cleanup_tmpdirs() {
cd
rm -r $TMPDIR
rm -r $TMPBUILDDIR
}
create_results_dirs() {
mkdir -p $BASE/netbsd/dbd
}
save_netbsd_results() {
local RUN=$1
local MACHINE=$2
mkdir -p $TMPDIR/$RUN/${MACHINE}
cp -pr obj/releasedir/${MACHINE} $TMPDIR/$RUN/
find $TMPDIR/$RUN/ -name MD5 -o -name SHA512 | xargs -r rm -v
}
#
# main
#
TMPBUILDDIR=$(mktemp --tmpdir=/srv/workspace/chroots/ -d -t rbuild-netbsd-build-XXXXXXXX) # used to build on tmpfs
TMPDIR=$(mktemp --tmpdir=/srv/reproducible-results -d -t rbuild-netbsd-results-XXXXXXXX) # accessable in schroots, used to compare results
DATE=$(date -u +'%Y-%m-%d')
START=$(date +'%s')
trap cleanup_tmpdirs INT TERM EXIT
cd $TMPBUILDDIR
echo "============================================================================="
echo "$(date -u) - Cloning the NetBSD git repository (which is synced with the NetBSD CVS repository)"
echo "============================================================================="
git clone --depth 1 https://github.com/jsonn/src
mv src netbsd
cd netbsd
NETBSD="$(git log -1)"
NETBSD_VERSION=$(git describe --always)
echo "This is netbsd $NETBSD_VERSION."
echo
git log -1
# from $src/share/mk/bsd.README:
# MKREPRO If "yes", create reproducable builds. This enables
# different switches to make two builds from the same source tree
# result in the same build results.
# also see http://man.netbsd.org/HEAD/usr/share/man/html5/mk.conf.html
export MKREPRO="yes"
# MKREPRO_TIMESTAMP is set to SOURCE_DATE_EPOCH of netbsd.git
# see http://cvsweb.netbsd.org/bsdweb.cgi/~checkout~/src/BUILDING
SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
MKREPRO_TIMESTAMP=${SOURCE_DATE_EPOCH}
echo "============================================================================="
echo "$(date -u) - Building NetBSD ${NETBSD_VERSION} - first build run."
echo "============================================================================="
export TZ="/usr/share/zoneinfo/Etc/GMT+12"
# actually build everything
for MACHINE in $MACHINES ; do
ionice -c 3 \
./build.sh -j $NUM_CPU -V MKREPRO_TIMESTAMP=$MKREPRO_TIMESTAMP -U -u -m ${MACHINE} release
# save results in b1
save_netbsd_results b1 ${MACHINE}
# cleanup and explicitly delete old tooldir to force re-creation for the next $MACHINE type
./build.sh -U -m ${MACHINE} cleandir
rm obj/tooldir.* -rf
echo "${MACHINE} done, first time."
done
echo "============================================================================="
echo "$(date -u) - Building NetBSD ${NETBSD_VERSION} - cleaning up between builds."
echo "============================================================================="
rm obj/releasedir -r
rm obj/destdir.* -r
# we keep the toolchain(s)
echo "============================================================================="
echo "$(date -u) - Building NetBSD - second build run."
echo "============================================================================="
export TZ="/usr/share/zoneinfo/Etc/GMT-14"
export LANG="fr_CH.UTF-8"
export LC_ALL="fr_CH.UTF-8"
export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/i/capture/the/path"
export CAPTURE_ENVIRONMENT="I capture the environment"
umask 0002
# use allmost all cores for second build
NEW_NUM_CPU=$(echo $NUM_CPU-1|bc)
for MACHINE in $MACHINES ; do
ionice -c 3 \
linux64 --uname-2.6 \
./build.sh -j $NEW_NUM_CPU -V MKREPRO_TIMESTAMP=$MKREPRO_TIMESTAMP -U -u -m ${MACHINE} release
# save results in b2
save_netbsd_results b2 ${MACHINE}
# cleanup and explicitly delete old tooldir to force re-creation for the next $MACHINE type
./build.sh -U -m ${MACHINE} cleandir
rm obj/tooldir.* -r
echo "${MACHINE} done, second time."
done
# reset environment to default values again
export LANG="en_GB.UTF-8"
unset LC_ALL
export TZ="/usr/share/zoneinfo/UTC"
export PATH="/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:"
umask 0022
# clean up builddir to save space on tmpfs
rm -r $TMPBUILDDIR/netbsd
# temporarily enable more debug
set -x
# run diffoscope on the results
TIMEOUT="30m"
DIFFOSCOPE="$(schroot --directory /tmp -c source:jenkins-reproducible-${DBDSUITE}-diffoscope diffoscope -- --version 2>&1)"
echo "============================================================================="
echo "$(date -u) - Running $DIFFOSCOPE on NetBSD build results..."
echo "============================================================================="
FILES_HTML=$(mktemp --tmpdir=$TMPDIR)
GOOD_FILES_HTML=$(mktemp --tmpdir=$TMPDIR)
BAD_FILES_HTML=$(mktemp --tmpdir=$TMPDIR)
GOOD_SECTION_HTML=$(mktemp --tmpdir=$TMPDIR)
BAD_SECTION_HTML=$(mktemp --tmpdir=$TMPDIR)
GOOD_FILES=0
ALL_FILES=0
SIZE=""
create_results_dirs
cd $TMPDIR/b1
tree .
for i in * ; do
cd $i
for j in $(find * -type f |sort -u ) ; do
let ALL_FILES+=1
call_diffoscope $i $j
get_filesize $j
if [ -f $TMPDIR/$i/$j.html ] ; then
mkdir -p $BASE/netbsd/dbd/$i/$(dirname $j)
mv $TMPDIR/$i/$j.html $BASE/netbsd/dbd/$i/$j.html
echo " <tr><td><a href=\"dbd/$i/$j.html\"><img src=\"/userContent/static/weather-showers-scattered.png\" alt=\"unreproducible icon\" /> $j</a> ($SIZE) is unreproducible.</td></tr>" >> $BAD_FILES_HTML
else
SHASUM=$(sha256sum $j|cut -d " " -f1)
echo " <tr><td><img src=\"/userContent/static/weather-clear.png\" alt=\"reproducible icon\" /> $j ($SHASUM, $SIZE) is reproducible.</td></tr>" >> $GOOD_FILES_HTML
let GOOD_FILES+=1
rm -f $BASE/netbsd/dbd/$i/$j.html # cleanup from previous (unreproducible) tests - if needed
fi
# for debugging #842250
if [ "$(basename $TMPDIR/b1/$i/$j)" = "base.tgz" ] ; then
for k in b1 b2 ; do
mkdir -p $BASE/netbsd/artifacts/$k/$i/$(dirname $j)
cp $TMPDIR/$k/$i/$j $BASE/netbsd/artifacts/$k/$i/$j
done
echo "artifacts published for debugging #842250 in https://tests.reproducible-builds.org/netbsd/artifacts/ - please dont forget to delete this directory later…"
fi
done
cd ..
if [ -s $GOOD_FILES_HTML ] ; then
echo " <table><tr><th>Reproducible artifacts for <code>$i</code></th></tr>" >> $GOOD_SECTION_HTML
cat $GOOD_FILES_HTML >> $GOOD_SECTION_HTML
echo " </table>" >> $GOOD_SECTION_HTML
rm $GOOD_FILES_HTML
fi
if [ -s $BAD_FILES_HTML ] ; then
echo " <table><tr><th>Unreproducible artifacts for <code>$i</code></th></tr>" >> $BAD_SECTION_HTML
cat $BAD_FILES_HTML >> $BAD_SECTION_HTML
echo " </table>" >> $BAD_SECTION_HTML
rm $BAD_FILES_HTML
fi
done
GOOD_PERCENT=$(echo "scale=1 ; ($GOOD_FILES*100/$ALL_FILES)" | bc)
# are we there yet?
if [ "$GOOD_PERCENT" = "100.0" ] ; then
MAGIC_SIGN="!"
else
MAGIC_SIGN="?"
fi
#
# finally create the webpage
#
cd $TMPDIR ; mkdir netbsd
PAGE=netbsd/netbsd.html
cat > $PAGE <<- EOF
<!DOCTYPE html>
<html lang="en-US">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width">
<title>Reproducible NetBSD $MAGIC_SIGN</title>
<link rel='stylesheet' href='global.css' type='text/css' media='all' />
</head>
<body>
<div id="logo">
<img src="NetBSD-smaller.png" />
<h1>Reproducible NetBSD $MAGIC_SIGN</h1>
</div>
<div class="content">
<div class="page-content">
EOF
write_page_intro NetBSD
write_page " <p>$GOOD_FILES ($GOOD_PERCENT%) out of $ALL_FILES built NetBSD files were reproducible in our test setup"
if [ "$GOOD_PERCENT" = "100.0" ] ; then
write_page "!"
else
write_page "."
fi
write_page " These tests were last run on $DATE for version ${NETBSD_VERSION} with MKREPRO=yes and MKREPRO_TIMESTAMP=$MKREPRO_TIMESTAMP and were compared using ${DIFFOSCOPE}.</p>"
write_variation_table NetBSD
cat $BAD_SECTION_HTML >> $PAGE
cat $GOOD_SECTION_HTML >> $PAGE
write_page " <p><pre>"
echo -n "$NETBSD" >> $PAGE
write_page " </pre></p>"
write_page " </div></div>"
write_page_footer NetBSD
publish_page
rm -f $FILES_HTML $GOOD_FILES_HTML $BAD_FILES_HTML $GOOD_SECTION_HTML $BAD_SECTION_HTML
# the end
calculate_build_duration
print_out_duration
irc_message reproducible-builds "$REPRODUCIBLE_URL/netbsd/ has been updated. ($GOOD_PERCENT% reproducible)"
echo "============================================================================="
# remove everything, we don't need it anymore...
cleanup_tmpdirs
trap - INT TERM EXIT
|