1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
|
#!/bin/bash
# Copyright 2014 Holger Levsen <holger@layer-acht.org>
# released under the GPLv=2
. /srv/jenkins/bin/common-functions.sh
common_init "$@"
# fetch git repo for the diffp command used later
if [ -d misc.git ] ; then
cd misc.git
git pull
cd ..
else
git clone git://git.debian.org/git/reproducible/misc.git misc.git
fi
# create sqlite db
PACKAGES_DB=/var/lib/jenkins/reproducible.db
if [ ! -f $PACKAGES_DB ] ; then
sqlite3 $PACKAGES_DB '
CREATE TABLE source_packages
(name TEXT NOT NULL,
version TEXT NOT NULL,
status TEXT NOT NULL
CHECK (status IN ("FTBFS","reproducible","unreproducible","404")),
build_date TEXT NOT NULL,
diffp_path TEXT,
PRIMARY KEY (name))'
fi
# this needs sid entries in sources.list:
grep deb-src /etc/apt/sources.list | grep sid
sudo apt-get update
# if $1 is an integer, build $1 random packages
if [[ $1 =~ ^-?[0-9]+$ ]] ; then
TMPFILE=$(mktemp)
curl http://ftp.de.debian.org/debian/dists/sid/main/source/Sources.xz > $TMPFILE
AMOUNT=$1
PACKAGES=$(xzcat $TMPFILE | grep "^Package" | cut -d " " -f2 | egrep -v "linux$"| sort -R | head -$AMOUNT | xargs echo)
rm $TMPFILE
else
PACKAGES="$@"
fi
set +x
echo
echo "=============================================================="
echo "The following source packages will be build: $PACKAGES"
echo "=============================================================="
echo
set -x
NUM_CPU=$(cat /proc/cpuinfo |grep ^processor|wc -l)
COUNT_TOTAL=0
COUNT_GOOD=0
COUNT_BAD=0
COUNT_SKIPPED=0
GOOD=""
BAD=""
SOURCELESS=""
SKIPPED=""
for SRCPACKAGE in $PACKAGES ; do
let "COUNT_TOTAL=COUNT_TOTAL+1"
rm b1 b2 -rf
set +e
DATE=$(date +'%Y-%m-%d %H:%M')
VERSION=$(apt-cache showsrc ${SRCPACKAGE} | grep ^Version | cut -d " " -f2 | head -1)
apt-get source --download-only ${SRCPACKAGE}=${VERSION}
RESULT=$?
if [ $RESULT != 0 ] ; then
SOURCELESS="${SOURCELESS} ${SRCPACKAGE}"
echo "Warning: ${SRCPACKAGE} is not a source package, or was removed or renamed. Please investigate."
sqlite3 $PACKAGES_DB "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"404\", \"$DATE\", \"\")"
else
STATUS=$(sqlite3 $PACKAGES_DB "SELECT status FROM source_packages WHERE name = \"${SRCPACKAGE}\" AND version = \"${VERSION}\"")
if [ "$STATUS" = "reproducible" ] && [ $(( $RANDOM % 100 )) -gt 25 ] ; then
echo "Package ${SRCPACKAGE} (${VERSION}) build reproducibly in the past and was thus randomly skipped."
let "COUNT_SKIPPED=COUNT_SKIPPED+1"
SKIPPED="${SRCPACKAGE} ${SKIPPED}"
continue
fi
sudo DEB_BUILD_OPTIONS="parallel=$NUM_CPU" pbuilder --build --basetgz /var/cache/pbuilder/base-reproducible.tgz --distribution sid ${SRCPACKAGE}_*.dsc
RESULT=$?
if [ $RESULT = 0 ] ; then
mkdir b1 b2
dcmd cp /var/cache/pbuilder/result/${SRCPACKAGE}_*.changes b1
sudo dcmd rm /var/cache/pbuilder/result/${SRCPACKAGE}_*.changes
sudo DEB_BUILD_OPTIONS="parallel=$NUM_CPU" pbuilder --build --basetgz /var/cache/pbuilder/base-reproducible.tgz --distribution sid ${SRCPACKAGE}_*.dsc
dcmd cp /var/cache/pbuilder/result/${SRCPACKAGE}_*.changes b2
sudo dcmd rm /var/cache/pbuilder/result/${SRCPACKAGE}_*.changes
set -e
cat b1/${SRCPACKAGE}_*.changes
mkdir -p results/_success
LOGFILE=$(ls ${SRCPACKAGE}_*.dsc)
LOGFILE=$(echo ${LOGFILE%.dsc}.diffp)
./misc.git/diffp b1/${SRCPACKAGE}_*.changes b2/${SRCPACKAGE}_*.changes | tee ./results/${LOGFILE}
if ! $(grep -qv '^\*\*\*\*\*' ./results/${LOGFILE}) ; then
mv ./results/${LOGFILE} ./results/_success/
figlet ${SRCPACKAGE}
echo
echo "${SRCPACKAGE} built successfully and reproducibly."
sqlite3 $PACKAGES_DB "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"reproducible\", \"$DATE\", \"\")"
let "COUNT_GOOD=COUNT_GOOD+1"
GOOD="${SRCPACKAGE} ${GOOD}"
touch results/___.dummy.log # not having any bad logs is not a reason for failure
else
echo "Warning: ${SRCPACKAGE} failed to build reproducibly."
sqlite3 $PACKAGES_DB "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"unreproducible\", \"$DATE\", \"\")"
let "COUNT_BAD=COUNT_BAD+1"
BAD="${SRCPACKAGE} ${BAD}"
rm -f results/dummy.log 2>/dev/null # just cleanup
fi
rm b1 b2 -rf
else
sqlite3 $PACKAGES_DB "REPLACE INTO source_packages VALUES (\"${SRCPACKAGE}\", \"${VERSION}\", \"FTBFS\", \"$DATE\", \"\")"
fi
dcmd rm ${SRCPACKAGE}_*.dsc
fi
set +x
echo "=============================================================="
echo "$COUNT_TOTAL of ${#PACKAGES} done."
echo "=============================================================="
set -x
done
set +x
echo
echo
echo "$COUNT_TOTAL packages attempted to build in total."
echo "$COUNT_GOOD packages successfully built reproducibly: ${GOOD}"
echo "$COUNT_SKIPPED packages skipped which were successfully built reproducibly in the past: ${SKIPPED}"
echo "$COUNT_BAD packages failed to built reproducibly: ${BAD}"
echo "The following source packages doesn't exist in sid: $SOURCELESS"
|