summaryrefslogtreecommitdiffstats
path: root/bin/find_dpkg_trigger_cycles.sh
blob: f3c70206b88b9e12ba0df4c55bba8c3b8e037a93 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
#!/bin/bash
#
# Copyright 2014 Johannes Schauer <j.schauer@email.de>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
# 
# Running
# =======
# 
# Just start `./find_dpkg_trigger_cycles.sh`. It will do the following:
# 
# 1. download apt sources and apt-file data for the amd64 Debian
#     using $1 as distro and store them in a directory tree rooted at
#    `./debian-sid-amd64`
# 2. go through all binary packages which have a file `DEBIAN/triggers` in their
#    control archive (the list is retrieved from binarycontrol.debian.net)
#    and for each package:
#   1. download and unpack its control archive
#   2. store all interest-await file triggers in the file `interested-file`
#   3. store all interest-await explicit triggers in the file `interested-explicit`
#   4. store all activate-await file triggers in the file `activated-file`
#   5. store all activate-await explicit triggers in the file `activated-explicit`
#   6. remove the downloaded binary package and unpacked control archive
# 3. go through `interested-file` and for each line:
#   1. calculate the dependency closure for the binary package and for
#      each package in the closure:
#     1. use `apt-file` to get all files of the package
#     2. check if the current file trigger matches any file in the package
#     3. store any hits in the file `result-file`
# 4. go through `interested-file` and for each line:
#   1. calculate the dependency closure for the binary package and for
#      each package in the closure:
#     1. check if the package activates the current file trigger
#     2. append any hits to the file `result-file`
# 5. go through `interested-explicit` and for each line:
#   1. calculate the dependency closure for the binary package and for
#      each package in the closure:
#     1. check if the package activate the current explicit trigger
#     2. store any hits in the file `result-explicit`
# 
# Files
# =====
# 
# interested-file
# ---------------
# 
# Associates binary packages to file triggers they are interested in. The first
# column is the binary package, the second column is either `interest` or
# `interest-await` and the last column the path they are interested in.
# 
# interested-explicit
# -------------------
# 
# Associates binary packages to explicit triggers they are interested in. The
# first column is the binary package, the second column is either `interest` or
# `interest-await` and the last column the name of the explicit trigger they are
# interested in.
# 
# activated-file
# --------------
# 
# Associates binary packages to file triggers they activate. The first column is
# the binary package, the second column is either `activate` or `activate-await`
# and the last column the path they activate.
# 
# activate-explicit
# -----------------
# 
# Associates binary packages to explicit triggers they activate. The first column
# is the binary package, the second column is either `activate` or
# `activate-await` and the last column the explicit trigger they activate.
# 
# result-file
# -----------
# 
# Associates binary packages with other binary packages they can form a file
# trigger cycle with. The first column is the binary package containing the file
# trigger, the second column is the file trigger, the third column is a binary
# package providing a path that triggers the binary package in the first column,
# the fourth column is the triggering path of provided by the binary package in
# the third column.
# 
# result-explicit
# ---------------
# 
# Associates binary packages with other binary packages they can form an explicit
# trigger cycle with. The first column is the binary package interested in the
# explicit trigger, the second column is the name of the explicit trigger, the
# third column is the binary package activating the trigger.

DEBUG=false
. /srv/jenkins/bin/common-functions.sh
common_init "$@"

ARCH="amd64"
DIST="$1"
DIRECTORY="`pwd`/debian-$DIST-$ARCH"

#FIXME: if the host has more than one arch enabled then those Packages files will be downloaded as well

APT_OPTS="-y"
APT_OPTS=$APT_OPTS" -o Apt::Architecture=$ARCH"
APT_OPTS=$APT_OPTS" -o Dir::Etc::TrustedParts=$DIRECTORY/etc/apt/trusted.gpg.d"
APT_OPTS=$APT_OPTS" -o Dir::Etc::Trusted=$DIRECTORY/etc/apt/trusted.gpg"
APT_OPTS=$APT_OPTS" -o Dir=$DIRECTORY/"
APT_OPTS=$APT_OPTS" -o Dir::Etc=$DIRECTORY/etc/apt/"
APT_OPTS=$APT_OPTS" -o Dir::Etc::SourceList=$DIRECTORY/etc/apt/sources.list"
APT_OPTS=$APT_OPTS" -o Dir::State=$DIRECTORY/var/lib/apt/"
APT_OPTS=$APT_OPTS" -o Dir::State::Status=$DIRECTORY/var/lib/dpkg/status"
APT_OPTS=$APT_OPTS" -o Dir::Cache=$DIRECTORY/var/cache/apt/"
#APT_OPTS=$APT_OPTS" -o Acquire::Check-Valid-Until=false" # because we use snapshot

mkdir -p $DIRECTORY
mkdir -p $DIRECTORY/etc/apt/
mkdir -p $DIRECTORY/etc/apt/trusted.gpg.d/
mkdir -p $DIRECTORY/etc/apt/sources.list.d/
mkdir -p $DIRECTORY/etc/apt/preferences.d/
mkdir -p $DIRECTORY/var/lib/apt/
mkdir -p $DIRECTORY/var/lib/apt/lists/partial/
mkdir -p $DIRECTORY/var/lib/dpkg/
mkdir -p $DIRECTORY/var/cache/apt/
mkdir -p $DIRECTORY/var/cache/apt/apt-file/

cp /etc/apt/trusted.gpg.d/* $DIRECTORY/etc/apt/trusted.gpg.d/

touch $DIRECTORY/var/lib/dpkg/status

echo deb $MIRROR $DIST main > $DIRECTORY/etc/apt/sources.list

apt-get $APT_OPTS update

APT_FILE_OPTS="--architecture $ARCH"
APT_FILE_OPTS=$APT_FILE_OPTS" --cache $DIRECTORY/var/cache/apt/apt-file"
APT_FILE_OPTS=$APT_FILE_OPTS" --sources-list $DIRECTORY/etc/apt/sources.list"

apt-file $APT_FILE_OPTS update

printf "" > $DIRECTORY/interested-file
printf "" > $DIRECTORY/interested-explicit
printf "" > $DIRECTORY/activated-file
printf "" > $DIRECTORY/activated-explicit

# find all binary packages with /triggers$
curl --globoff "http://binarycontrol.debian.net/?q=&path=${DIST}%2F[^%2F]%2B%2Ftriggers%24&format=pkglist" \
	| xargs apt-get $APT_OPTS --print-uris download \
	| sed -ne "s/^'\([^']\+\)'\s\+\([^_]\+\)_.*/\2 \1/p" \
	| sort \
	| while read pkg url; do
	echo "working on $pkg..." >&2
	tmpdir=`mktemp -d`
	curl --retry 2 --location --silent "$url" \
		| dpkg-deb --ctrl-tarfile /dev/stdin \
		| tar -C "$tmpdir" --exclude=./md5sums -x
	if [ ! -f "$tmpdir/triggers" ]; then
		rm -r "$tmpdir"
		continue
	fi
	# find all triggers that are either interest or interest-await
	# and which are file triggers (start with a slash)
	egrep "^\s*interest(-await)?\s+/" "$tmpdir/triggers" | while read line; do
		echo "$pkg $line"
	done >> $DIRECTORY/interested-file
	egrep "^\s*interest(-await)?\s+[^/]" "$tmpdir/triggers" | while read line; do
		echo "$pkg $line"
	done >> $DIRECTORY/interested-explicit
	egrep "^\s*activate(-await)?\s+/" "$tmpdir/triggers" | while read line; do
		echo "$pkg $line"
	done >> $DIRECTORY/activated-file
	egrep "^\s*activate(-await)?\s+[^/]" "$tmpdir/triggers" | while read line; do
		echo "$pkg $line"
	done >> $DIRECTORY/activated-explicit
	rm -r "$tmpdir"
done

printf "" > $DIRECTORY/result-file

# go through those that are interested in a path and check them against the
# files provided by its dependency closure
cat $DIRECTORY/interested-file | while read pkg ttype ipath; do
	echo "working on $pkg..." >&2
	echo "getting dependency closure..." >&2
	# go through all packages in the dependency closure and check if any
	# of the files they ship match one of the interested paths
	dose-ceve -c $pkg -T cudf -t deb \
		$DIRECTORY/var/lib/apt/lists/*_dists_${DIST}_main_binary-${ARCH}_Packages \
		| awk '/^package:/ { print $2 }' \
		| apt-file $APT_FILE_OPTS show -F --from-file - \
		| sed -ne "s ^\([^:]\+\):\s\+\(${ipath}/.*\) \1\t\2 p" \
		| while read dep cpath; do
			[ "$pkg" != "$dep" ] || continue
			echo "$pkg $ipath $dep $cpath"
		done >> $DIRECTORY/result-file
done

# go through those that are interested in a path and check them against the
# packages in the dependency closure which activate such a path
cat $DIRECTORY/interested-file | while read pkg ttype ipath; do
	echo "working on $pkg..." >&2
	echo "getting dependency closure..." >&2
	# go through all packages in the dependency closure and check if any
	# of them activate a matching path
	dose-ceve -c $pkg -T cudf -t deb \
		$DIRECTORY/var/lib/apt/lists/*_dists_${DIST}_main_binary-${ARCH}_Packages \
		| awk '/^package:/ { print $2 }' \
		| while read dep; do
			[ "$pkg" != "$dep" ] || continue
			# using the space as sed delimeter because ipath has slashes
			# a space should work because neither package names nor paths have them
			sed -ne "s ^$dep\s\+activate\(-await\)\?\s\+\($ipath.*\) \2 p" $DIRECTORY/activated-file | while read cpath; do
				echo "$pkg $ipath $dep $cpath"
			done
		done >> $DIRECTORY/result-file
done

printf "" > $DIRECTORY/result-explicit

# go through those that are interested in an explicit trigger and check them
# against the packages in their dependency closure which activate it
cat $DIRECTORY/interested-explicit | while read pkg ttype iname; do
	echo "working on $pkg..." >&2
	echo "getting dependency closure..." >&2
	# go through all packages in the dependency closure and check if any of
	# them activate the trigger in which this package is interested
	dose-ceve -c $pkg -T cudf -t deb \
		$DIRECTORY/var/lib/apt/lists/*_dists_${DIST}_main_binary-${ARCH}_Packages \
		| awk '/^package:/ { print $2 }' \
		| while read dep; do
			[ "$pkg" != "$dep" ] || continue
			if egrep "^$dep\s+activate(-await)?\s+$iname\s*$" $DIRECTORY/activated-explicit > /dev/null; then
				echo "$pkg $iname $dep"
			fi
		done >> $DIRECTORY/result-explicit
done

echo "+----------------------------------------------------------+"
echo "|                     result summary                       |"
echo "+----------------------------------------------------------+"
echo ""
echo "number of found file based trigger cycles:"
wc -l < $DIRECTORY/result-file
if [ `wc -l < $DIRECTORY/result-file` -ne 0 ]; then
	echo "Warning: found file based trigger cycles"
	echo "number of packages creating file based trigger cycles:"
	awk '{ print $1 }' $DIRECTORY/result-file | sort | uniq | wc -l
	echo "unique packages creating file based trigger cycles:"
	awk '{ print $1 }' $DIRECTORY/result-file | sort | uniq
fi
echo "number of found explicit trigger cycles:"
wc -l < $DIRECTORY/result-explicit
if [ `wc -l < $DIRECTORY/result-explicit` -ne 0 ]; then
	echo "Warning: found explicit trigger cycles"
	echo "number of packages creating explicit trigger cycles:"
	awk '{ print $1 }' $DIRECTORY/result-explicit | sort | uniq | wc -l
	echo "unique packages creating explicit trigger cycles:"
	awk '{ print $1 }' $DIRECTORY/result-explicit | sort | uniq
fi
if [ `wc -l < $DIRECTORY/result-file` -ne 0 ]; then
	echo ""
	echo ""
	echo "+----------------------------------------------------------+"
	echo "|               file based trigger cycles                  |"
	echo "+----------------------------------------------------------+"
	echo ""
	echo "# Associates binary packages with other binary packages they can form a file"
	echo "# trigger cycle with. The first column is the binary package containing the file"
	echo "# trigger, the second column is the file trigger, the third column is a binary"
	echo "# package providing a path that triggers the binary package in the first column,"
	echo "# the fourth column is the triggering path of provided by the binary package in"
	echo "# the third column."
	echo ""
	cat $DIRECTORY/result-file
fi
if [ `wc -l < $DIRECTORY/result-explicit` -ne 0 ]; then
	echo ""
	echo ""
	echo "+----------------------------------------------------------+"
	echo "|               explicit trigger cycles                    |"
	echo "+----------------------------------------------------------+"
	echo ""
	echo "# Associates binary packages with other binary packages they can form an explicit"
	echo "# trigger cycle with. The first column is the binary package interested in the"
	echo "# explicit trigger, the second column is the name of the explicit trigger, the"
	echo "# third column is the binary package activating the trigger."
	echo ""
	cat $DIRECTORY/result-explicit
fi