summaryrefslogtreecommitdiffstats
path: root/bin
diff options
context:
space:
mode:
Diffstat (limited to 'bin')
-rwxr-xr-xbin/lvc/run_test_suite174
1 files changed, 126 insertions, 48 deletions
diff --git a/bin/lvc/run_test_suite b/bin/lvc/run_test_suite
index 9939abca..154a4a6c 100755
--- a/bin/lvc/run_test_suite
+++ b/bin/lvc/run_test_suite
@@ -1,44 +1,97 @@
-#!/bin/sh
+#!/bin/bash
set -e
set -u
+set -o pipefail
NAME=$(basename ${0})
+GENERAL_DEPENDENCIES="
+cucumber
+devscripts
+dnsmasq-base
+gawk
+git
+i18nspector
+libav-tools
+libcap2-bin
+libsikuli-script-java
+libvirt-clients
+libvirt-daemon-system
+libvirt-dev
+libvirt0
+openjdk-7-jre
+openssh-server
+ovmf
+python-jabberbot
+python-potr
+qemu-kvm
+qemu-system-x86
+ruby-guestfs
+ruby-json
+ruby-libvirt
+ruby-net-irc
+ruby-packetfu
+ruby-rb-inotify
+ruby-rjb
+ruby-rspec
+ruby-test-unit
+seabios
+tcpdump
+unclutter
+virt-viewer
+xvfb
+"
+
usage() {
- echo "Usage: $NAME [OPTION]... [FEATURE]...
-Sets up an appropriate environment and tests FEATUREs (all by default). Note
-that this script must be run from the Tails source directory root.
+ echo "Usage: $NAME [OPTION]... [--] [CUCUMBER_ARGS]...
+Sets up an appropriate environment and invokes cucumber. Note that this script
+must be run from the Tails source directory root.
Options for '@product' features:
- --capture FILE Captures the test session into FILE using VP8 encoding.
- Requires ffmpeg and libvpx1.
- --debug Display various debugging information while running the
- test suite.
+ --artifacts-base-uri URI
+ Pretend that the artifact is located at URI when printing
+ its location during a scenario failure. This is useful if
+ you intend to serve the artifacts via the web, for
+ instance.
+ --capture Captures failed scenarios into videos stored in the
+ temporary directory (see --tmpdir below) using x264
+ encoding. Requires x264.
+ --capture-all Keep videos for all scenarios, including those that
+ succeed (implies --capture).
--pause-on-fail On failure, pause test suite until pressing Enter. This is
useful for investigating the state of the VM guest to see
exactly why a test failed.
- --keep-snapshots Don't ever delete the background snapshots. This can a big
- time saver when debugging new features.
+ --keep-snapshots Don't ever delete any snapshots (including ones marked as
+ temporary). This can be a big time saver when debugging new
+ features.
--retry-find Print a warning whenever Sikuli fails to find an image
and allow *one* retry after pressing ENTER. This is useful
for updating outdated images.
- --temp-dir Directory where various temporary files are written
+ --tmpdir Directory where various temporary files are written
during a test, e.g. VM snapshots and memory dumps,
failure screenshots, pcap files and disk images
- (default is /tmp/DebianToaster).
+ (default is TMPDIR in the environment, and if unset,
+ /tmp/DebianToaster).
--view Shows the test session in a windows. Requires x11vnc
and xtightvncviewer.
--vnc-server-only Starts a VNC server for the test session. Requires x11vnc.
- --iso IMAGE Test '@product' features using IMAGE. If none is given,
- the ISO with most recent creation date (according to the
- ISO's label) in the current directory will be used.
+ --iso IMAGE Test '@product' features using IMAGE.
--old-iso IMAGE For some '@product' features (e.g. usb_install) we need
an older version of Tails, which this options sets to
- IMAGE. If none is given, the ISO with the least recent
- creation date will be used.
+ IMAGE. If none is given, it defaults to the same IMAGE
+ given by --iso, which will be good enough for most testing
+ purposes.
Note that '@source' features has no relevant options.
+
+CUCUMBER_ARGS can be used to specify which features to be run, but also any
+cucumber option, although then you must pass \`--\` first to let this wrapper
+script know that we're done with *its* options. For debugging purposes, a
+'debug' formatter has been added so pretty debugging can be enabled with
+\`--format debug\`. You could even combine the default (pretty) formatter with
+pretty debugging printed to a file with \`--format pretty --format debug
+--out debug.log\`.
"
}
@@ -48,11 +101,25 @@ error() {
exit 1
}
-check_dependency() {
- if ! which "${1}" >/dev/null && \
- ! dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
- error "'${1}' is missing, please install it and run again. Aborting..."
+package_installed() {
+ local ret
+ set +o pipefail
+ if dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
+ ret=0
+ else
+ ret=1
fi
+ set -o pipefail
+ return ${ret}
+}
+
+check_dependencies() {
+ while [ -n "${1:-}" ]; do
+ if ! which "${1}" >/dev/null && ! package_installed "${1}" ; then
+ error "'${1}' is missing, please install it and run again."
+ fi
+ shift
+ done
}
display_in_use() {
@@ -67,11 +134,13 @@ next_free_display() {
echo ":${display_nr}"
}
+test_suite_cleanup() {
+ (kill -0 ${XVFB_PID} 2>/dev/null && kill ${XVFB_PID}) || /bin/true
+}
+
start_xvfb() {
Xvfb $TARGET_DISPLAY -screen 0 1024x768x24+32 >/dev/null 2>&1 &
XVFB_PID=$!
- trap "kill -0 ${XVFB_PID} 2>/dev/null && kill -9 ${XVFB_PID}; \
- rm -f /tmp/.X${TARGET_DISPLAY#:}-lock" EXIT
# Wait for Xvfb to run on TARGET_DISPLAY
until display_in_use $TARGET_DISPLAY; do
sleep 1
@@ -82,42 +151,51 @@ start_xvfb() {
}
start_vnc_server() {
- check_dependency x11vnc
+ check_dependencies x11vnc
VNC_SERVER_PORT="$(x11vnc -listen localhost -display ${TARGET_DISPLAY} \
- -bg -nopw 2>&1 | \
+ -bg -nopw -forever 2>&1 | \
grep -m 1 "^PORT=[0-9]\+" | sed 's/^PORT=//')"
echo "VNC server running on: localhost:${VNC_SERVER_PORT}"
}
start_vnc_viewer() {
- check_dependency xtightvncviewer
+ check_dependencies xtightvncviewer
xtightvncviewer -viewonly localhost:${VNC_SERVER_PORT} 1>/dev/null 2>&1 &
}
capture_session() {
+ check_dependencies libvpx1
echo "Capturing guest display into ${CAPTURE_FILE}"
- ffmpeg -f x11grab -s 1024x768 -r 15 -i ${TARGET_DISPLAY}.0 -an \
+ avconv -f x11grab -s 1024x768 -r 15 -i ${TARGET_DISPLAY}.0 -an \
-vcodec libvpx -y "${CAPTURE_FILE}" >/dev/null 2>&1 &
}
# main script
-CAPTURE_FILE=
+# Unset all environment variables used by this script to pass options
+# to cucumber, except TMPDIR since we explicitly want to support
+# setting it that way.
+ARTIFACTS_BASE_URI=
+CAPTURE=
+CAPTURE_ALL=
+LOG_FILE=
VNC_VIEWER=
VNC_SERVER=
-DEBUG=
PAUSE_ON_FAIL=
KEEP_SNAPSHOTS=
SIKULI_RETRY_FINDFAILED=
-TEMP_DIR=
ISO=
OLD_ISO=
-LONGOPTS="view,vnc-server-only,capture:,help,temp-dir:,keep-snapshots,retry-find,iso:,old-iso:,debug,pause-on-fail"
+LONGOPTS="artifacts-base-uri:,view,vnc-server-only,capture,capture-all,help,tmpdir:,keep-snapshots,retry-find,iso:,old-iso:,pause-on-fail"
OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
eval set -- "$OPTS"
while [ $# -gt 0 ]; do
case $1 in
+ --artifacts-base-uri)
+ shift
+ export ARTIFACTS_BASE_URI="${1}"
+ ;;
--view)
VNC_VIEWER=yes
VNC_SERVER=yes
@@ -127,11 +205,13 @@ while [ $# -gt 0 ]; do
VNC_SERVER=yes
;;
--capture)
- shift
- CAPTURE_FILE="$1"
+ check_dependencies x264
+ export CAPTURE="yes"
;;
- --debug)
- export DEBUG="yes"
+ --capture-all)
+ check_dependencies x264
+ export CAPTURE="yes"
+ export CAPTURE_ALL="yes"
;;
--pause-on-fail)
export PAUSE_ON_FAIL="yes"
@@ -142,9 +222,9 @@ while [ $# -gt 0 ]; do
--retry-find)
export SIKULI_RETRY_FINDFAILED="yes"
;;
- --temp-dir)
+ --tmpdir)
shift
- export TEMP_DIR="$(readlink -f $1)"
+ export TMPDIR="$(readlink -f $1)"
;;
--iso)
shift
@@ -166,26 +246,21 @@ while [ $# -gt 0 ]; do
shift
done
-for dep in ffmpeg git libvirt-bin libvirt-dev libavcodec-extra-53 libvpx1 \
- virt-viewer libsikuli-script-java ovmf tcpdump xvfb; do
- check_dependency "${dep}"
-done
+trap "test_suite_cleanup" EXIT HUP INT QUIT TERM
+
+check_dependencies ${GENERAL_DEPENDENCIES}
TARGET_DISPLAY=$(next_free_display)
start_xvfb
-if [ -n "${CAPTURE_FILE}" ]; then
- capture_session
-fi
-if [ -n "${VNC_SERVER}" ]; then
+if [ -n "${VNC_SERVER:-}" ]; then
start_vnc_server
fi
-if [ -n "${VNC_VIEWER}" ]; then
+if [ -n "${VNC_VIEWER:-}" ]; then
start_vnc_viewer
fi
-export JAVA_HOME="/usr/lib/jvm/java-7-openjdk-amd64"
export SIKULI_HOME="/usr/share/java"
export SIKULI_IMAGE_PATH="/srv/jenkins/features/images/"
export RUBYLIB="/srv/jenkins"
@@ -193,7 +268,10 @@ export FEATURE_PATH="/srv/jenkins/features"
export VM_XML_PATH="/srv/jenkins/features/domains"
export DISPLAY=${TARGET_DISPLAY}
CUCUMBEROPTS="--verbose --backtrace --expand"
-check_dependency cucumber
+check_dependencies cucumber
+
+set -x
+
if [ -z "${*}" ]; then
cucumber $CUCUMBEROPTS --format ExtraHooks::Pretty $FEATURE_PATH
else