diff options
Diffstat (limited to 'bin/lvc/run_test_suite')
| -rwxr-xr-x | bin/lvc/run_test_suite | 174 |
1 files changed, 126 insertions, 48 deletions
diff --git a/bin/lvc/run_test_suite b/bin/lvc/run_test_suite index 9939abca..154a4a6c 100755 --- a/bin/lvc/run_test_suite +++ b/bin/lvc/run_test_suite @@ -1,44 +1,97 @@ -#!/bin/sh +#!/bin/bash set -e set -u +set -o pipefail NAME=$(basename ${0}) +GENERAL_DEPENDENCIES=" +cucumber +devscripts +dnsmasq-base +gawk +git +i18nspector +libav-tools +libcap2-bin +libsikuli-script-java +libvirt-clients +libvirt-daemon-system +libvirt-dev +libvirt0 +openjdk-7-jre +openssh-server +ovmf +python-jabberbot +python-potr +qemu-kvm +qemu-system-x86 +ruby-guestfs +ruby-json +ruby-libvirt +ruby-net-irc +ruby-packetfu +ruby-rb-inotify +ruby-rjb +ruby-rspec +ruby-test-unit +seabios +tcpdump +unclutter +virt-viewer +xvfb +" + usage() { - echo "Usage: $NAME [OPTION]... [FEATURE]... -Sets up an appropriate environment and tests FEATUREs (all by default). Note -that this script must be run from the Tails source directory root. + echo "Usage: $NAME [OPTION]... [--] [CUCUMBER_ARGS]... +Sets up an appropriate environment and invokes cucumber. Note that this script +must be run from the Tails source directory root. Options for '@product' features: - --capture FILE Captures the test session into FILE using VP8 encoding. - Requires ffmpeg and libvpx1. - --debug Display various debugging information while running the - test suite. + --artifacts-base-uri URI + Pretend that the artifact is located at URI when printing + its location during a scenario failure. This is useful if + you intend to serve the artifacts via the web, for + instance. + --capture Captures failed scenarios into videos stored in the + temporary directory (see --tmpdir below) using x264 + encoding. Requires x264. + --capture-all Keep videos for all scenarios, including those that + succeed (implies --capture). --pause-on-fail On failure, pause test suite until pressing Enter. This is useful for investigating the state of the VM guest to see exactly why a test failed. - --keep-snapshots Don't ever delete the background snapshots. This can a big - time saver when debugging new features. + --keep-snapshots Don't ever delete any snapshots (including ones marked as + temporary). This can be a big time saver when debugging new + features. --retry-find Print a warning whenever Sikuli fails to find an image and allow *one* retry after pressing ENTER. This is useful for updating outdated images. - --temp-dir Directory where various temporary files are written + --tmpdir Directory where various temporary files are written during a test, e.g. VM snapshots and memory dumps, failure screenshots, pcap files and disk images - (default is /tmp/DebianToaster). + (default is TMPDIR in the environment, and if unset, + /tmp/DebianToaster). --view Shows the test session in a windows. Requires x11vnc and xtightvncviewer. --vnc-server-only Starts a VNC server for the test session. Requires x11vnc. - --iso IMAGE Test '@product' features using IMAGE. If none is given, - the ISO with most recent creation date (according to the - ISO's label) in the current directory will be used. + --iso IMAGE Test '@product' features using IMAGE. --old-iso IMAGE For some '@product' features (e.g. usb_install) we need an older version of Tails, which this options sets to - IMAGE. If none is given, the ISO with the least recent - creation date will be used. + IMAGE. If none is given, it defaults to the same IMAGE + given by --iso, which will be good enough for most testing + purposes. Note that '@source' features has no relevant options. + +CUCUMBER_ARGS can be used to specify which features to be run, but also any +cucumber option, although then you must pass \`--\` first to let this wrapper +script know that we're done with *its* options. For debugging purposes, a +'debug' formatter has been added so pretty debugging can be enabled with +\`--format debug\`. You could even combine the default (pretty) formatter with +pretty debugging printed to a file with \`--format pretty --format debug +--out debug.log\`. " } @@ -48,11 +101,25 @@ error() { exit 1 } -check_dependency() { - if ! which "${1}" >/dev/null && \ - ! dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then - error "'${1}' is missing, please install it and run again. Aborting..." +package_installed() { + local ret + set +o pipefail + if dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then + ret=0 + else + ret=1 fi + set -o pipefail + return ${ret} +} + +check_dependencies() { + while [ -n "${1:-}" ]; do + if ! which "${1}" >/dev/null && ! package_installed "${1}" ; then + error "'${1}' is missing, please install it and run again." + fi + shift + done } display_in_use() { @@ -67,11 +134,13 @@ next_free_display() { echo ":${display_nr}" } +test_suite_cleanup() { + (kill -0 ${XVFB_PID} 2>/dev/null && kill ${XVFB_PID}) || /bin/true +} + start_xvfb() { Xvfb $TARGET_DISPLAY -screen 0 1024x768x24+32 >/dev/null 2>&1 & XVFB_PID=$! - trap "kill -0 ${XVFB_PID} 2>/dev/null && kill -9 ${XVFB_PID}; \ - rm -f /tmp/.X${TARGET_DISPLAY#:}-lock" EXIT # Wait for Xvfb to run on TARGET_DISPLAY until display_in_use $TARGET_DISPLAY; do sleep 1 @@ -82,42 +151,51 @@ start_xvfb() { } start_vnc_server() { - check_dependency x11vnc + check_dependencies x11vnc VNC_SERVER_PORT="$(x11vnc -listen localhost -display ${TARGET_DISPLAY} \ - -bg -nopw 2>&1 | \ + -bg -nopw -forever 2>&1 | \ grep -m 1 "^PORT=[0-9]\+" | sed 's/^PORT=//')" echo "VNC server running on: localhost:${VNC_SERVER_PORT}" } start_vnc_viewer() { - check_dependency xtightvncviewer + check_dependencies xtightvncviewer xtightvncviewer -viewonly localhost:${VNC_SERVER_PORT} 1>/dev/null 2>&1 & } capture_session() { + check_dependencies libvpx1 echo "Capturing guest display into ${CAPTURE_FILE}" - ffmpeg -f x11grab -s 1024x768 -r 15 -i ${TARGET_DISPLAY}.0 -an \ + avconv -f x11grab -s 1024x768 -r 15 -i ${TARGET_DISPLAY}.0 -an \ -vcodec libvpx -y "${CAPTURE_FILE}" >/dev/null 2>&1 & } # main script -CAPTURE_FILE= +# Unset all environment variables used by this script to pass options +# to cucumber, except TMPDIR since we explicitly want to support +# setting it that way. +ARTIFACTS_BASE_URI= +CAPTURE= +CAPTURE_ALL= +LOG_FILE= VNC_VIEWER= VNC_SERVER= -DEBUG= PAUSE_ON_FAIL= KEEP_SNAPSHOTS= SIKULI_RETRY_FINDFAILED= -TEMP_DIR= ISO= OLD_ISO= -LONGOPTS="view,vnc-server-only,capture:,help,temp-dir:,keep-snapshots,retry-find,iso:,old-iso:,debug,pause-on-fail" +LONGOPTS="artifacts-base-uri:,view,vnc-server-only,capture,capture-all,help,tmpdir:,keep-snapshots,retry-find,iso:,old-iso:,pause-on-fail" OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@") eval set -- "$OPTS" while [ $# -gt 0 ]; do case $1 in + --artifacts-base-uri) + shift + export ARTIFACTS_BASE_URI="${1}" + ;; --view) VNC_VIEWER=yes VNC_SERVER=yes @@ -127,11 +205,13 @@ while [ $# -gt 0 ]; do VNC_SERVER=yes ;; --capture) - shift - CAPTURE_FILE="$1" + check_dependencies x264 + export CAPTURE="yes" ;; - --debug) - export DEBUG="yes" + --capture-all) + check_dependencies x264 + export CAPTURE="yes" + export CAPTURE_ALL="yes" ;; --pause-on-fail) export PAUSE_ON_FAIL="yes" @@ -142,9 +222,9 @@ while [ $# -gt 0 ]; do --retry-find) export SIKULI_RETRY_FINDFAILED="yes" ;; - --temp-dir) + --tmpdir) shift - export TEMP_DIR="$(readlink -f $1)" + export TMPDIR="$(readlink -f $1)" ;; --iso) shift @@ -166,26 +246,21 @@ while [ $# -gt 0 ]; do shift done -for dep in ffmpeg git libvirt-bin libvirt-dev libavcodec-extra-53 libvpx1 \ - virt-viewer libsikuli-script-java ovmf tcpdump xvfb; do - check_dependency "${dep}" -done +trap "test_suite_cleanup" EXIT HUP INT QUIT TERM + +check_dependencies ${GENERAL_DEPENDENCIES} TARGET_DISPLAY=$(next_free_display) start_xvfb -if [ -n "${CAPTURE_FILE}" ]; then - capture_session -fi -if [ -n "${VNC_SERVER}" ]; then +if [ -n "${VNC_SERVER:-}" ]; then start_vnc_server fi -if [ -n "${VNC_VIEWER}" ]; then +if [ -n "${VNC_VIEWER:-}" ]; then start_vnc_viewer fi -export JAVA_HOME="/usr/lib/jvm/java-7-openjdk-amd64" export SIKULI_HOME="/usr/share/java" export SIKULI_IMAGE_PATH="/srv/jenkins/features/images/" export RUBYLIB="/srv/jenkins" @@ -193,7 +268,10 @@ export FEATURE_PATH="/srv/jenkins/features" export VM_XML_PATH="/srv/jenkins/features/domains" export DISPLAY=${TARGET_DISPLAY} CUCUMBEROPTS="--verbose --backtrace --expand" -check_dependency cucumber +check_dependencies cucumber + +set -x + if [ -z "${*}" ]; then cucumber $CUCUMBEROPTS --format ExtraHooks::Pretty $FEATURE_PATH else |