rough attempt to grab the good cucumber bits from recent tails

1 files changed, 126 insertions, 48 deletions

diff --git a/bin/lvc/run_test_suite b/bin/lvc/run_test_suite
index 9939abca..154a4a6c 100755
--- a/bin/lvc/run_test_suite
+++ b/bin/lvc/run_test_suite
@@ -1,44 +1,97 @@
-#!/bin/sh
+#!/bin/bash
 
 set -e
 set -u
+set -o pipefail
 
 NAME=$(basename ${0})
 
+GENERAL_DEPENDENCIES="
+cucumber
+devscripts
+dnsmasq-base
+gawk
+git
+i18nspector
+libav-tools
+libcap2-bin
+libsikuli-script-java
+libvirt-clients
+libvirt-daemon-system
+libvirt-dev
+libvirt0
+openjdk-7-jre
+openssh-server
+ovmf
+python-jabberbot
+python-potr
+qemu-kvm
+qemu-system-x86
+ruby-guestfs
+ruby-json
+ruby-libvirt
+ruby-net-irc
+ruby-packetfu
+ruby-rb-inotify
+ruby-rjb
+ruby-rspec
+ruby-test-unit
+seabios
+tcpdump
+unclutter
+virt-viewer
+xvfb
+"
+
 usage() {
-    echo "Usage: $NAME [OPTION]... [FEATURE]...
-Sets up an appropriate environment and tests FEATUREs (all by default). Note
-that this script must be run from the Tails source directory root.
+    echo "Usage: $NAME [OPTION]... [--] [CUCUMBER_ARGS]...
+Sets up an appropriate environment and invokes cucumber. Note that this script
+must be run from the Tails source directory root.
 
 Options for '@product' features:
-  --capture FILE     Captures the test session into FILE using VP8 encoding.
-                     Requires ffmpeg and libvpx1.
-  --debug            Display various debugging information while running the
-                     test suite.
+  --artifacts-base-uri URI
+                     Pretend that the artifact is located at URI when printing
+                     its location during a scenario failure. This is useful if
+                     you intend to serve the artifacts via the web, for
+                     instance.
+  --capture          Captures failed scenarios into videos stored in the
+                     temporary directory (see --tmpdir below) using x264
+                     encoding. Requires x264.
+  --capture-all      Keep videos for all scenarios, including those that
+                     succeed (implies --capture).
   --pause-on-fail    On failure, pause test suite until pressing Enter. This is
                      useful for investigating the state of the VM guest to see
                      exactly why a test failed.
-  --keep-snapshots   Don't ever delete the background snapshots. This can a big
-                     time saver when debugging new features.
+  --keep-snapshots   Don't ever delete any snapshots (including ones marked as
+                     temporary). This can be a big time saver when debugging new
+                     features.
   --retry-find       Print a warning whenever Sikuli fails to find an image
                      and allow *one* retry after pressing ENTER. This is useful
                      for updating outdated images.
-  --temp-dir         Directory where various temporary files are written
+  --tmpdir           Directory where various temporary files are written
                      during a test, e.g. VM snapshots and memory dumps,
                      failure screenshots, pcap files and disk images
-                     (default is /tmp/DebianToaster).
+                     (default is TMPDIR in the environment, and if unset,
+                     /tmp/DebianToaster).
   --view             Shows the test session in a windows. Requires x11vnc
                      and xtightvncviewer.
   --vnc-server-only  Starts a VNC server for the test session. Requires x11vnc.
-  --iso IMAGE        Test '@product' features using IMAGE. If none is given,
-                     the ISO with most recent creation date (according to the
-                     ISO's label) in the current directory will be used.
+  --iso IMAGE        Test '@product' features using IMAGE.
   --old-iso IMAGE    For some '@product' features (e.g. usb_install) we need
                      an older version of Tails, which this options sets to
-                     IMAGE. If none is given, the ISO with the least recent
-                     creation date will be used.
+                     IMAGE. If none is given, it defaults to the same IMAGE
+                     given by --iso, which will be good enough for most testing
+                     purposes.
 
 Note that '@source' features has no relevant options.
+
+CUCUMBER_ARGS can be used to specify which features to be run, but also any
+cucumber option, although then you must pass \`--\` first to let this wrapper
+script know that we're done with *its* options. For debugging purposes, a
+'debug' formatter has been added so pretty debugging can be enabled with
+\`--format debug\`. You could even combine the default (pretty) formatter with
+pretty debugging printed to a file with \`--format pretty --format debug
+--out debug.log\`.
 "
 }
 
@@ -48,11 +101,25 @@ error() {
     exit 1
 }
 
-check_dependency() {
-    if ! which "${1}" >/dev/null && \
-       ! dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
-        error "'${1}' is missing, please install it and run again. Aborting..."
+package_installed() {
+    local ret
+    set +o pipefail
+    if dpkg -s "${1}" 2>/dev/null | grep -q "^Status:.*installed"; then
+        ret=0
+    else
+        ret=1
     fi
+    set -o pipefail
+    return ${ret}
+}
+
+check_dependencies() {
+    while [ -n "${1:-}" ]; do
+        if ! which "${1}" >/dev/null && ! package_installed "${1}" ; then
+            error "'${1}' is missing, please install it and run again."
+        fi
+        shift
+    done
 }
 
 display_in_use() {
@@ -67,11 +134,13 @@ next_free_display() {
     echo ":${display_nr}"
 }
 
+test_suite_cleanup() {
+    (kill -0 ${XVFB_PID} 2>/dev/null && kill ${XVFB_PID}) || /bin/true
+}
+
 start_xvfb() {
     Xvfb $TARGET_DISPLAY -screen 0 1024x768x24+32 >/dev/null 2>&1 &
     XVFB_PID=$!
-    trap "kill -0 ${XVFB_PID} 2>/dev/null && kill -9 ${XVFB_PID}; \
-          rm -f /tmp/.X${TARGET_DISPLAY#:}-lock" EXIT
     # Wait for Xvfb to run on TARGET_DISPLAY
     until display_in_use $TARGET_DISPLAY; do
 	sleep 1
@@ -82,42 +151,51 @@ start_xvfb() {
 }
 
 start_vnc_server() {
-    check_dependency x11vnc
+    check_dependencies x11vnc
     VNC_SERVER_PORT="$(x11vnc -listen localhost -display ${TARGET_DISPLAY} \
-                              -bg -nopw 2>&1 | \
+                              -bg -nopw -forever 2>&1 | \
                                   grep -m 1 "^PORT=[0-9]\+" | sed 's/^PORT=//')"
     echo "VNC server running on: localhost:${VNC_SERVER_PORT}"
 }
 
 start_vnc_viewer() {
-    check_dependency xtightvncviewer
+    check_dependencies xtightvncviewer
     xtightvncviewer -viewonly localhost:${VNC_SERVER_PORT} 1>/dev/null 2>&1 &
 }
 
 capture_session() {
+    check_dependencies libvpx1
     echo "Capturing guest display into ${CAPTURE_FILE}"
-    ffmpeg -f x11grab -s 1024x768 -r 15 -i ${TARGET_DISPLAY}.0 -an \
+    avconv -f x11grab -s 1024x768 -r 15 -i ${TARGET_DISPLAY}.0 -an \
         -vcodec libvpx -y "${CAPTURE_FILE}" >/dev/null 2>&1 &
 }
 
 # main script
 
-CAPTURE_FILE=
+# Unset all environment variables used by this script to pass options
+# to cucumber, except TMPDIR since we explicitly want to support
+# setting it that way.
+ARTIFACTS_BASE_URI=
+CAPTURE=
+CAPTURE_ALL=
+LOG_FILE=
 VNC_VIEWER=
 VNC_SERVER=
-DEBUG=
 PAUSE_ON_FAIL=
 KEEP_SNAPSHOTS=
 SIKULI_RETRY_FINDFAILED=
-TEMP_DIR=
 ISO=
 OLD_ISO=
 
-LONGOPTS="view,vnc-server-only,capture:,help,temp-dir:,keep-snapshots,retry-find,iso:,old-iso:,debug,pause-on-fail"
+LONGOPTS="artifacts-base-uri:,view,vnc-server-only,capture,capture-all,help,tmpdir:,keep-snapshots,retry-find,iso:,old-iso:,pause-on-fail"
 OPTS=$(getopt -o "" --longoptions $LONGOPTS -n "${NAME}" -- "$@")
 eval set -- "$OPTS"
 while [ $# -gt 0 ]; do
     case $1 in
+        --artifacts-base-uri)
+            shift
+            export ARTIFACTS_BASE_URI="${1}"
+            ;;
         --view)
             VNC_VIEWER=yes
             VNC_SERVER=yes
@@ -127,11 +205,13 @@ while [ $# -gt 0 ]; do
             VNC_SERVER=yes
             ;;
         --capture)
-            shift
-            CAPTURE_FILE="$1"
+            check_dependencies x264
+            export CAPTURE="yes"
             ;;
-        --debug)
-            export DEBUG="yes"
+        --capture-all)
+            check_dependencies x264
+            export CAPTURE="yes"
+            export CAPTURE_ALL="yes"
             ;;
         --pause-on-fail)
             export PAUSE_ON_FAIL="yes"
@@ -142,9 +222,9 @@ while [ $# -gt 0 ]; do
         --retry-find)
             export SIKULI_RETRY_FINDFAILED="yes"
             ;;
-        --temp-dir)
+        --tmpdir)
             shift
-            export TEMP_DIR="$(readlink -f $1)"
+            export TMPDIR="$(readlink -f $1)"
             ;;
         --iso)
             shift
@@ -166,26 +246,21 @@ while [ $# -gt 0 ]; do
     shift
 done
 
-for dep in ffmpeg git libvirt-bin libvirt-dev libavcodec-extra-53 libvpx1 \
-           virt-viewer libsikuli-script-java ovmf tcpdump xvfb; do
-    check_dependency "${dep}"
-done
+trap "test_suite_cleanup" EXIT HUP INT QUIT TERM
+
+check_dependencies ${GENERAL_DEPENDENCIES}
 
 TARGET_DISPLAY=$(next_free_display)
 
 start_xvfb
 
-if [ -n "${CAPTURE_FILE}" ]; then
-    capture_session
-fi
-if [ -n "${VNC_SERVER}" ]; then
+if [ -n "${VNC_SERVER:-}" ]; then
     start_vnc_server
 fi
-if [ -n "${VNC_VIEWER}" ]; then
+if [ -n "${VNC_VIEWER:-}" ]; then
     start_vnc_viewer
 fi
 
-export JAVA_HOME="/usr/lib/jvm/java-7-openjdk-amd64"
 export SIKULI_HOME="/usr/share/java"
 export SIKULI_IMAGE_PATH="/srv/jenkins/features/images/"
 export RUBYLIB="/srv/jenkins"
@@ -193,7 +268,10 @@ export FEATURE_PATH="/srv/jenkins/features"
 export VM_XML_PATH="/srv/jenkins/features/domains"
 export DISPLAY=${TARGET_DISPLAY}
 CUCUMBEROPTS="--verbose --backtrace --expand"
-check_dependency cucumber
+check_dependencies cucumber
+
+set -x
+
 if [ -z "${*}" ]; then
     cucumber $CUCUMBEROPTS --format ExtraHooks::Pretty $FEATURE_PATH
 else