lvc: grab updates from tails (01371c19bd..6ae59c49e5)

Signed-off-by: Holger Levsen <holger@layer-acht.org>

13 files changed, 881 insertions, 391 deletions

diff --git a/cucumber/features/support/config.rb b/cucumber/features/support/config.rb
index 13578d55..54a0f1cd 100644
--- a/cucumber/features/support/config.rb
+++ b/cucumber/features/support/config.rb
@@ -74,25 +74,11 @@ LIBVIRT_REMOTE_SHELL_PORT = 13370 + Integer($executor_number)
 MISC_FILES_DIR = "/srv/jenkins/cucumber/features/misc_files"
 SERVICES_EXPECTED_ON_ALL_IFACES =
   [
-   ["cupsd",    "0.0.0.0", "631"],
-   ["dhclient", "0.0.0.0", "*"]
+   ["cupsd",    "*", "631"],
+   ["dhclient", "*", "*"]
   ]
 # OpenDNS
 SOME_DNS_SERVER = "208.67.222.222"
-TOR_AUTHORITIES =
-  # List grabbed from Tor's sources, src/or/config.c:~750.
-  [
-   "86.59.21.38",
-   "128.31.0.39",
-   "194.109.206.212",
-   "82.94.251.203",
-   "199.254.238.52",
-   "131.188.40.189",
-   "193.23.244.244",
-   "208.83.223.34",
-   "171.25.193.9",
-   "154.35.175.225",
-  ]
 VM_XML_PATH = "/srv/jenkins/cucumber/features/domains"
 
 #TAILS_SIGNING_KEY = cmd_helper(". #{Dir.pwd}/config/amnesia; echo ${AMNESIA_DEV_KEYID}").tr(' ', '').chomp
diff --git a/cucumber/features/support/env.rb b/cucumber/features/support/env.rb
index 53f502e1..c52affff 100644
--- a/cucumber/features/support/env.rb
+++ b/cucumber/features/support/env.rb
@@ -23,6 +23,10 @@ def create_git
   Dir.mkdir 'config'
   FileUtils.touch('config/base_branch')
   Dir.mkdir('config/APT_overlays.d')
+  Dir.mkdir('config/APT_snapshots.d')
+  ['debian', 'debian-security', 'torproject'].map do |origin|
+    Dir.mkdir("config/APT_snapshots.d/#{origin}")
+  end
   Dir.mkdir 'debian'
   File.open('debian/changelog', 'w') do |changelog|
     changelog.write(<<END_OF_CHANGELOG)
@@ -88,3 +92,35 @@ RSpec::Matchers.define :have_suite do |suite|
     "expected an output with #{suite}"
   end
 end
+
+RSpec::Matchers.define :have_tagged_snapshot do |tag|
+  match do |string|
+    # e.g.: `http://tagged.snapshots.deb.tails.boum.org/0.10`
+    %r{^http://tagged\.snapshots\.deb\.tails\.boum\.org/#{Regexp.escape(tag)}/[a-z-]+$}.match(string)
+  end
+  failure_message_for_should do |string|
+    "expected the mirror to be #{tag}\nCurrent mirror: #{string}"
+  end
+  failure_message_for_should_not do |string|
+    "expected the mirror not to be #{tag}\nCurrent mirror: #{string}"
+  end
+  description do
+    "expected an output with #{tag}"
+  end
+end
+
+RSpec::Matchers.define :have_time_based_snapshot do |tag|
+  match do |string|
+    # e.g.: `http://time-based.snapshots.deb.tails.boum.org/debian/2016060602`
+    %r{^http://time\-based\.snapshots\.deb\.tails\.boum\.org/[^/]+/\d+}.match(string)
+  end
+  failure_message_for_should do |string|
+    "expected the mirror to be a time-based snapshot\nCurrent mirror: #{string}"
+  end
+  failure_message_for_should_not do |string|
+    "expected the mirror not to be a time-based snapshot\nCurrent mirror: #{string}"
+  end
+  description do
+    "expected a time-based snapshot"
+  end
+end
diff --git a/cucumber/features/support/extra_hooks.rb b/cucumber/features/support/extra_hooks.rb
index 16196a55..c2c57494 100644
--- a/cucumber/features/support/extra_hooks.rb
+++ b/cucumber/features/support/extra_hooks.rb
@@ -1,18 +1,21 @@
 # Make the code below work with cucumber >= 2.0. Once we stop
 # supporting <2.0 we should probably do this differently, but this way
 # we can easily support both at the same time.
+
 begin
   if not(Cucumber::Core::Ast::Feature.instance_methods.include?(:accept_hook?))
-    require 'gherkin/tag_expression'
+    if Gem::Version.new(Cucumber::VERSION) >= Gem::Version.new('2.4.0')
+      require 'cucumber/core/gherkin/tag_expression'
+    else
+      require 'gherkin/tag_expression'
+      Cucumber::Core::Gherkin = Gherkin
+    end
     class Cucumber::Core::Ast::Feature
       # Code inspired by Cucumber::Core::Test::Case.match_tags?() in
       # cucumber-ruby-core 1.1.3, lib/cucumber/core/test/case.rb:~59.
       def accept_hook?(hook)
-        tag_expr = Gherkin::TagExpression.new(hook.tag_expressions.flatten)
-        tags = @tags.map do |t|
-          Gherkin::Formatter::Model::Tag.new(t.name, t.line)
-        end
-        tag_expr.evaluate(tags)
+        tag_expr = Cucumber::Core::Gherkin::TagExpression.new(hook.tag_expressions.flatten)
+        tag_expr.evaluate(@tags)
       end
     end
   end
@@ -53,10 +56,10 @@ if not($at_exit_print_artifacts_dir_patching_done)
       alias old_print_stats print_stats
     end
     def print_stats(*args)
-      if Dir.exists?(ARTIFACTS_DIR) and Dir.entries(ARTIFACTS_DIR).size > 2
-        @io.puts "Artifacts directory: #{ARTIFACTS_DIR}"
-        @io.puts
-      end
+      @io.puts "Artifacts directory: #{ARTIFACTS_DIR}"
+      @io.puts
+      @io.puts "Debug log:           #{ARTIFACTS_DIR}/debug.log"
+      @io.puts
       if self.class.method_defined?(:old_print_stats)
         old_print_stats(*args)
       end
@@ -74,7 +77,16 @@ def info_log(message = "", options = {})
 end
 
 def debug_log(message, options = {})
-  $debug_log_fns.each { |fn| fn.call(message, options) } if $debug_log_fns
+  options[:timestamp] = true unless options.has_key?(:timestamp)
+  if $debug_log_fns
+    if options[:timestamp]
+      # Force UTC so the local timezone difference vs UTC won't be
+      # added to the result.
+      elapsed = (Time.now - TIME_AT_START.to_f).utc.strftime("%H:%M:%S.%9N")
+      message = "#{elapsed}: #{message}"
+    end
+    $debug_log_fns.each { |fn| fn.call(message, options) }
+  end
 end
 
 require 'cucumber/formatter/pretty'
@@ -104,8 +116,11 @@ module ExtraFormatters
   # anything. We only use it do hook into the correct events so we can
   # add our extra hooks.
   class ExtraHooks
-    def initialize(*args)
+    def initialize(runtime, io, options)
       # We do not care about any of the arguments.
+      # XXX: We should be able to just have `*args` for the arguments
+      # in the prototype, but since moving to cucumber 2.4 that breaks
+      # this formatter for some unknown reason.
     end
 
     def before_feature(feature)
@@ -127,8 +142,8 @@ module ExtraFormatters
 
   # The pretty formatter with debug logging mixed into its output.
   class PrettyDebug < Cucumber::Formatter::Pretty
-    def initialize(*args)
-      super(*args)
+    def initialize(runtime, io, options)
+      super(runtime, io, options)
       $debug_log_fns ||= []
       $debug_log_fns << self.method(:debug_log)
     end
@@ -160,6 +175,13 @@ AfterConfiguration do |config|
   # AfterConfiguration hook multiple times. We only want our
   # ExtraHooks formatter to be loaded once, otherwise the hooks would
   # be run miltiple times.
-  extra_hooks = ['ExtraFormatters::ExtraHooks', '/dev/null']
-  config.formats << extra_hooks if not(config.formats.include?(extra_hooks))
+  extra_hooks = [
+    ['ExtraFormatters::ExtraHooks', '/dev/null'],
+    ['Cucumber::Formatter::Pretty', "#{ARTIFACTS_DIR}/pretty.log"],
+    ['Cucumber::Formatter::Json', "#{ARTIFACTS_DIR}/cucumber.json"],
+    ['ExtraFormatters::PrettyDebug', "#{ARTIFACTS_DIR}/debug.log"],
+  ]
+  extra_hooks.each do |hook|
+    config.formats << hook if not(config.formats.include?(hook))
+  end
 end
diff --git a/cucumber/features/support/helpers/dogtail.rb b/cucumber/features/support/helpers/dogtail.rb
new file mode 100644
index 00000000..2a92649b
--- /dev/null
+++ b/cucumber/features/support/helpers/dogtail.rb
@@ -0,0 +1,233 @@
+module Dogtail
+  module Mouse
+    LEFT_CLICK = 1
+    MIDDLE_CLICK = 2
+    RIGHT_CLICK = 3
+  end
+
+  TREE_API_NODE_SEARCHES = [
+    :button,
+    :child,
+    :childLabelled,
+    :childNamed,
+    :dialog,
+    :menu,
+    :menuItem,
+    :tab,
+    :textentry,
+  ]
+
+  TREE_API_NODE_SEARCH_FIELDS = [
+    :parent,
+  ]
+
+  TREE_API_NODE_ACTIONS = [
+    :click,
+    :doubleClick,
+    :grabFocus,
+    :keyCombo,
+    :point,
+    :typeText,
+  ]
+
+  TREE_API_APP_SEARCHES = TREE_API_NODE_SEARCHES + [
+    :dialog,
+    :window,
+  ]
+
+  # We want to keep this class immutable so that handles always are
+  # left intact when doing new (proxied) method calls.  This way we
+  # can support stuff like:
+  #
+  #     app = Dogtail::Application.new('gedit')
+  #     menu = app.menu('Menu')
+  #     menu.click()
+  #     menu.something_else()
+  #     menu.click()
+  #
+  # i.e. the object referenced by `menu` is never modified by method
+  # calls and can be used as expected.
+
+  class Application
+    @@node_counter ||= 0
+
+    def initialize(app_name, opts = {})
+      @var = "node#{@@node_counter += 1}"
+      @app_name = app_name
+      @opts = opts
+      @opts[:user] ||= LIVE_USER
+      @find_code = "dogtail.tree.root.application('#{@app_name}')"
+      script_lines = [
+        "import dogtail.config",
+        "import dogtail.tree",
+        "import dogtail.predicate",
+        "dogtail.config.logDebugToFile = False",
+        "dogtail.config.logDebugToStdOut = False",
+        "dogtail.config.blinkOnActions = True",
+        "dogtail.config.searchShowingOnly = True",
+        "#{@var} = #{@find_code}",
+      ]
+      run(script_lines)
+    end
+
+    def to_s
+      @var
+    end
+
+    def run(code)
+      code = code.join("\n") if code.class == Array
+      c = RemoteShell::PythonCommand.new($vm, code, user: @opts[:user])
+      if c.failure?
+        raise RuntimeError.new("The Dogtail script raised: #{c.exception}")
+      end
+      return c
+    end
+
+    def child?(*args)
+      !!child(*args)
+    rescue
+      false
+    end
+
+    def exist?
+      run("dogtail.config.searchCutoffCount = 0")
+      run(@find_code)
+      return true
+    rescue
+      return false
+    ensure
+      run("dogtail.config.searchCutoffCount = 20")
+    end
+
+    def self.value_to_s(v)
+      if v == true
+        'True'
+      elsif v == false
+        'False'
+      elsif v.class == String
+        "'#{v}'"
+      elsif [Fixnum, Float].include?(v.class)
+        v.to_s
+      else
+        raise "#{self.class.name} does not know how to handle argument type '#{v.class}'"
+      end
+    end
+
+    # Generates a Python-style parameter list from `args`. If the last
+    # element of `args` is a Hash, it's used as Python's kwargs dict.
+    # In the end, the resulting string should be possible to copy-paste
+    # into the parentheses of a Python function call.
+    # Example: [42, {:foo => 'bar'}] => "42, foo = 'bar'"
+    def self.args_to_s(args)
+      return "" if args.size == 0
+      args_list = args
+      args_hash = nil
+      if args_list.class == Array && args_list.last.class == Hash
+        *args_list, args_hash = args_list
+      end
+      (
+        (args_list.nil? ? [] : args_list.map { |e| self.value_to_s(e) }) +
+        (args_hash.nil? ? [] : args_hash.map { |k, v| "#{k}=#{self.value_to_s(v)}" })
+      ).join(', ')
+    end
+
+    # Equivalent to the Tree API's Node.findChildren(), with the
+    # arguments constructing a GenericPredicate to use as parameter.
+    def children(*args)
+      non_predicates = [:recursive, :showingOnly]
+      findChildren_opts = []
+      findChildren_opts_hash = Hash.new
+      if args.last.class == Hash
+        args_hash = args.last
+        non_predicates.each do |opt|
+          if args_hash.has_key?(opt)
+            findChildren_opts_hash[opt] = args_hash[opt]
+            args_hash.delete(opt)
+          end
+        end
+      end
+      findChildren_opts = ""
+      if findChildren_opts_hash.size > 0
+        findChildren_opts = ", " + self.class.args_to_s([findChildren_opts_hash])
+      end
+      predicate_opts = self.class.args_to_s(args)
+      nodes_var = "nodes#{@@node_counter += 1}"
+      find_script_lines = [
+        "#{nodes_var} = #{@var}.findChildren(dogtail.predicate.GenericPredicate(#{predicate_opts})#{findChildren_opts})",
+        "print(len(#{nodes_var}))",
+      ]
+      size = run(find_script_lines).stdout.chomp.to_i
+      return size.times.map do |i|
+        Node.new("#{nodes_var}[#{i}]", @opts)
+      end
+    end
+
+    def get_field(key)
+      run("print(#{@var}.#{key})").stdout.chomp
+    end
+
+    def set_field(key, value)
+      run("#{@var}.#{key} = #{self.class.value_to_s(value)}")
+    end
+
+    def text
+      get_field('text')
+    end
+
+    def text=(value)
+      set_field('text', value)
+    end
+
+    def name
+      get_field('name')
+    end
+
+    def roleName
+      get_field('roleName')
+    end
+
+    TREE_API_APP_SEARCHES.each do |method|
+      define_method(method) do |*args|
+        args_str = self.class.args_to_s(args)
+        method_call = "#{method.to_s}(#{args_str})"
+        Node.new("#{@var}.#{method_call}", @opts)
+      end
+    end
+
+    TREE_API_NODE_SEARCH_FIELDS.each do |field|
+      define_method(field) do
+        Node.new("#{@var}.#{field}", @opts)
+      end
+    end
+
+  end
+
+  class Node < Application
+
+    def initialize(expr, opts = {})
+      @expr = expr
+      @opts = opts
+      @opts[:user] ||= LIVE_USER
+      @find_code = expr
+      @var = "node#{@@node_counter += 1}"
+      run("#{@var} = #{@find_code}")
+    end
+
+    TREE_API_NODE_SEARCHES.each do |method|
+      define_method(method) do |*args|
+        args_str = self.class.args_to_s(args)
+        method_call = "#{method.to_s}(#{args_str})"
+        Node.new("#{@var}.#{method_call}", @opts)
+      end
+    end
+
+    TREE_API_NODE_ACTIONS.each do |method|
+      define_method(method) do |*args|
+        args_str = self.class.args_to_s(args)
+        method_call = "#{method.to_s}(#{args_str})"
+        run("#{@var}.#{method_call}")
+      end
+    end
+
+  end
+end
diff --git a/cucumber/features/support/helpers/exec_helper.rb b/cucumber/features/support/helpers/exec_helper.rb
deleted file mode 100644
index 70d22d37..00000000
--- a/cucumber/features/support/helpers/exec_helper.rb
+++ /dev/null
@@ -1,90 +0,0 @@
-require 'json'
-require 'socket'
-require 'io/wait'
-
-class VMCommand
-
-  attr_reader :cmd, :returncode, :stdout, :stderr
-
-  def initialize(vm, cmd, options = {})
-    @cmd = cmd
-    @returncode, @stdout, @stderr = VMCommand.execute(vm, cmd, options)
-  end
-
-  def VMCommand.wait_until_remote_shell_is_up(vm, timeout = 180)
-    try_for(timeout, :msg => "Remote shell seems to be down") do
-      Timeout::timeout(20) do
-        VMCommand.execute(vm, "echo 'hello?'")
-      end
-    end
-  end
-
-  # The parameter `cmd` cannot contain newlines. Separate multiple
-  # commands using ";" instead.
-  # If `:spawn` is false the server will block until it has finished
-  # executing `cmd`. If it's true the server won't block, and the
-  # response will always be [0, "", ""] (only used as an
-  # ACK). execute() will always block until a response is received,
-  # though. Spawning is useful when starting processes in the
-  # background (or running scripts that does the same) like our
-  # onioncircuits wrapper, or any application we want to interact with.
-  def VMCommand.execute(vm, cmd, options = {})
-    options[:user] ||= "root"
-    options[:spawn] ||= false
-    type = options[:spawn] ? "spawn" : "call"
-    socket = TCPSocket.new("127.0.0.1", vm.get_remote_shell_port)
-    debug_log("#{type}ing as #{options[:user]}: #{cmd}")
-    begin
-      sleep 0.5
-      while socket.ready?
-        s = socket.recv(1024)
-        debug_log("#{type} pre-exit-debris: #{s}") if not(options[:spawn])
-      end
-      socket.puts( "\nexit\n")
-      sleep 1
-      s = socket.readline(sep = "\007")
-      debug_log("#{type} post-exit-read: #{s}") if not(options[:spawn])
-      while socket.ready?
-        s = socket.recv(1024)
-        debug_log("#{type} post-exit-debris: #{s}") if not(options[:spawn])
-      end
-      socket.puts( cmd + "\n")
-      s = socket.readline(sep = "\000")
-      debug_log("#{type} post-cmd-read: #{s}") if not(options[:spawn])
-      s.chomp!("\000")
-    ensure
-      debug_log("closing the remote-command socket") if not(options[:spawn])
-      socket.close
-    end
-    (s, s_err, x) = s.split("\037")
-    s_err = "" if s_err.nil?
-    (s, s_retcode, y) = s.split("\003")
-    (s, s_out, z) = s.split("\002")
-    s_out = "" if s_out.nil?
-
-    if (s_retcode.to_i.to_s == s_retcode.to_s && x.nil? && y.nil? && z.nil?) then
-      debug_log("returning [returncode=`#{s_retcode.to_i}`,\n\toutput=`#{s_out}`,\n\tstderr=`#{s_err}`]\nwhile discarding `#{s}`.") if not(options[:spawn])
-      return [s_retcode.to_i, s_out, s_err]
-    else
-      debug_log("failed to parse results, retrying\n")
-      return VMCommand.execute(vm, cmd, options)
-    end
-  end
-
-  def success?
-    return @returncode == 0
-  end
-
-  def failure?
-    return not(success?)
-  end
-
-  def to_s
-    "Return status: #{@returncode}\n" +
-    "STDOUT:\n" +
-    @stdout +
-    "STDERR:\n" +
-    @stderr
-  end
-
-end
diff --git a/cucumber/features/support/helpers/firewall_helper.rb b/cucumber/features/support/helpers/firewall_helper.rb
index fce363c5..f88091de 100644
--- a/cucumber/features/support/helpers/firewall_helper.rb
+++ b/cucumber/features/support/helpers/firewall_helper.rb
@@ -1,121 +1,94 @@
 require 'packetfu'
-require 'ipaddr'
 
-# Extent IPAddr with a private/public address space checks
-class IPAddr
-  PrivateIPv4Ranges = [
-    IPAddr.new("10.0.0.0/8"),
-    IPAddr.new("172.16.0.0/12"),
-    IPAddr.new("192.168.0.0/16"),
-    IPAddr.new("255.255.255.255/32")
-  ]
-
-  PrivateIPv6Ranges = [
-    IPAddr.new("fc00::/7")
-  ]
-
-  def private?
-    private_ranges = self.ipv4? ? PrivateIPv4Ranges : PrivateIPv6Ranges
-    private_ranges.any? { |range| range.include?(self) }
-  end
-
-  def public?
-    !private?
-  end
+def looks_like_dhcp_packet?(eth_packet, protocol, sport, dport, ip_packet)
+  protocol == "udp" && sport == 68 && dport == 67 && 
+    eth_packet.eth_daddr == "ff:ff:ff:ff:ff:ff" &&
+    ip_packet && ip_packet.ip_daddr == "255.255.255.255"
 end
 
-class FirewallLeakCheck
-  attr_reader :ipv4_tcp_leaks, :ipv4_nontcp_leaks, :ipv6_leaks, :nonip_leaks, :mac_leaks
-
-  def initialize(pcap_file, options = {})
-    options[:accepted_hosts] ||= []
-    options[:ignore_lan] ||= true
-    @pcap_file = pcap_file
-    packets = PacketFu::PcapFile.new.file_to_array(:filename => @pcap_file)
-    mac_leaks = Set.new
-    ipv4_tcp_packets = []
-    ipv4_nontcp_packets = []
-    ipv6_packets = []
-    nonip_packets = []
-    packets.each do |p|
-      if PacketFu::EthPacket.can_parse?(p)
-        packet = PacketFu::EthPacket.parse(p)
-        mac_leaks << packet.eth_saddr
-        mac_leaks << packet.eth_daddr
-      end
-
-      if PacketFu::TCPPacket.can_parse?(p)
-        ipv4_tcp_packets << PacketFu::TCPPacket.parse(p)
-      elsif PacketFu::IPPacket.can_parse?(p)
-        ipv4_nontcp_packets << PacketFu::IPPacket.parse(p)
-      elsif PacketFu::IPv6Packet.can_parse?(p)
-        ipv6_packets << PacketFu::IPv6Packet.parse(p)
-      elsif PacketFu::Packet.can_parse?(p)
-        nonip_packets << PacketFu::Packet.parse(p)
-      else
-        save_pcap_file
-        raise "Found something in the pcap file that cannot be parsed"
-      end
+# Returns the unique edges (based on protocol, source/destination
+# address/port) in the graph of all network flows.
+def pcap_connections_helper(pcap_file, opts = {})
+  opts[:ignore_dhcp] = true unless opts.has_key?(:ignore_dhcp)
+  connections = Array.new
+  packets = PacketFu::PcapFile.new.file_to_array(:filename => pcap_file)
+  packets.each do |p|
+    if PacketFu::EthPacket.can_parse?(p)
+      eth_packet = PacketFu::EthPacket.parse(p)
+    else
+      raise 'Found something that is not an ethernet packet'
+    end
+    sport = nil
+    dport = nil
+    if PacketFu::IPv6Packet.can_parse?(p)
+      ip_packet = PacketFu::IPv6Packet.parse(p)
+      protocol = 'ipv6'
+    elsif PacketFu::TCPPacket.can_parse?(p)
+      ip_packet = PacketFu::TCPPacket.parse(p)
+      protocol = 'tcp'
+      sport = ip_packet.tcp_sport
+      dport = ip_packet.tcp_dport
+    elsif PacketFu::UDPPacket.can_parse?(p)
+      ip_packet = PacketFu::UDPPacket.parse(p)
+      protocol = 'udp'
+      sport = ip_packet.udp_sport
+      dport = ip_packet.udp_dport
+    elsif PacketFu::ICMPPacket.can_parse?(p)
+      ip_packet = PacketFu::ICMPPacket.parse(p)
+      protocol = 'icmp'
+    elsif PacketFu::IPPacket.can_parse?(p)
+      ip_packet = PacketFu::IPPacket.parse(p)
+      protocol = 'ip'
+    else
+      raise "Found something that cannot be parsed"
     end
-    ipv4_tcp_hosts = filter_hosts_from_ippackets(ipv4_tcp_packets,
-                                                 options[:ignore_lan])
-    accepted = Set.new(options[:accepted_hosts])
-    @mac_leaks = mac_leaks
-    @ipv4_tcp_leaks = ipv4_tcp_hosts.select { |host| !accepted.member?(host) }
-    @ipv4_nontcp_leaks = filter_hosts_from_ippackets(ipv4_nontcp_packets,
-                                                     options[:ignore_lan])
-    @ipv6_leaks = filter_hosts_from_ippackets(ipv6_packets,
-                                              options[:ignore_lan])
-    @nonip_leaks = nonip_packets
-  end
 
-  def save_pcap_file
-    save_failure_artifact("Network capture", @pcap_file)
-  end
+    next if opts[:ignore_dhcp] &&
+            looks_like_dhcp_packet?(eth_packet, protocol,
+                                    sport, dport, ip_packet)
 
-  # Returns a list of all unique destination IP addresses found in
-  # `packets`. Exclude LAN hosts if ignore_lan is set.
-  def filter_hosts_from_ippackets(packets, ignore_lan)
-    hosts = []
-    packets.each do |p|
-      candidate = nil
-      if p.kind_of?(PacketFu::IPPacket)
-        candidate = p.ip_daddr
-      elsif p.kind_of?(PacketFu::IPv6Packet)
-        candidate = p.ipv6_header.ipv6_daddr
-      else
-        save_pcap_file
-        raise "Expected an IP{v4,v6} packet, but got something else:\n" +
-              p.peek_format
-      end
-      if candidate != nil and (not(ignore_lan) or IPAddr.new(candidate).public?)
-        hosts << candidate
+    packet_info = {
+      mac_saddr: eth_packet.eth_saddr,
+      mac_daddr: eth_packet.eth_daddr,
+      protocol: protocol,
+      sport: sport,
+      dport: dport,
+    }
+
+    begin
+      packet_info[:saddr] = ip_packet.ip_saddr
+      packet_info[:daddr] = ip_packet.ip_daddr
+    rescue NoMethodError, NameError
+      begin
+        packet_info[:saddr] = ip_packet.ipv6_saddr
+        packet_info[:daddr] = ip_packet.ipv6_daddr
+      rescue NoMethodError, NameError
+        puts "We were hit by #11508. PacketFu bug? Packet info: #{ip_packet}"
+        packet_info[:saddr] = nil
+        packet_info[:daddr] = nil
       end
     end
-    hosts.uniq
+    connections << packet_info
   end
+  connections.uniq.map { |p| OpenStruct.new(p) }
+end
 
-  def assert_no_leaks
-    err = ""
-    if !@ipv4_tcp_leaks.empty?
-      err += "The following IPv4 TCP non-Tor Internet hosts were " +
-        "contacted:\n" + ipv4_tcp_leaks.join("\n")
-    end
-    if !@ipv4_nontcp_leaks.empty?
-      err += "The following IPv4 non-TCP Internet hosts were contacted:\n" +
-        ipv4_nontcp_leaks.join("\n")
-    end
-    if !@ipv6_leaks.empty?
-      err += "The following IPv6 Internet hosts were contacted:\n" +
-        ipv6_leaks.join("\n")
-    end
-    if !@nonip_leaks.empty?
-      err += "Some non-IP packets were sent\n"
-    end
-    if !err.empty?
-      save_pcap_file
-      raise err
-    end
+class FirewallAssertionFailedError < Test::Unit::AssertionFailedError
+end
+
+# These assertions are made from the perspective of the system under
+# testing when it comes to the concepts of "source" and "destination".
+def assert_all_connections(pcap_file, opts = {}, &block)
+  all = pcap_connections_helper(pcap_file, opts)
+  good = all.find_all(&block)
+  bad = all - good
+  unless bad.empty?
+    raise FirewallAssertionFailedError.new(
+            "Unexpected connections were made:\n" +
+            bad.map { |e| "  #{e}" } .join("\n"))
   end
+end
 
+def assert_no_connections(pcap_file, opts = {}, &block)
+  assert_all_connections(pcap_file, opts) { |*args| not(block.call(*args)) }
 end
diff --git a/cucumber/features/support/helpers/misc_helpers.rb b/cucumber/features/support/helpers/misc_helpers.rb
index 7e09411f..865d2978 100644
--- a/cucumber/features/support/helpers/misc_helpers.rb
+++ b/cucumber/features/support/helpers/misc_helpers.rb
@@ -1,4 +1,6 @@
 require 'date'
+require 'io/console'
+require 'pry'
 require 'timeout'
 require 'test/unit'
 
@@ -28,8 +30,12 @@ end
 
 # Call block (ignoring any exceptions it may throw) repeatedly with
 # one second breaks until it returns true, or until `timeout` seconds have
-# passed when we throw a Timeout::Error exception.
+# passed when we throw a Timeout::Error exception. If `timeout` is `nil`,
+# then we just run the code block with no timeout.
 def try_for(timeout, options = {})
+  if block_given? && timeout.nil?
+    return yield
+  end
   options[:delay] ||= 1
   last_exception = nil
   # Create a unique exception used only for this particular try_for
@@ -76,11 +82,12 @@ def try_for(timeout, options = {})
   # ends up there immediately.
 rescue unique_timeout_exception => e
   msg = options[:msg] || 'try_for() timeout expired'
+  exc_class = options[:exception] || Timeout::Error
   if last_exception
     msg += "\nLast ignored exception was: " +
            "#{last_exception.class}: #{last_exception}"
   end
-  raise Timeout::Error.new(msg)
+  raise exc_class.new(msg)
 end
 
 class TorFailure < StandardError
@@ -89,6 +96,19 @@ end
 class MaxRetriesFailure < StandardError
 end
 
+def force_new_tor_circuit()
+  debug_log("Forcing new Tor circuit...")
+  # Tor rate limits NEWNYM to at most one per 10 second period.
+  interval = 10
+  if $__last_newnym
+    elapsed = Time.now - $__last_newnym
+    # We sleep an extra second to avoid tight timings.
+    sleep interval - elapsed + 1 if 0 < elapsed && elapsed < interval
+  end
+  $vm.execute_successfully('tor_control_send "signal NEWNYM"', :libs => 'tor')
+  $__last_newnym = Time.now
+end
+
 # This will retry the block up to MAX_NEW_TOR_CIRCUIT_RETRIES
 # times. The block must raise an exception for a run to be considered
 # as a failure. After a failure recovery_proc will be called (if
@@ -105,11 +125,6 @@ def retry_tor(recovery_proc = nil, &block)
                :operation_name => 'Tor operation', &block)
 end
 
-def retry_i2p(recovery_proc = nil, &block)
-  retry_action(15, :recovery_proc => recovery_proc,
-               :operation_name => 'I2P operation', &block)
-end
-
 def retry_action(max_retries, options = {}, &block)
   assert(max_retries.is_a?(Integer), "max_retries must be an integer")
   options[:recovery_proc] ||= nil
@@ -120,6 +135,10 @@ def retry_action(max_retries, options = {}, &block)
     begin
       block.call
       return
+    rescue NameError => e
+      # NameError most likely means typos, and hiding that is rarely
+      # (never?) a good idea, so we rethrow them.
+      raise e
     rescue Exception => e
       if retries <= max_retries
         debug_log("#{options[:operation_name]} failed (Try #{retries} of " +
@@ -136,16 +155,15 @@ def retry_action(max_retries, options = {}, &block)
   end
 end
 
+alias :retry_times :retry_action
+
+class TorBootstrapFailure < StandardError
+end
+
 def wait_until_tor_is_working
   try_for(270) { $vm.execute('/usr/local/sbin/tor-has-bootstrapped').success? }
-rescue Timeout::Error => e
-  c = $vm.execute("journalctl SYSLOG_IDENTIFIER=restart-tor")
-  if c.success?
-    debug_log("From the journal:\n" + c.stdout.sub(/^/, "  "))
-  else
-    debug_log("Nothing was in the journal about 'restart-tor'")
-  end
-  raise e
+rescue Timeout::Error
+  raise TorBootstrapFailure.new('Tor failed to bootstrap')
 end
 
 def convert_bytes_mod(unit)
@@ -177,13 +195,14 @@ def convert_from_bytes(size, unit)
   return size.to_f/convert_bytes_mod(unit).to_f
 end
 
-def cmd_helper(cmd)
+def cmd_helper(cmd, env = {})
   if cmd.instance_of?(Array)
     cmd << {:err => [:child, :out]}
   elsif cmd.instance_of?(String)
     cmd += " 2>&1"
   end
-  IO.popen(cmd) do |p|
+  env = ENV.to_h.merge(env)
+  IO.popen(env, cmd) do |p|
     out = p.readlines.join("\n")
     p.close
     ret = $?
@@ -192,11 +211,23 @@ def cmd_helper(cmd)
   end
 end
 
-# This command will grab all router IP addresses from the Tor
-# consensus in the VM + the hardcoded TOR_AUTHORITIES.
-def get_all_tor_nodes
-  cmd = 'awk "/^r/ { print \$6 }" /var/lib/tor/cached-microdesc-consensus'
-  $vm.execute(cmd).stdout.chomp.split("\n") + TOR_AUTHORITIES
+def all_tor_hosts
+  nodes = Array.new
+  chutney_torrcs = Dir.glob(
+    "#{$config['TMPDIR']}/chutney-data/nodes/*/torrc"
+  )
+  chutney_torrcs.each do |torrc|
+    open(torrc) do |f|
+      nodes += f.grep(/^(Or|Dir)Port\b/).map do |line|
+        { address: $vmnet.bridge_ip_addr, port: line.split.last.to_i }
+      end
+    end
+  end
+  return nodes
+end
+
+def allowed_hosts_under_tor_enforcement
+  all_tor_hosts + @lan_hosts
 end
 
 def get_free_space(machine, path)
@@ -246,8 +277,68 @@ def info_log_artifact_location(type, path)
   info_log("#{type.capitalize}: #{path}")
 end
 
+def notify_user(message)
+  alarm_script = $config['NOTIFY_USER_COMMAND']
+  return if alarm_script.nil? || alarm_script.empty?
+  cmd_helper(alarm_script.gsub('%m', message))
+end
+
 def pause(message = "Paused")
+  notify_user(message)
+  STDERR.puts
+  STDERR.puts message
+  # Ring the ASCII bell for a helpful notification in most terminal
+  # emulators.
+  STDOUT.write "\a"
   STDERR.puts
-  STDERR.puts "#{message} (Press ENTER to continue!)"
-  STDIN.gets
+  loop do
+    STDERR.puts "Return: Continue; d: Debugging REPL"
+    c = STDIN.getch
+    case c
+    when "\r"
+      return
+    when "d"
+      binding.pry(quiet: true)
+    end
+  end
+end
+
+def dbus_send(service, object_path, method, *args, **opts)
+  opts ||= {}
+  ruby_type_to_dbus_type = {
+    String => 'string',
+    Fixnum => 'int32',
+  }
+  typed_args = args.map do |arg|
+    type = ruby_type_to_dbus_type[arg.class]
+    assert_not_nil(type, "No DBus type conversion for Ruby type '#{arg.class}'")
+    "#{type}:#{arg}"
+  end
+  ret = $vm.execute_successfully(
+    "dbus-send --print-reply --dest=#{service} #{object_path} " +
+    "    #{method} #{typed_args.join(' ')}",
+    **opts
+  ).stdout.lines
+  # The first line written is about timings and other stuff we don't
+  # care about; we only care about the return values.
+  ret.shift
+  ret.map! do |s|
+    type, val = /^\s*(\S+)\s+(\S+)$/.match(s)[1,2]
+    case type
+    when 'string'
+      # Unquote
+      val[1, val.length - 2]
+    when 'int32'
+      val.to_i
+    else
+      raise "No Ruby type conversion for DBus type '#{type}'"
+    end
+  end
+  if ret.size == 0
+    return nil
+  elsif ret.size == 1
+    return ret.first
+  else
+    return ret
+  end
 end
diff --git a/cucumber/features/support/helpers/remote_shell.rb b/cucumber/features/support/helpers/remote_shell.rb
new file mode 100644
index 00000000..b890578b
--- /dev/null
+++ b/cucumber/features/support/helpers/remote_shell.rb
@@ -0,0 +1,171 @@
+require 'base64'
+require 'json'
+require 'socket'
+require 'timeout'
+
+module RemoteShell
+  class ServerFailure < StandardError
+  end
+
+  # Used to differentiate vs Timeout::Error, which is thrown by
+  # try_for() (by default) and often wraps around remote shell usage
+  # -- in that case we don't want to catch that "outer" exception in
+  # our handling of remote shell timeouts below.
+  class Timeout < ServerFailure
+  end
+
+  DEFAULT_TIMEOUT = 20*60
+
+  # Counter providing unique id:s for each communicate() call.
+  @@request_id ||= 0
+
+  def communicate(vm, *args, **opts)
+    opts[:timeout] ||= DEFAULT_TIMEOUT
+    socket = TCPSocket.new("127.0.0.1", vm.get_remote_shell_port)
+    id = (@@request_id += 1)
+    # Since we already have defined our own Timeout in the current
+    # scope, we have to be more careful when referring to the Timeout
+    # class from the 'timeout' module. However, note that we want it
+    # to throw our own Timeout exception.
+    Object::Timeout.timeout(opts[:timeout], Timeout) do
+      socket.puts(JSON.dump([id] + args))
+      socket.flush
+      loop do
+        line = socket.readline("\n").chomp("\n")
+        response_id, status, *rest = JSON.load(line)
+        if response_id == id
+          if status != "success"
+            if status == "error" and rest.class == Array and rest.size == 1
+              msg = rest.first
+              raise ServerFailure.new("#{msg}")
+            else
+              raise ServerFailure.new("Uncaught exception: #{status}: #{rest}")
+            end
+          end
+          return rest
+        else
+          debug_log("Dropped out-of-order remote shell response: " +
+                    "got id #{response_id} but expected id #{id}")
+        end
+      end
+    end
+  ensure
+    socket.close if defined?(socket) && socket
+  end
+
+  module_function :communicate
+  private :communicate
+
+  class ShellCommand
+    # If `:spawn` is false the server will block until it has finished
+    # executing `cmd`. If it's true the server won't block, and the
+    # response will always be [0, "", ""] (only used as an
+    # ACK). execute() will always block until a response is received,
+    # though. Spawning is useful when starting processes in the
+    # background (or running scripts that does the same) or any
+    # application we want to interact with.
+    def self.execute(vm, cmd, **opts)
+      opts[:user] ||= "root"
+      opts[:spawn] = false unless opts.has_key?(:spawn)
+      type = opts[:spawn] ? "spawn" : "call"
+      debug_log("#{type}ing as #{opts[:user]}: #{cmd}")
+      ret = RemoteShell.communicate(vm, 'sh_' + type, opts[:user], cmd, **opts)
+      debug_log("#{type} returned: #{ret}") if not(opts[:spawn])
+      return ret
+    end
+
+    attr_reader :cmd, :returncode, :stdout, :stderr
+
+    def initialize(vm, cmd, **opts)
+      @cmd = cmd
+      @returncode, @stdout, @stderr = self.class.execute(vm, cmd, **opts)
+    end
+
+    def success?
+      return @returncode == 0
+    end
+
+    def failure?
+      return not(success?)
+    end
+
+    def to_s
+      "Return status: #{@returncode}\n" +
+        "STDOUT:\n" +
+        @stdout +
+        "STDERR:\n" +
+        @stderr
+    end
+  end
+
+  class PythonCommand
+    def self.execute(vm, code, **opts)
+      opts[:user] ||= "root"
+      show_code = code.chomp
+      if show_code["\n"]
+        show_code = "\n" + show_code.lines.map { |l| " "*4 + l.chomp } .join("\n")
+      end
+      debug_log("executing Python as #{opts[:user]}: #{show_code}")
+      ret = RemoteShell.communicate(
+        vm, 'python_execute', opts[:user], code, **opts
+      )
+      debug_log("execution complete")
+      return ret
+    end
+
+    attr_reader :code, :exception, :stdout, :stderr
+
+    def initialize(vm, code, **opts)
+      @code = code
+      @exception, @stdout, @stderr = self.class.execute(vm, code, **opts)
+    end
+
+    def success?
+      return @exception == nil
+    end
+
+    def failure?
+      return not(success?)
+    end
+
+    def to_s
+      "Exception: #{@exception}\n" +
+        "STDOUT:\n" +
+        @stdout +
+        "STDERR:\n" +
+        @stderr
+    end
+  end
+
+  # An IO-like object that is more or less equivalent to a File object
+  # opened in rw mode.
+  class File
+    def self.open(vm, mode, path, *args, **opts)
+      debug_log("opening file #{path} in '#{mode}' mode")
+      ret = RemoteShell.communicate(vm, 'file_' + mode, path, *args, **opts)
+      if ret.size != 1
+        raise ServerFailure.new("expected 1 value but got #{ret.size}")
+      end
+      debug_log("#{mode} complete")
+      return ret.first
+    end
+
+    attr_reader :vm, :path
+
+    def initialize(vm, path)
+      @vm, @path = vm, path
+    end
+
+    def read()
+      Base64.decode64(self.class.open(@vm, 'read', @path))
+    end
+
+    def write(data)
+      self.class.open(@vm, 'write', @path, Base64.encode64(data))
+    end
+
+    def append(data)
+      self.class.open(@vm, 'append', @path, Base64.encode64(data))
+    end
+  end
+end
diff --git a/cucumber/features/support/helpers/sikuli_helper.rb b/cucumber/features/support/helpers/sikuli_helper.rb
index 553abd97..167eded3 100644
--- a/cucumber/features/support/helpers/sikuli_helper.rb
+++ b/cucumber/features/support/helpers/sikuli_helper.rb
@@ -1,9 +1,19 @@
 require 'rjb'
 require 'rjbextension'
 $LOAD_PATH << ENV['SIKULI_HOME']
-require 'sikuli-script.jar'
+begin
+  require 'sikulixapi.jar'
+  USING_SIKULIX = true
+rescue LoadError
+  require 'sikuli-script.jar'
+  USING_SIKULIX = false
+end
 Rjb::load
 
+def using_sikulix?
+  USING_SIKULIX
+end
+
 package_members = [
                    "java.io.FileOutputStream",
                    "java.io.PrintStream",
@@ -16,11 +26,18 @@ package_members = [
                    "org.sikuli.script.Pattern",
                    "org.sikuli.script.Region",
                    "org.sikuli.script.Screen",
-                   "org.sikuli.script.Settings",
                   ]
 
+if using_sikulix?
+  package_members << "org.sikuli.basics.Settings"
+  package_members << "org.sikuli.script.ImagePath"
+else
+  package_members << "org.sikuli.script.Settings"
+end
+
 translations = Hash[
                     "org.sikuli.script", "Sikuli",
+                    "org.sikuli.basics", "Sikuli",
                     "java.lang", "Java::Lang",
                     "java.io", "Java::Io",
                    ]
@@ -186,13 +203,20 @@ def sikuli_script_proxy.new(*args)
   end
 
   def s.hide_cursor
-    self.hover_point(self.w, self.h/2)
+    self.hover_point(self.w - 1, self.h/2)
   end
 
   s
 end
 
 # Configure sikuli
+if using_sikulix?
+  Sikuli::ImagePath.add("#{Dir.pwd}/features/images/")
+else
+  java.lang.System.setProperty("SIKULI_IMAGE_PATH",
+                               "#{Dir.pwd}/features/images/")
+  ENV["SIKULI_IMAGE_PATH"] = "#{Dir.pwd}/features/images/"
+end
 
 # ruby and rjb doesn't play well together when it comes to static
 # fields (and possibly methods) so we instantiate and access the field
@@ -210,5 +234,5 @@ sikuli_settings.MinSimilarity = 0.9
 sikuli_settings.ActionLogs = true
 sikuli_settings.DebugLogs = false
 sikuli_settings.InfoLogs = true
-sikuli_settings.ProfileLogs = true
+sikuli_settings.ProfileLogs = false
 sikuli_settings.WaitScanRate = 0.25
diff --git a/cucumber/features/support/helpers/sniffing_helper.rb b/cucumber/features/support/helpers/sniffing_helper.rb
index 213411eb..38b13820 100644
--- a/cucumber/features/support/helpers/sniffing_helper.rb
+++ b/cucumber/features/support/helpers/sniffing_helper.rb
@@ -22,8 +22,18 @@ class Sniffer
   end
 
   def capture(filter="not ether src host #{@vmnet.bridge_mac} and not ether proto \\arp and not ether proto \\rarp")
-    job = IO.popen(["/usr/sbin/tcpdump", "-n", "-i", @vmnet.bridge_name, "-w",
-                    @pcap_file, "-U", filter, :err => ["/dev/null", "w"]])
+    job = IO.popen(
+      [
+        "/usr/sbin/tcpdump",
+        "-n",
+        "-U",
+        "--immediate-mode",
+        "-i", @vmnet.bridge_name,
+        "-w", @pcap_file,
+        filter,
+        :err => ["/dev/null", "w"]
+      ]
+    )
     @pid = job.pid
   end
 
diff --git a/cucumber/features/support/helpers/storage_helper.rb b/cucumber/features/support/helpers/storage_helper.rb
index de782eed..3bbdb69c 100644
--- a/cucumber/features/support/helpers/storage_helper.rb
+++ b/cucumber/features/support/helpers/storage_helper.rb
@@ -25,7 +25,8 @@ class VMStorage
     rescue Libvirt::RetrieveError
       @pool = nil
     end
-    if @pool and not(KEEP_SNAPSHOTS)
+    if @pool and (not(KEEP_SNAPSHOTS) or
+                  (KEEP_SNAPSHOTS and not(Dir.exists?(@pool_path))))
       VMStorage.clear_storage_pool(@pool)
       @pool = nil
     end
@@ -79,6 +80,10 @@ class VMStorage
     VMStorage.clear_storage_pool_volumes(@pool)
   end
 
+  def list_volumes
+    @pool.list_volumes
+  end
+
   def delete_volume(name)
     @pool.lookup_volume_by_name(name).delete
   end
@@ -144,13 +149,7 @@ class VMStorage
   end
 
   def disk_mklabel(name, parttype)
-    disk = {
-      :path => disk_path(name),
-      :opts => {
-        :format => disk_format(name)
-      }
-    }
-    guestfs_disk_helper(disk) do |g, disk_handle|
+    guestfs_disk_helper(name) do |g, disk_handle|
       g.part_init(disk_handle, parttype)
     end
   end
@@ -158,13 +157,7 @@ class VMStorage
   def disk_mkpartfs(name, parttype, fstype, opts = {})
     opts[:label] ||= nil
     opts[:luks_password] ||= nil
-    disk = {
-      :path => disk_path(name),
-      :opts => {
-        :format => disk_format(name)
-      }
-    }
-    guestfs_disk_helper(disk) do |g, disk_handle|
+    guestfs_disk_helper(name) do |g, disk_handle|
       g.part_disk(disk_handle, parttype)
       g.part_set_name(disk_handle, 1, opts[:label]) if opts[:label]
       primary_partition = g.list_partitions()[0]
@@ -182,13 +175,7 @@ class VMStorage
   end
 
   def disk_mkswap(name, parttype)
-    disk = {
-      :path => disk_path(name),
-      :opts => {
-        :format => disk_format(name)
-      }
-    }
-    guestfs_disk_helper(disk) do |g, disk_handle|
+    guestfs_disk_helper(name) do |g, disk_handle|
       g.part_disk(disk_handle, parttype)
       primary_partition = g.list_partitions()[0]
       g.mkswap(primary_partition)
@@ -206,7 +193,13 @@ class VMStorage
                          Guestfs::EVENT_TRACE)
     g.set_autosync(1)
     disks.each do |disk|
-      g.add_drive_opts(disk[:path], disk[:opts])
+      if disk.class == String
+        g.add_drive_opts(disk_path(disk), format: disk_format(disk))
+      elsif disk.class == Hash
+        g.add_drive_opts(disk[:path], disk[:opts])
+      else
+        raise "cannot handle type '#{disk.class}'"
+      end
     end
     g.launch()
     yield(g, *g.list_devices())
diff --git a/cucumber/features/support/helpers/vm_helper.rb b/cucumber/features/support/helpers/vm_helper.rb
index 5d02c115..be3ae5ff 100644
--- a/cucumber/features/support/helpers/vm_helper.rb
+++ b/cucumber/features/support/helpers/vm_helper.rb
@@ -1,3 +1,4 @@
+require 'ipaddr'
 require 'libvirt'
 require 'rexml/document'
 
@@ -55,11 +56,6 @@ class VMNet
     IPAddr.new(net_xml.elements['network/ip'].attributes['address']).to_s
   end
 
-  def guest_real_mac
-    net_xml = REXML::Document.new(@net.xml_desc)
-    net_xml.elements['network/ip/dhcp/host/'].attributes['mac']
-  end
-
   def bridge_mac
     File.open("/sys/class/net/#{bridge_name}/address", "rb").read.chomp
   end
@@ -68,7 +64,7 @@ end
 
 class VM
 
-  attr_reader :domain, :display, :vmnet, :storage
+  attr_reader :domain, :domain_name, :display, :vmnet, :storage
 
   def initialize(virt, xml_path, vmnet, storage, x_display)
     @virt = virt
@@ -114,8 +110,20 @@ class VM
     end
   end
 
-  def real_mac
-    @vmnet.guest_real_mac
+  def real_mac(alias_name)
+    REXML::Document.new(@domain.xml_desc)
+      .elements["domain/devices/interface[@type='network']/" +
+                "alias[@name='#{alias_name}']"]
+      .parent.elements['mac'].attributes['address'].to_s
+  end
+
+  def all_real_macs
+    macs = []
+    REXML::Document.new(@domain.xml_desc)
+      .elements.each("domain/devices/interface[@type='network']") do |nic|
+      macs << nic.elements['mac'].attributes['address'].to_s
+    end
+    macs
   end
 
   def set_hardware_clock(time)
@@ -131,6 +139,11 @@ class VM
     update(domain_rexml.to_s)
   end
 
+  def network_link_state
+    REXML::Document.new(@domain.xml_desc)
+      .elements['domain/devices/interface/link'].attributes['state']
+  end
+
   def set_network_link_state(state)
     domain_xml = REXML::Document.new(@domain.xml_desc)
     domain_xml.elements['domain/devices/interface/link'].attributes['state'] = state
@@ -158,30 +171,44 @@ class VM
     update(domain_xml.to_s)
   end
 
-  def set_cdrom_image(image)
-    image = nil if image == ''
-    domain_xml = REXML::Document.new(@domain.xml_desc)
-    domain_xml.elements.each('domain/devices/disk') do |e|
-      if e.attribute('device').to_s == "cdrom"
-        if image.nil?
-          e.elements.delete('source')
-        else
-          if ! e.elements['source']
-            e.add_element('source')
-          end
-          e.elements['source'].attributes['file'] = image
-        end
-        if is_running?
-          @domain.update_device(e.to_s)
-        else
-          update(domain_xml.to_s)
-        end
-      end
+  def add_cdrom_device
+    if is_running?
+      raise "Can't attach a CDROM device to a running domain"
+    end
+    domain_rexml = REXML::Document.new(@domain.xml_desc)
+    if domain_rexml.elements["domain/devices/disk[@device='cdrom']"]
+      raise "A CDROM device already exists"
     end
+    cdrom_rexml = REXML::Document.new(File.read("#{@xml_path}/cdrom.xml")).root
+    domain_rexml.elements['domain/devices'].add_element(cdrom_rexml)
+    update(domain_rexml.to_s)
   end
 
-  def remove_cdrom
-    set_cdrom_image(nil)
+  def remove_cdrom_device
+    if is_running?
+      raise "Can't detach a CDROM device to a running domain"
+    end
+    domain_rexml = REXML::Document.new(@domain.xml_desc)
+    cdrom_el = domain_rexml.elements["domain/devices/disk[@device='cdrom']"]
+    if cdrom_el.nil?
+      raise "No CDROM device is present"
+    end
+    domain_rexml.elements["domain/devices"].delete_element(cdrom_el)
+    update(domain_rexml.to_s)
+  end
+
+  def eject_cdrom
+    execute_successfully('/usr/bin/eject -m')
+  end
+
+  def remove_cdrom_image
+    domain_rexml = REXML::Document.new(@domain.xml_desc)
+    cdrom_el = domain_rexml.elements["domain/devices/disk[@device='cdrom']"]
+    if cdrom_el.nil?
+      raise "No CDROM device is present"
+    end
+    cdrom_el.delete_element('source')
+    update(domain_rexml.to_s)
   rescue Libvirt::Error => e
     # While the CD-ROM is removed successfully we still get this
     # error, so let's ignore it.
@@ -192,12 +219,27 @@ class VM
     raise e if not(Regexp.new(acceptable_error).match(e.to_s))
   end
 
+  def set_cdrom_image(image)
+    if image.nil? or image == ''
+      raise "Can't set cdrom image to an empty string"
+    end
+    remove_cdrom_image
+    domain_rexml = REXML::Document.new(@domain.xml_desc)
+    cdrom_el = domain_rexml.elements["domain/devices/disk[@device='cdrom']"]
+    cdrom_el.add_element('source', { 'file' => image })
+    update(domain_rexml.to_s)
+  end
+
   def set_cdrom_boot(image)
     if is_running?
       raise "boot settings can only be set for inactive vms"
     end
-    set_boot_device('cdrom')
+    domain_rexml = REXML::Document.new(@domain.xml_desc)
+    if not domain_rexml.elements["domain/devices/disk[@device='cdrom']"]
+      add_cdrom_device
+    end
     set_cdrom_image(image)
+    set_boot_device('cdrom')
   end
 
   def list_disk_devs
@@ -209,6 +251,16 @@ class VM
     return ret
   end
 
+  def plug_device(xml)
+    if is_running?
+      @domain.attach_device(xml.to_s)
+    else
+      domain_xml = REXML::Document.new(@domain.xml_desc)
+      domain_xml.elements['domain/devices'].add_element(xml)
+      update(domain_xml.to_s)
+    end
+  end
+
   def plug_drive(name, type)
     if disk_plugged?(name)
       raise "disk '#{name}' already plugged"
@@ -238,13 +290,7 @@ class VM
     xml.elements['disk/target'].attributes['bus'] = type
     xml.elements['disk/target'].attributes['removable'] = removable_usb if removable_usb
 
-    if is_running?
-      @domain.attach_device(xml.to_s)
-    else
-      domain_xml = REXML::Document.new(@domain.xml_desc)
-      domain_xml.elements['domain/devices'].add_element(xml)
-      update(domain_xml.to_s)
-    end
+    plug_device(xml)
   end
 
   def disk_xml_desc(name)
@@ -320,9 +366,16 @@ class VM
     end
     plug_drive(name, type) if not(disk_plugged?(name))
     set_boot_device('hd')
-    # For some reason setting the boot device doesn't prevent cdrom
-    # boot unless it's empty
-    remove_cdrom
+    # XXX:Stretch: since our isotesters upgraded QEMU from
+    # 2.5+dfsg-4~bpo8+1 to 2.6+dfsg-3.1~bpo8+1 it seems we must remove
+    # the CDROM device to allow disk boot. This is not the case with the same
+    # version on Debian Sid. Let's hope we can remove this ugly
+    # workaround when we only support running the automated test suite
+    # on Stretch.
+    domain_rexml = REXML::Document.new(@domain.xml_desc)
+    if domain_rexml.elements["domain/devices/disk[@device='cdrom']"]
+      remove_cdrom_device
+    end
   end
 
   # XXX-9p: Shares don't work together with snapshot save+restore. See
@@ -353,59 +406,6 @@ class VM
     return list
   end
 
-  def set_ram_size(size, unit = "KiB")
-    raise "System memory can only be added to inactive vms" if is_running?
-    domain_xml = REXML::Document.new(@domain.xml_desc)
-    domain_xml.elements['domain/memory'].text = size
-    domain_xml.elements['domain/memory'].attributes['unit'] = unit
-    domain_xml.elements['domain/currentMemory'].text = size
-    domain_xml.elements['domain/currentMemory'].attributes['unit'] = unit
-    update(domain_xml.to_s)
-  end
-
-  def get_ram_size_in_bytes
-    domain_xml = REXML::Document.new(@domain.xml_desc)
-    unit = domain_xml.elements['domain/memory'].attribute('unit').to_s
-    size = domain_xml.elements['domain/memory'].text.to_i
-    return convert_to_bytes(size, unit)
-  end
-
-  def set_arch(arch)
-    raise "System architecture can only be set to inactive vms" if is_running?
-    domain_xml = REXML::Document.new(@domain.xml_desc)
-    domain_xml.elements['domain/os/type'].attributes['arch'] = arch
-    update(domain_xml.to_s)
-  end
-
-  def add_hypervisor_feature(feature)
-    raise "Hypervisor features can only be added to inactive vms" if is_running?
-    domain_xml = REXML::Document.new(@domain.xml_desc)
-    domain_xml.elements['domain/features'].add_element(feature)
-    update(domain_xml.to_s)
-  end
-
-  def drop_hypervisor_feature(feature)
-    raise "Hypervisor features can only be fropped from inactive vms" if is_running?
-    domain_xml = REXML::Document.new(@domain.xml_desc)
-    domain_xml.elements['domain/features'].delete_element(feature)
-    update(domain_xml.to_s)
-  end
-
-  def disable_pae_workaround
-    # add_hypervisor_feature("nonpae") results in a libvirt error, and
-    # drop_hypervisor_feature("pae") alone won't disable pae. Hence we
-    # use this workaround.
-    xml = <<EOF
-  <qemu:commandline xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
-    <qemu:arg value='-cpu'/>
-    <qemu:arg value='qemu32,-pae'/>
-  </qemu:commandline>
-EOF
-    domain_xml = REXML::Document.new(@domain.xml_desc)
-    domain_xml.elements['domain'].add_element(REXML::Document.new(xml))
-    update(domain_xml.to_s)
-  end
-
   def set_os_loader(type)
     if is_running?
       raise "boot settings can only be set for inactive vms"
@@ -431,7 +431,7 @@ EOF
 
   def execute(cmd, options = {})
     options[:user] ||= "root"
-    options[:spawn] ||= false
+    options[:spawn] = false unless options.has_key?(:spawn)
     if options[:libs]
       libs = options[:libs]
       options.delete(:libs)
@@ -442,7 +442,7 @@ EOF
       cmds << cmd
       cmd = cmds.join(" && ")
     end
-    return VMCommand.new(self, cmd, options)
+    return RemoteShell::ShellCommand.new(self, cmd, options)
   end
 
   def execute_successfully(*args)
@@ -470,7 +470,9 @@ EOF
   end
 
   def has_network?
-    return execute("/sbin/ifconfig eth0 | grep -q 'inet addr'").success?
+    nmcli_info = execute('nmcli device show eth0').stdout
+    has_ipv4_addr = /^IP4.ADDRESS(\[\d+\])?:\s*([0-9.\/]+)$/.match(nmcli_info)
+    network_link_state == 'up' && has_ipv4_addr
   end
 
   def has_process?(process)
@@ -483,7 +485,7 @@ EOF
 
   def select_virtual_desktop(desktop_number, user = LIVE_USER)
     assert(desktop_number >= 0 && desktop_number <=3,
-           "Only values between 0 and 3 are valid virtual desktop numbers")
+           "Only values between 0 and 1 are valid virtual desktop numbers")
     execute_successfully(
       "xdotool set_desktop '#{desktop_number}'",
       :user => user
@@ -504,11 +506,17 @@ EOF
       # Often when xdotool fails to focus a window it'll work when retried
       # after redrawing the screen.  Switching to a new virtual desktop then
       # back seems to be a reliable way to handle this.
-      select_virtual_desktop(3)
+      # Sadly we have to rely on a lot of sleep() here since there's
+      # little on the screen etc that we truly can rely on.
+      sleep 5
+      select_virtual_desktop(1)
+      sleep 5
       select_virtual_desktop(0)
-      sleep 5 # there aren't any visual indicators which can be used here
+      sleep 5
       do_focus(window_title, user)
     end
+  rescue
+    # noop
   end
 
   def file_exist?(file)
diff --git a/cucumber/features/support/hooks.rb b/cucumber/features/support/hooks.rb
index 1bb6cfd5..a55d361a 100644
--- a/cucumber/features/support/hooks.rb
+++ b/cucumber/features/support/hooks.rb
@@ -14,18 +14,23 @@ AfterConfiguration do |config|
   prioritized_features = [
     # Features not using snapshots but using large amounts of scratch
     # space for other reasons:
-    'features/erase_memory.feature',
     'features/untrusted_partitions.feature',
     # Features using temporary snapshots:
     'features/apt.feature',
-    'features/i2p.feature',
     'features/root_access_control.feature',
     'features/time_syncing.feature',
     'features/tor_bridges.feature',
+    # Features using large amounts of scratch space for other reasons:
+    'features/erase_memory.feature',
     # This feature needs the almost biggest snapshot (USB install,
     # excluding persistence) and will create yet another disk and
     # install Tails on it. This should be the peak of disk usage.
     'features/usb_install.feature',
+    # This feature needs a copy of the ISO and creates a new disk.
+    'features/usb_upgrade.feature',
+    # This feature needs a very big snapshot (USB install with persistence)
+    # and another, network-enabled snapshot.
+    'features/emergency_shutdown.feature',
   ]
   feature_files = config.feature_files
   # The &-intersection is specified to keep the element ordering of
@@ -127,6 +132,21 @@ def save_failure_artifact(type, path)
   $failure_artifacts << [type, path]
 end
 
+# Due to Tails' Tor enforcement, we only allow contacting hosts that
+# are Tor nodes or located on the LAN. However, when we try
+# to verify that only such hosts are contacted we have a problem --
+# we run all Tor nodes (via Chutney) *and* LAN hosts (used on some
+# tests) on the same host, the one running the test suite. Hence we
+# need to always explicitly track which nodes are LAN or not.
+#
+# Warning: when a host is added via this function, it is only added
+# for the current scenario. As such, if this is done before saving a
+# snapshot, it will not remain after the snapshot is loaded.
+def add_lan_host(ipaddr, port)
+  @lan_hosts ||= []
+  @lan_hosts << { address: ipaddr, port: port }
+end
+
 BeforeFeature('@product') do |feature|
   if TAILS_ISO.nil?
     raise "No ISO image specified, and none could be found in the " +
@@ -159,6 +179,7 @@ BeforeFeature('@product') do |feature|
     $vmstorage = VMStorage.new($virt, VM_XML_PATH)
     $started_first_product_feature = true
   end
+  ensure_chutney_is_running
 end
 
 AfterFeature('@product') do
@@ -169,6 +190,10 @@ AfterFeature('@product') do
       end
     end
   end
+  $vmstorage.list_volumes.each do |vol_name|
+    next if vol_name == '__internal'
+    $vmstorage.delete_volume(vol_name)
+  end
 end
 
 # Cucumber Before hooks are executed in the order they are listed, and
@@ -198,6 +223,8 @@ Before('@product') do |scenario|
   @os_loader = "MBR"
   @sudo_password = "asdf"
   @persistence_password = "asdf"
+  # See comment for add_lan_host() above.
+  @lan_hosts ||= []
 end
 
 # Cucumber After hooks are executed in the *reverse* order they are
@@ -224,6 +251,11 @@ After('@product') do |scenario|
     info_log("Scenario failed at time #{elapsed}")
     screen_capture = @screen.capture
     save_failure_artifact("Screenshot", screen_capture.getFilename)
+    if scenario.exception.kind_of?(FirewallAssertionFailedError)
+      Dir.glob("#{$config["TMPDIR"]}/*.pcap").each do |pcap_file|
+        save_failure_artifact("Network capture", pcap_file)
+      end
+    end
     $failure_artifacts.sort!
     $failure_artifacts.each do |type, file|
       artifact_name = sanitize_filename("#{elapsed}_#{scenario.name}#{File.extname(file)}")
@@ -233,7 +265,12 @@ After('@product') do |scenario|
       info_log
       info_log_artifact_location(type, artifact_path)
     end
-    pause("Scenario failed") if $config["PAUSE_ON_FAIL"]
+    if $config["INTERACTIVE_DEBUGGING"]
+      pause(
+        "Scenario failed: #{scenario.name}. " +
+        "The error was: #{scenario.exception.class.name}: #{scenario.exception}"
+      )
+    end
   else
     if @video_path && File.exist?(@video_path) && not($config['CAPTURE_ALL'])
       FileUtils.rm(@video_path)
@@ -252,14 +289,10 @@ end
 After('@product', '@check_tor_leaks') do |scenario|
   @tor_leaks_sniffer.stop
   if scenario.passed?
-    if @bridge_hosts.nil?
-      expected_tor_nodes = get_all_tor_nodes
-    else
-      expected_tor_nodes = @bridge_hosts
+    allowed_nodes = @bridge_hosts ? @bridge_hosts : allowed_hosts_under_tor_enforcement
+    assert_all_connections(@tor_leaks_sniffer.pcap_file) do |c|
+      allowed_nodes.include?({ address: c.daddr, port: c.dport })
     end
-    leaks = FirewallLeakCheck.new(@tor_leaks_sniffer.pcap_file,
-                                  :accepted_hosts => expected_tor_nodes)
-    leaks.assert_no_leaks
   end
 end