Merge pull request #13585 (nixos-tests-splitup)
This splits a few NixOS tests (namely Chromium, VirtualBox and the networking tests) into several subtests that are exposed via attributes. The networking tests were already split up but they didn't expose an attribute set of available tests but used a function attribute to specify the resulting test instead. A new function callSubTests in nixos/release.nix is now responsible for gathering subtests, which is also used for the installer and boot tests. The latter is now placed in a tests.boot.* namespace rather than "polluting" the tests attribute set with its subtest.
This commit is contained in:
aszlig
commit
f70ec0de69
|
|
@ -48,7 +48,7 @@ in rec {
|
|||
(all nixos.ova)
|
||||
|
||||
#(all nixos.tests.containers)
|
||||
(all nixos.tests.chromium)
|
||||
(all nixos.tests.chromium.stable)
|
||||
(all nixos.tests.firefox)
|
||||
(all nixos.tests.firewall)
|
||||
nixos.tests.gnome3.x86_64-linux # FIXME: i686-linux
|
||||
|
|
@ -63,7 +63,7 @@ in rec {
|
|||
(all nixos.tests.installer.btrfsSimple)
|
||||
(all nixos.tests.installer.btrfsSubvols)
|
||||
(all nixos.tests.installer.btrfsSubvolDefault)
|
||||
(all nixos.tests.bootBiosCdrom)
|
||||
(all nixos.tests.boot.biosCdrom)
|
||||
(all nixos.tests.ipv6)
|
||||
(all nixos.tests.kde4)
|
||||
#(all nixos.tests.lightdm)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,25 @@ let
|
|||
|
||||
forAllSystems = genAttrs supportedSystems;
|
||||
|
||||
callTest = fn: args: forAllSystems (system: hydraJob (import fn ({ inherit system; } // args)));
|
||||
importTest = fn: args: system: import fn ({
|
||||
inherit system;
|
||||
} // args);
|
||||
|
||||
callTest = fn: args: forAllSystems (system: hydraJob (importTest fn args system));
|
||||
|
||||
callSubTests = fn: args: let
|
||||
discover = attrs: let
|
||||
subTests = filterAttrs (const (hasAttr "test")) attrs;
|
||||
in mapAttrs (const (t: hydraJob t.test)) subTests;
|
||||
|
||||
discoverForSystem = system: mapAttrs (_: test: {
|
||||
${system} = test;
|
||||
}) (discover (importTest fn args system));
|
||||
|
||||
# If the test is only for a particular system, use only the specified
|
||||
# system instead of generating attributes for all available systems.
|
||||
in if args ? system then discover (import fn args)
|
||||
else foldAttrs (a: b: a // b) {} (map discoverForSystem supportedSystems);
|
||||
|
||||
pkgs = import nixpkgs { system = "x86_64-linux"; };
|
||||
|
||||
|
|
@ -215,8 +233,9 @@ in rec {
|
|||
tests.avahi = callTest tests/avahi.nix {};
|
||||
tests.bittorrent = callTest tests/bittorrent.nix {};
|
||||
tests.blivet = callTest tests/blivet.nix {};
|
||||
tests.boot = callSubTests tests/boot.nix {};
|
||||
tests.cadvisor = hydraJob (import tests/cadvisor.nix { system = "x86_64-linux"; });
|
||||
tests.chromium = callTest tests/chromium.nix {};
|
||||
tests.chromium = callSubTests tests/chromium.nix {};
|
||||
tests.cjdns = callTest tests/cjdns.nix {};
|
||||
tests.containers = callTest tests/containers.nix {};
|
||||
tests.docker = hydraJob (import tests/docker.nix { system = "x86_64-linux"; });
|
||||
|
|
@ -232,18 +251,7 @@ in rec {
|
|||
tests.gnome3-gdm = callTest tests/gnome3-gdm.nix {};
|
||||
tests.grsecurity = callTest tests/grsecurity.nix {};
|
||||
tests.i3wm = callTest tests/i3wm.nix {};
|
||||
tests.installer.grub1 = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).grub1.test);
|
||||
tests.installer.lvm = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).lvm.test);
|
||||
tests.installer.luksroot = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).luksroot.test);
|
||||
tests.installer.separateBoot = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).separateBoot.test);
|
||||
tests.installer.separateBootFat = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).separateBootFat.test);
|
||||
tests.installer.simple = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).simple.test);
|
||||
tests.installer.simpleLabels = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).simpleLabels.test);
|
||||
tests.installer.simpleProvided = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).simpleProvided.test);
|
||||
tests.installer.swraid = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).swraid.test);
|
||||
tests.installer.btrfsSimple = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).btrfsSimple.test);
|
||||
tests.installer.btrfsSubvols = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).btrfsSubvols.test);
|
||||
tests.installer.btrfsSubvolDefault = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).btrfsSubvolDefault.test);
|
||||
tests.installer = callSubTests tests/installer.nix {};
|
||||
tests.influxdb = callTest tests/influxdb.nix {};
|
||||
tests.ipv6 = callTest tests/ipv6.nix {};
|
||||
tests.jenkins = callTest tests/jenkins.nix {};
|
||||
|
|
@ -262,24 +270,8 @@ in rec {
|
|||
tests.mysqlReplication = callTest tests/mysql-replication.nix {};
|
||||
tests.nat.firewall = callTest tests/nat.nix { withFirewall = true; };
|
||||
tests.nat.standalone = callTest tests/nat.nix { withFirewall = false; };
|
||||
tests.networking.networkd.loopback = callTest tests/networking.nix { networkd = true; test = "loopback"; };
|
||||
tests.networking.networkd.static = callTest tests/networking.nix { networkd = true; test = "static"; };
|
||||
tests.networking.networkd.dhcpSimple = callTest tests/networking.nix { networkd = true; test = "dhcpSimple"; };
|
||||
tests.networking.networkd.dhcpOneIf = callTest tests/networking.nix { networkd = true; test = "dhcpOneIf"; };
|
||||
tests.networking.networkd.bond = callTest tests/networking.nix { networkd = true; test = "bond"; };
|
||||
tests.networking.networkd.bridge = callTest tests/networking.nix { networkd = true; test = "bridge"; };
|
||||
tests.networking.networkd.macvlan = callTest tests/networking.nix { networkd = true; test = "macvlan"; };
|
||||
tests.networking.networkd.sit = callTest tests/networking.nix { networkd = true; test = "sit"; };
|
||||
tests.networking.networkd.vlan = callTest tests/networking.nix { networkd = true; test = "vlan"; };
|
||||
tests.networking.scripted.loopback = callTest tests/networking.nix { networkd = false; test = "loopback"; };
|
||||
tests.networking.scripted.static = callTest tests/networking.nix { networkd = false; test = "static"; };
|
||||
tests.networking.scripted.dhcpSimple = callTest tests/networking.nix { networkd = false; test = "dhcpSimple"; };
|
||||
tests.networking.scripted.dhcpOneIf = callTest tests/networking.nix { networkd = false; test = "dhcpOneIf"; };
|
||||
tests.networking.scripted.bond = callTest tests/networking.nix { networkd = false; test = "bond"; };
|
||||
tests.networking.scripted.bridge = callTest tests/networking.nix { networkd = false; test = "bridge"; };
|
||||
tests.networking.scripted.macvlan = callTest tests/networking.nix { networkd = false; test = "macvlan"; };
|
||||
tests.networking.scripted.sit = callTest tests/networking.nix { networkd = false; test = "sit"; };
|
||||
tests.networking.scripted.vlan = callTest tests/networking.nix { networkd = false; test = "vlan"; };
|
||||
tests.networking.networkd = callSubTests tests/networking.nix { networkd = true; };
|
||||
tests.networking.scripted = callSubTests tests/networking.nix { networkd = false; };
|
||||
# TODO: put in networking.nix after the test becomes more complete
|
||||
tests.networkingProxy = callTest tests/networking-proxy.nix {};
|
||||
tests.nfs3 = callTest tests/nfs.nix { version = 3; };
|
||||
|
|
@ -299,12 +291,8 @@ in rec {
|
|||
tests.simple = callTest tests/simple.nix {};
|
||||
tests.tomcat = callTest tests/tomcat.nix {};
|
||||
tests.udisks2 = callTest tests/udisks2.nix {};
|
||||
tests.virtualbox = hydraJob (import tests/virtualbox.nix { system = "x86_64-linux"; });
|
||||
tests.virtualbox = callSubTests tests/virtualbox.nix { system = "x86_64-linux"; };
|
||||
tests.xfce = callTest tests/xfce.nix {};
|
||||
tests.bootBiosCdrom = forAllSystems (system: hydraJob (import tests/boot.nix { inherit system; }).bootBiosCdrom);
|
||||
tests.bootBiosUsb = forAllSystems (system: hydraJob (import tests/boot.nix { inherit system; }).bootBiosUsb);
|
||||
tests.bootUefiCdrom = forAllSystems (system: hydraJob (import tests/boot.nix { inherit system; }).bootUefiCdrom);
|
||||
tests.bootUefiUsb = forAllSystems (system: hydraJob (import tests/boot.nix { inherit system; }).bootUefiUsb);
|
||||
|
||||
|
||||
/* Build a bunch of typical closures so that Hydra can keep track of
|
||||
|
|
|
|||
|
|
@ -30,17 +30,17 @@ let
|
|||
'';
|
||||
};
|
||||
in {
|
||||
bootBiosCdrom = makeBootTest "bios-cdrom" ''
|
||||
biosCdrom = makeBootTest "bios-cdrom" ''
|
||||
cdrom => glob("${iso}/iso/*.iso")
|
||||
'';
|
||||
bootBiosUsb = makeBootTest "bios-usb" ''
|
||||
biosUsb = makeBootTest "bios-usb" ''
|
||||
usb => glob("${iso}/iso/*.iso")
|
||||
'';
|
||||
bootUefiCdrom = makeBootTest "uefi-cdrom" ''
|
||||
uefiCdrom = makeBootTest "uefi-cdrom" ''
|
||||
cdrom => glob("${iso}/iso/*.iso"),
|
||||
bios => '${pkgs.OVMF}/FV/OVMF.fd'
|
||||
'';
|
||||
bootUefiUsb = makeBootTest "uefi-usb" ''
|
||||
uefiUsb = makeBootTest "uefi-usb" ''
|
||||
usb => glob("${iso}/iso/*.iso"),
|
||||
bios => '${pkgs.OVMF}/FV/OVMF.fd'
|
||||
'';
|
||||
|
|
|
|||
|
|
@ -1,13 +1,10 @@
|
|||
import ./make-test.nix (
|
||||
{ pkgs
|
||||
, channelMap ? {
|
||||
stable = pkgs.chromium;
|
||||
#beta = pkgs.chromiumBeta;
|
||||
#dev = pkgs.chromiumDev;
|
||||
}
|
||||
, ...
|
||||
}: rec {
|
||||
name = "chromium";
|
||||
{ system ? builtins.currentSystem }:
|
||||
|
||||
with import ../lib/testing.nix { inherit system; };
|
||||
with pkgs.lib;
|
||||
|
||||
mapAttrs (channel: chromiumPkg: makeTest rec {
|
||||
name = "chromium-${channel}";
|
||||
meta = with pkgs.stdenv.lib.maintainers; {
|
||||
maintainers = [ aszlig ];
|
||||
};
|
||||
|
|
@ -16,6 +13,7 @@ import ./make-test.nix (
|
|||
|
||||
machine.imports = [ ./common/x11.nix ];
|
||||
machine.virtualisation.memorySize = 2047;
|
||||
machine.environment.systemPackages = [ chromiumPkg ];
|
||||
|
||||
startupHTML = pkgs.writeText "chromium-startup.html" ''
|
||||
<!DOCTYPE html>
|
||||
|
|
@ -105,15 +103,13 @@ import ./make-test.nix (
|
|||
closeWin;
|
||||
}
|
||||
|
||||
sub chromiumTest {
|
||||
my ($channel, $pkg, $code) = @_;
|
||||
$machine->waitForX;
|
||||
|
||||
my $url = "file://${startupHTML}";
|
||||
my $args = "--user-data-dir=/tmp/chromium-$channel";
|
||||
my $args = "--user-data-dir=/tmp/chromium-${channel}";
|
||||
$machine->execute(
|
||||
"ulimit -c unlimited; ".
|
||||
"$pkg/bin/chromium $args \"$url\" & disown"
|
||||
"chromium $args \"$url\" & disown"
|
||||
);
|
||||
$machine->waitForText(qr/Type to search or enter a URL to navigate/);
|
||||
$machine->waitUntilSucceeds("${xdo "check-startup" ''
|
||||
|
|
@ -125,22 +121,11 @@ import ./make-test.nix (
|
|||
''}");
|
||||
|
||||
createAndWaitForNewWin;
|
||||
$machine->screenshot($channel."_emptywin");
|
||||
$machine->screenshot("empty_windows");
|
||||
closeWin;
|
||||
|
||||
$machine->screenshot($channel."_startup_done");
|
||||
$machine->screenshot("startup_done");
|
||||
|
||||
subtest("Chromium $channel", $code);
|
||||
|
||||
$machine->shutdown;
|
||||
}
|
||||
|
||||
for (${let
|
||||
mkArray = name: pkg: "[\"${name}\", \"${pkg}\"]";
|
||||
chanArrays = pkgs.lib.mapAttrsToList mkArray channelMap;
|
||||
in pkgs.lib.concatStringsSep ", " chanArrays}) {
|
||||
my ($channel, $pkg) = @$_;
|
||||
chromiumTest $channel, $pkg, sub {
|
||||
testNewWin "check sandbox", sub {
|
||||
$machine->succeed("${xdo "type-url" ''
|
||||
search --sync --onlyvisible --name "new tab"
|
||||
|
|
@ -154,7 +139,7 @@ import ./make-test.nix (
|
|||
key --delay 1000 Return
|
||||
''}");
|
||||
|
||||
$machine->screenshot($channel."_sandbox");
|
||||
$machine->screenshot("sandbox_info");
|
||||
|
||||
$machine->succeed("${xdo "submit-url" ''
|
||||
search --sync --onlyvisible --name "sandbox status"
|
||||
|
|
@ -172,7 +157,11 @@ import ./make-test.nix (
|
|||
&& $clipboard =~ /seccomp.*sandbox.*yes/mi
|
||||
&& $clipboard =~ /you are adequately sandboxed/mi;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
$machine->shutdown;
|
||||
'';
|
||||
})
|
||||
}) {
|
||||
stable = pkgs.chromium;
|
||||
beta = pkgs.chromiumBeta;
|
||||
dev = pkgs.chromiumDev;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
import ./make-test.nix ({ pkgs, networkd, test, ... }:
|
||||
let
|
||||
{ system ? builtins.currentSystem, networkd }:
|
||||
|
||||
with import ../lib/testing.nix { inherit system; };
|
||||
with pkgs.lib;
|
||||
|
||||
let
|
||||
router = { config, pkgs, ... }:
|
||||
with pkgs.lib;
|
||||
let
|
||||
|
|
@ -30,6 +34,7 @@ import ./make-test.nix ({ pkgs, networkd, test, ... }:
|
|||
'');
|
||||
};
|
||||
};
|
||||
|
||||
testCases = {
|
||||
loopback = {
|
||||
name = "Loopback";
|
||||
|
|
@ -397,10 +402,10 @@ import ./make-test.nix ({ pkgs, networkd, test, ... }:
|
|||
'';
|
||||
};
|
||||
};
|
||||
case = testCases.${test};
|
||||
in case // {
|
||||
name = "${case.name}-Networking-${if networkd then "Networkd" else "Scripted"}";
|
||||
|
||||
in mapAttrs (const (attrs: makeTest (attrs // {
|
||||
name = "${attrs.name}-Networking-${if networkd then "Networkd" else "Scripted"}";
|
||||
meta = with pkgs.stdenv.lib.maintainers; {
|
||||
maintainers = [ wkennington ];
|
||||
};
|
||||
})
|
||||
}))) testCases
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
{ debug ? false, ... } @ args:
|
||||
{ system ? builtins.currentSystem, debug ? false }:
|
||||
|
||||
import ./make-test.nix ({ pkgs, ... }: with pkgs.lib; let
|
||||
with import ../lib/testing.nix { inherit system; };
|
||||
with pkgs.lib;
|
||||
|
||||
let
|
||||
testVMConfig = vmName: attrs: { config, pkgs, ... }: let
|
||||
guestAdditions = pkgs.linuxPackages.virtualboxGuestAdditions;
|
||||
|
||||
|
|
@ -314,13 +316,10 @@ import ./make-test.nix ({ pkgs, ... }: with pkgs.lib; let
|
|||
test2.vmScript = dhcpScript;
|
||||
};
|
||||
|
||||
in {
|
||||
name = "virtualbox";
|
||||
meta = with pkgs.stdenv.lib.maintainers; {
|
||||
maintainers = [ aszlig wkennington ];
|
||||
};
|
||||
mkVBoxTest = name: testScript: makeTest {
|
||||
name = "virtualbox-${name}";
|
||||
|
||||
machine = { pkgs, lib, config, ... }: {
|
||||
machine = { lib, config, ... }: {
|
||||
imports = let
|
||||
mkVMConf = name: val: val.machine // { key = "${name}-config"; };
|
||||
vmConfigs = mapAttrsToList mkVMConf vboxVMs;
|
||||
|
|
@ -342,15 +341,27 @@ in {
|
|||
$machine->succeed(ru("VBoxManage ".$_[0]));
|
||||
};
|
||||
|
||||
sub removeUUIDs {
|
||||
return join("\n", grep { $_ !~ /^UUID:/ } split(/\n/, $_[0]))."\n";
|
||||
}
|
||||
|
||||
${concatStrings (mapAttrsToList (_: getAttr "testSubs") vboxVMs)}
|
||||
|
||||
$machine->waitForX;
|
||||
|
||||
${mkLog "$HOME/.config/VirtualBox/VBoxSVC.log" "HOST-SVC"}
|
||||
|
||||
createVM_simple;
|
||||
${testScript}
|
||||
'';
|
||||
|
||||
subtest "simple-gui", sub {
|
||||
meta = with pkgs.stdenv.lib.maintainers; {
|
||||
maintainers = [ aszlig wkennington ];
|
||||
};
|
||||
};
|
||||
|
||||
in mapAttrs mkVBoxTest {
|
||||
simple-gui = ''
|
||||
createVM_simple;
|
||||
$machine->succeed(ru "VirtualBox &");
|
||||
$machine->waitForWindow(qr/Oracle VM VirtualBox Manager/);
|
||||
$machine->sleep(5);
|
||||
|
|
@ -369,32 +380,26 @@ in {
|
|||
$machine->sendKeys("ctrl-q");
|
||||
$machine->sleep(5);
|
||||
$machine->screenshot("gui_manager_stopped");
|
||||
};
|
||||
'';
|
||||
|
||||
cleanup_simple;
|
||||
|
||||
subtest "simple-cli", sub {
|
||||
simple-cli = ''
|
||||
createVM_simple;
|
||||
vbm("startvm simple");
|
||||
waitForStartup_simple;
|
||||
$machine->screenshot("cli_started");
|
||||
waitForVMBoot_simple;
|
||||
$machine->screenshot("cli_booted");
|
||||
shutdownVM_simple;
|
||||
};
|
||||
|
||||
subtest "privilege-escalation", sub {
|
||||
$machine->nest("Checking for privilege escalation", sub {
|
||||
$machine->fail("test -e '/root/VirtualBox VMs'");
|
||||
$machine->fail("test -e '/root/.config/VirtualBox'");
|
||||
$machine->succeed("test -e '/home/alice/VirtualBox VMs'");
|
||||
};
|
||||
});
|
||||
|
||||
destroyVM_simple;
|
||||
shutdownVM_simple;
|
||||
'';
|
||||
|
||||
sub removeUUIDs {
|
||||
return join("\n", grep { $_ !~ /^UUID:/ } split(/\n/, $_[0]))."\n";
|
||||
}
|
||||
|
||||
subtest "host-usb-permissions", sub {
|
||||
host-usb-permissions = ''
|
||||
my $userUSB = removeUUIDs vbm("list usbhost");
|
||||
print STDERR $userUSB;
|
||||
my $rootUSB = removeUUIDs $machine->succeed("VBoxManage list usbhost");
|
||||
|
|
@ -403,9 +408,9 @@ in {
|
|||
die "USB host devices differ for root and normal user"
|
||||
if $userUSB ne $rootUSB;
|
||||
die "No USB host devices found" if $userUSB =~ /<none>/;
|
||||
};
|
||||
'';
|
||||
|
||||
subtest "systemd-detect-virt", sub {
|
||||
systemd-detect-virt = ''
|
||||
createVM_detectvirt;
|
||||
vbm("startvm detectvirt");
|
||||
waitForStartup_detectvirt;
|
||||
|
|
@ -416,9 +421,9 @@ in {
|
|||
destroyVM_detectvirt;
|
||||
die "systemd-detect-virt returned \"$result\" instead of \"oracle\""
|
||||
if $result ne "oracle";
|
||||
};
|
||||
'';
|
||||
|
||||
subtest "net-hostonlyif", sub {
|
||||
net-hostonlyif = ''
|
||||
createVM_test1;
|
||||
createVM_test2;
|
||||
|
||||
|
|
@ -446,6 +451,5 @@ in {
|
|||
|
||||
destroyVM_test1;
|
||||
destroyVM_test2;
|
||||
};
|
||||
'';
|
||||
}) args
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue