Merge pull request #13585 (nixos-tests-splitup)

This splits a few NixOS tests (namely Chromium, VirtualBox and the
networking tests) into several subtests that are exposed via attributes.

The networking tests were already split up but they didn't expose an
attribute set of available tests but used a function attribute to
specify the resulting test instead.

A new function callSubTests in nixos/release.nix is now responsible for
gathering subtests, which is also used for the installer and boot tests.
The latter is now placed in a tests.boot.* namespace rather than
"polluting" the tests attribute set with its subtest.

nixos/release-combined.nix

@@ -48,7 +48,7 @@ in rec {
         (all nixos.ova)
 
         #(all nixos.tests.containers)
-        (all nixos.tests.chromium)
+        (all nixos.tests.chromium.stable)
         (all nixos.tests.firefox)
         (all nixos.tests.firewall)
         nixos.tests.gnome3.x86_64-linux # FIXME: i686-linux
@@ -63,7 +63,7 @@ in rec {
         (all nixos.tests.installer.btrfsSimple)
         (all nixos.tests.installer.btrfsSubvols)
         (all nixos.tests.installer.btrfsSubvolDefault)
-        (all nixos.tests.bootBiosCdrom)
+        (all nixos.tests.boot.biosCdrom)
         (all nixos.tests.ipv6)
         (all nixos.tests.kde4)
         #(all nixos.tests.lightdm)

nixos/release.nix

@@ -13,7 +13,25 @@ let
 
   forAllSystems = genAttrs supportedSystems;
 
-  callTest = fn: args: forAllSystems (system: hydraJob (import fn ({ inherit system; } // args)));
+  importTest = fn: args: system: import fn ({
+    inherit system;
+  } // args);
+
+  callTest = fn: args: forAllSystems (system: hydraJob (importTest fn args system));
+
+  callSubTests = fn: args: let
+    discover = attrs: let
+      subTests = filterAttrs (const (hasAttr "test")) attrs;
+    in mapAttrs (const (t: hydraJob t.test)) subTests;
+
+    discoverForSystem = system: mapAttrs (_: test: {
+      ${system} = test;
+    }) (discover (importTest fn args system));
+
+  # If the test is only for a particular system, use only the specified
+  # system instead of generating attributes for all available systems.
+  in if args ? system then discover (import fn args)
+     else foldAttrs (a: b: a // b) {} (map discoverForSystem supportedSystems);
 
   pkgs = import nixpkgs { system = "x86_64-linux"; };
 
@@ -215,8 +233,9 @@ in rec {
   tests.avahi = callTest tests/avahi.nix {};
   tests.bittorrent = callTest tests/bittorrent.nix {};
   tests.blivet = callTest tests/blivet.nix {};
+  tests.boot = callSubTests tests/boot.nix {};
   tests.cadvisor = hydraJob (import tests/cadvisor.nix { system = "x86_64-linux"; });
-  tests.chromium = callTest tests/chromium.nix {};
+  tests.chromium = callSubTests tests/chromium.nix {};
   tests.cjdns = callTest tests/cjdns.nix {};
   tests.containers = callTest tests/containers.nix {};
   tests.docker = hydraJob (import tests/docker.nix { system = "x86_64-linux"; });
@@ -232,18 +251,7 @@ in rec {
   tests.gnome3-gdm = callTest tests/gnome3-gdm.nix {};
   tests.grsecurity = callTest tests/grsecurity.nix {};
   tests.i3wm = callTest tests/i3wm.nix {};
-  tests.installer.grub1 = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).grub1.test);
+  tests.installer = callSubTests tests/installer.nix {};
-  tests.installer.lvm = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).lvm.test);
-  tests.installer.luksroot = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).luksroot.test);
-  tests.installer.separateBoot = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).separateBoot.test);
-  tests.installer.separateBootFat = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).separateBootFat.test);
-  tests.installer.simple = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).simple.test);
-  tests.installer.simpleLabels = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).simpleLabels.test);
-  tests.installer.simpleProvided = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).simpleProvided.test);
-  tests.installer.swraid = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).swraid.test);
-  tests.installer.btrfsSimple = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).btrfsSimple.test);
-  tests.installer.btrfsSubvols = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).btrfsSubvols.test);
-  tests.installer.btrfsSubvolDefault = forAllSystems (system: hydraJob (import tests/installer.nix { inherit system; }).btrfsSubvolDefault.test);
   tests.influxdb = callTest tests/influxdb.nix {};
   tests.ipv6 = callTest tests/ipv6.nix {};
   tests.jenkins = callTest tests/jenkins.nix {};
@@ -262,24 +270,8 @@ in rec {
   tests.mysqlReplication = callTest tests/mysql-replication.nix {};
   tests.nat.firewall = callTest tests/nat.nix { withFirewall = true; };
   tests.nat.standalone = callTest tests/nat.nix { withFirewall = false; };
-  tests.networking.networkd.loopback = callTest tests/networking.nix { networkd = true; test = "loopback"; };
+  tests.networking.networkd = callSubTests tests/networking.nix { networkd = true; };
-  tests.networking.networkd.static = callTest tests/networking.nix { networkd = true; test = "static"; };
+  tests.networking.scripted = callSubTests tests/networking.nix { networkd = false; };
-  tests.networking.networkd.dhcpSimple = callTest tests/networking.nix { networkd = true; test = "dhcpSimple"; };
-  tests.networking.networkd.dhcpOneIf = callTest tests/networking.nix { networkd = true; test = "dhcpOneIf"; };
-  tests.networking.networkd.bond = callTest tests/networking.nix { networkd = true; test = "bond"; };
-  tests.networking.networkd.bridge = callTest tests/networking.nix { networkd = true; test = "bridge"; };
-  tests.networking.networkd.macvlan = callTest tests/networking.nix { networkd = true; test = "macvlan"; };
-  tests.networking.networkd.sit = callTest tests/networking.nix { networkd = true; test = "sit"; };
-  tests.networking.networkd.vlan = callTest tests/networking.nix { networkd = true; test = "vlan"; };
-  tests.networking.scripted.loopback = callTest tests/networking.nix { networkd = false; test = "loopback"; };
-  tests.networking.scripted.static = callTest tests/networking.nix { networkd = false; test = "static"; };
-  tests.networking.scripted.dhcpSimple = callTest tests/networking.nix { networkd = false; test = "dhcpSimple"; };
-  tests.networking.scripted.dhcpOneIf = callTest tests/networking.nix { networkd = false; test = "dhcpOneIf"; };
-  tests.networking.scripted.bond = callTest tests/networking.nix { networkd = false; test = "bond"; };
-  tests.networking.scripted.bridge = callTest tests/networking.nix { networkd = false; test = "bridge"; };
-  tests.networking.scripted.macvlan = callTest tests/networking.nix { networkd = false; test = "macvlan"; };
-  tests.networking.scripted.sit = callTest tests/networking.nix { networkd = false; test = "sit"; };
-  tests.networking.scripted.vlan = callTest tests/networking.nix { networkd = false; test = "vlan"; };
   # TODO: put in networking.nix after the test becomes more complete
   tests.networkingProxy = callTest tests/networking-proxy.nix {};
   tests.nfs3 = callTest tests/nfs.nix { version = 3; };
@@ -299,12 +291,8 @@ in rec {
   tests.simple = callTest tests/simple.nix {};
   tests.tomcat = callTest tests/tomcat.nix {};
   tests.udisks2 = callTest tests/udisks2.nix {};
-  tests.virtualbox = hydraJob (import tests/virtualbox.nix { system = "x86_64-linux"; });
+  tests.virtualbox = callSubTests tests/virtualbox.nix { system = "x86_64-linux"; };
   tests.xfce = callTest tests/xfce.nix {};
-  tests.bootBiosCdrom = forAllSystems (system: hydraJob (import tests/boot.nix { inherit system; }).bootBiosCdrom);
-  tests.bootBiosUsb = forAllSystems (system: hydraJob (import tests/boot.nix { inherit system; }).bootBiosUsb);
-  tests.bootUefiCdrom = forAllSystems (system: hydraJob (import tests/boot.nix { inherit system; }).bootUefiCdrom);
-  tests.bootUefiUsb = forAllSystems (system: hydraJob (import tests/boot.nix { inherit system; }).bootUefiUsb);
 
 
   /* Build a bunch of typical closures so that Hydra can keep track of

nixos/tests/boot.nix

@@ -30,17 +30,17 @@ let
         '';
     };
 in {
-    bootBiosCdrom = makeBootTest "bios-cdrom" ''
+    biosCdrom = makeBootTest "bios-cdrom" ''
         cdrom => glob("${iso}/iso/*.iso")
       '';
-    bootBiosUsb = makeBootTest "bios-usb" ''
+    biosUsb = makeBootTest "bios-usb" ''
         usb => glob("${iso}/iso/*.iso")
       '';
-    bootUefiCdrom = makeBootTest "uefi-cdrom" ''
+    uefiCdrom = makeBootTest "uefi-cdrom" ''
         cdrom => glob("${iso}/iso/*.iso"),
         bios => '${pkgs.OVMF}/FV/OVMF.fd'
       '';
-    bootUefiUsb = makeBootTest "uefi-usb" ''
+    uefiUsb = makeBootTest "uefi-usb" ''
         usb => glob("${iso}/iso/*.iso"),
         bios => '${pkgs.OVMF}/FV/OVMF.fd'
       '';

nixos/tests/chromium.nix

@@ -1,13 +1,10 @@
-import ./make-test.nix (
+{ system ? builtins.currentSystem }:
-{ pkgs
+
-, channelMap ? {
+with import ../lib/testing.nix { inherit system; };
-    stable = pkgs.chromium;
+with pkgs.lib;
-    #beta   = pkgs.chromiumBeta;
+
-    #dev    = pkgs.chromiumDev;
+mapAttrs (channel: chromiumPkg: makeTest rec {
-  }
+  name = "chromium-${channel}";
-, ...
-}: rec {
-  name = "chromium";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ aszlig ];
   };
@@ -16,6 +13,7 @@ import ./make-test.nix (
 
   machine.imports = [ ./common/x11.nix ];
   machine.virtualisation.memorySize = 2047;
+  machine.environment.systemPackages = [ chromiumPkg ];
 
   startupHTML = pkgs.writeText "chromium-startup.html" ''
     <!DOCTYPE html>
@@ -105,15 +103,13 @@ import ./make-test.nix (
       closeWin;
     }
 
-    sub chromiumTest {
-      my ($channel, $pkg, $code) = @_;
     $machine->waitForX;
 
     my $url = "file://${startupHTML}";
-      my $args = "--user-data-dir=/tmp/chromium-$channel";
+    my $args = "--user-data-dir=/tmp/chromium-${channel}";
     $machine->execute(
       "ulimit -c unlimited; ".
-        "$pkg/bin/chromium $args \"$url\" & disown"
+      "chromium $args \"$url\" & disown"
     );
     $machine->waitForText(qr/Type to search or enter a URL to navigate/);
     $machine->waitUntilSucceeds("${xdo "check-startup" ''
@@ -125,22 +121,11 @@ import ./make-test.nix (
     ''}");
 
     createAndWaitForNewWin;
-      $machine->screenshot($channel."_emptywin");
+    $machine->screenshot("empty_windows");
     closeWin;
 
-      $machine->screenshot($channel."_startup_done");
+    $machine->screenshot("startup_done");
 
-      subtest("Chromium $channel", $code);
-
-      $machine->shutdown;
-    }
-
-    for (${let
-      mkArray = name: pkg: "[\"${name}\", \"${pkg}\"]";
-      chanArrays = pkgs.lib.mapAttrsToList mkArray channelMap;
-    in pkgs.lib.concatStringsSep ", " chanArrays}) {
-      my ($channel, $pkg) = @$_;
-      chromiumTest $channel, $pkg, sub {
     testNewWin "check sandbox", sub {
       $machine->succeed("${xdo "type-url" ''
         search --sync --onlyvisible --name "new tab"
@@ -154,7 +139,7 @@ import ./make-test.nix (
         key --delay 1000 Return
       ''}");
 
-          $machine->screenshot($channel."_sandbox");
+      $machine->screenshot("sandbox_info");
 
       $machine->succeed("${xdo "submit-url" ''
         search --sync --onlyvisible --name "sandbox status"
@@ -172,7 +157,11 @@ import ./make-test.nix (
           && $clipboard =~ /seccomp.*sandbox.*yes/mi
           && $clipboard =~ /you are adequately sandboxed/mi;
     };
-      };
+
-    }
+    $machine->shutdown;
   '';
-})
+}) {
+  stable = pkgs.chromium;
+  beta   = pkgs.chromiumBeta;
+  dev    = pkgs.chromiumDev;
+}

nixos/tests/networking.nix

@@ -1,4 +1,8 @@
-import ./make-test.nix ({ pkgs, networkd, test, ... }:
+{ system ? builtins.currentSystem, networkd }:
+
+with import ../lib/testing.nix { inherit system; };
+with pkgs.lib;
+
 let
   router = { config, pkgs, ... }:
     with pkgs.lib;
@@ -30,6 +34,7 @@ import ./make-test.nix ({ pkgs, networkd, test, ... }:
         '');
       };
     };
+
   testCases = {
     loopback = {
       name = "Loopback";
@@ -397,10 +402,10 @@ import ./make-test.nix ({ pkgs, networkd, test, ... }:
         '';
     };
   };
-    case = testCases.${test};
+
-  in case // {
+in mapAttrs (const (attrs: makeTest (attrs // {
-    name = "${case.name}-Networking-${if networkd then "Networkd" else "Scripted"}";
+  name = "${attrs.name}-Networking-${if networkd then "Networkd" else "Scripted"}";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ wkennington ];
   };
-  })
+}))) testCases

nixos/tests/virtualbox.nix

@@ -1,7 +1,9 @@
-{ debug ? false, ... } @ args:
+{ system ? builtins.currentSystem, debug ? false }:
 
-import ./make-test.nix ({ pkgs, ... }: with pkgs.lib; let
+with import ../lib/testing.nix { inherit system; };
+with pkgs.lib;
 
+let
   testVMConfig = vmName: attrs: { config, pkgs, ... }: let
     guestAdditions = pkgs.linuxPackages.virtualboxGuestAdditions;
 
@@ -314,13 +316,10 @@ import ./make-test.nix ({ pkgs, ... }: with pkgs.lib; let
     test2.vmScript = dhcpScript;
   };
 
-in {
+  mkVBoxTest = name: testScript: makeTest {
-  name = "virtualbox";
+    name = "virtualbox-${name}";
-  meta = with pkgs.stdenv.lib.maintainers; {
-    maintainers = [ aszlig wkennington ];
-  };
 
-  machine = { pkgs, lib, config, ... }: {
+    machine = { lib, config, ... }: {
       imports = let
         mkVMConf = name: val: val.machine // { key = "${name}-config"; };
         vmConfigs = mapAttrsToList mkVMConf vboxVMs;
@@ -342,15 +341,27 @@ in {
         $machine->succeed(ru("VBoxManage ".$_[0]));
       };
 
+      sub removeUUIDs {
+        return join("\n", grep { $_ !~ /^UUID:/ } split(/\n/, $_[0]))."\n";
+      }
+
       ${concatStrings (mapAttrsToList (_: getAttr "testSubs") vboxVMs)}
 
       $machine->waitForX;
 
       ${mkLog "$HOME/.config/VirtualBox/VBoxSVC.log" "HOST-SVC"}
 
-    createVM_simple;
+      ${testScript}
+    '';
 
-    subtest "simple-gui", sub {
+    meta = with pkgs.stdenv.lib.maintainers; {
+      maintainers = [ aszlig wkennington ];
+    };
+  };
+
+in mapAttrs mkVBoxTest {
+  simple-gui = ''
+    createVM_simple;
     $machine->succeed(ru "VirtualBox &");
     $machine->waitForWindow(qr/Oracle VM VirtualBox Manager/);
     $machine->sleep(5);
@@ -369,32 +380,26 @@ in {
     $machine->sendKeys("ctrl-q");
     $machine->sleep(5);
     $machine->screenshot("gui_manager_stopped");
-    };
+  '';
 
-    cleanup_simple;
+  simple-cli = ''
-
+    createVM_simple;
-    subtest "simple-cli", sub {
     vbm("startvm simple");
     waitForStartup_simple;
     $machine->screenshot("cli_started");
     waitForVMBoot_simple;
     $machine->screenshot("cli_booted");
-      shutdownVM_simple;
-    };
 
-    subtest "privilege-escalation", sub {
+    $machine->nest("Checking for privilege escalation", sub {
       $machine->fail("test -e '/root/VirtualBox VMs'");
       $machine->fail("test -e '/root/.config/VirtualBox'");
       $machine->succeed("test -e '/home/alice/VirtualBox VMs'");
-    };
+    });
 
-    destroyVM_simple;
+    shutdownVM_simple;
+  '';
 
-    sub removeUUIDs {
+  host-usb-permissions = ''
-      return join("\n", grep { $_ !~ /^UUID:/ } split(/\n/, $_[0]))."\n";
-    }
-
-    subtest "host-usb-permissions", sub {
     my $userUSB = removeUUIDs vbm("list usbhost");
     print STDERR $userUSB;
     my $rootUSB = removeUUIDs $machine->succeed("VBoxManage list usbhost");
@@ -403,9 +408,9 @@ in {
     die "USB host devices differ for root and normal user"
       if $userUSB ne $rootUSB;
     die "No USB host devices found" if $userUSB =~ /<none>/;
-    };
+  '';
 
-    subtest "systemd-detect-virt", sub {
+  systemd-detect-virt = ''
     createVM_detectvirt;
     vbm("startvm detectvirt");
     waitForStartup_detectvirt;
@@ -416,9 +421,9 @@ in {
     destroyVM_detectvirt;
     die "systemd-detect-virt returned \"$result\" instead of \"oracle\""
       if $result ne "oracle";
-    };
+  '';
 
-    subtest "net-hostonlyif", sub {
+  net-hostonlyif = ''
     createVM_test1;
     createVM_test2;
 
@@ -446,6 +451,5 @@ in {
 
     destroyVM_test1;
     destroyVM_test2;
-    };
   '';
-}) args
+}