Merge pull request #50233 from Ekleog/fix-test-system

Fix meta.tests system

doc/meta.xml

@@ -255,12 +255,22 @@ meta.platforms = stdenv.lib.platforms.linux;
      <varname>tests</varname>
     </term>
     <listitem>
+     <warning>
+      <para>
+       This attribute is special in that it is not actually under the
+       <literal>meta</literal> attribute set but rather under the
+       <literal>passthru</literal> attribute set. This is due to a current
+       limitation of Nix, and will change as soon as Nixpkgs will be able to
+       depend on a new enough version of Nix. See
+       <link xlink:href="https://github.com/NixOS/nix/issues/2532">the relevant
+       issue</link> for more details.
+      </para>
+     </warning>
      <para>
       An attribute set with as values tests. A test is a derivation, which
       builds successfully when the test passes, and fails to build otherwise. A
-      derivation that is a test requires some <literal>meta</literal> elements
+      derivation that is a test needs to have <literal>meta.timeout</literal>
-      to be defined: <literal>needsVMSupport</literal> (automatically filled-in
+      defined.
-      for NixOS tests) and <literal>timeout</literal>.
      </para>
      <para>
       The NixOS tests are available as <literal>nixosTests</literal> in
@@ -270,7 +280,7 @@ meta.platforms = stdenv.lib.platforms.linux;
 { /* ... */, nixosTests }:
 {
   # ...
-  meta.tests = {
+  passthru.tests = {
     basic-functionality-and-dovecot-integration = nixosTests.opensmtpd;
   };
 }
@@ -294,17 +304,6 @@ meta.platforms = stdenv.lib.platforms.linux;
      </para>
     </listitem>
    </varlistentry>
-   <varlistentry>
-    <term>
-     <varname>needsVMSupport</varname>
-    </term>
-    <listitem>
-     <para>
-      A boolan that states whether the derivation requires build-time support
-      for Virtual Machine to build successfully.
-     </para>
-    </listitem>
-   </varlistentry>
    <varlistentry>
     <term>
      <varname>hydraPlatforms</varname>

nixos/lib/build-vms.nix

@@ -1,6 +1,4 @@
-{ system, minimal ? false, config ? {} }:
+{ system, pkgs, minimal ? false, config ? {} }:
-
-let pkgs = import ../.. { inherit system config; }; in
 
 with pkgs.lib;
 with import ../lib/qemu-flags.nix { inherit pkgs; };

nixos/lib/testing.nix

@@ -1,6 +1,6 @@
-{ system, minimal ? false, config ? {} }:
+{ system, pkgs, minimal ? false, config ? {} }:
 
-with import ./build-vms.nix { inherit system minimal config; };
+with import ./build-vms.nix { inherit system pkgs minimal config; };
 with pkgs;
 
 let
@@ -70,8 +70,6 @@ in rec {
             mv $i $out/coverage-data/$(dirname $(dirname $i))
           done
         '';
-
-        meta.needsVMSupport = true;
     };
 
 

nixos/modules/installer/tools/nixos-build-vms/build-vms.nix

@@ -1,9 +1,13 @@
 { system ? builtins.currentSystem
+, config ? {}
 , networkExpr
 }:
 
 let nodes = import networkExpr; in
 
-with import ../../../../lib/testing.nix { inherit system; };
+with import ../../../../lib/testing.nix {
+  inherit system;
+  pkgs = import ../.. { inherit system config; }
+};
 
 (makeTest { inherit nodes; testScript = ""; }).driver

nixos/release.nix

@@ -14,28 +14,19 @@ let
   versionSuffix =
     (if stableBranch then "." else "pre") + "${toString nixpkgs.revCount}.${nixpkgs.shortRev}";
 
-  importTest = fn: args: system: import fn ({
+  # Run the tests for each platform.  You can run a test by doing
+  # e.g. ‘nix-build -A tests.login.x86_64-linux’, or equivalently,
+  # ‘nix-build tests/login.nix -A result’.
+  allTestsForSystem = system:
+    import ./tests/all-tests.nix {
       inherit system;
-  } // args);
+      pkgs = import nixpkgs { inherit system; };
-
+      callTest = t: {
-  # Note: only supportedSystems are considered.
+        ${system} = hydraJob t.test;
-  callTestOnMatchingSystems = systems: fn: args:
+      };
-    forMatchingSystems
+    };
-      (intersectLists supportedSystems systems)
+  allTests =
-      (system: hydraJob (importTest fn args system));
+    foldAttrs recursiveUpdate {} (map allTestsForSystem supportedSystems);
-  callTest = callTestOnMatchingSystems supportedSystems;
-
-  callSubTests = callSubTestsOnMatchingSystems supportedSystems;
-  callSubTestsOnMatchingSystems = systems: fn: args: let
-    discover = attrs: let
-      subTests = filterAttrs (const (hasAttr "test")) attrs;
-    in mapAttrs (const (t: hydraJob t.test)) subTests;
-
-    discoverForSystem = system: mapAttrs (_: test: {
-      ${system} = test;
-    }) (discover (importTest fn args system));
-
-  in foldAttrs mergeAttrs {} (map discoverForSystem (intersectLists systems supportedSystems));
 
   pkgs = import nixpkgs { system = "x86_64-linux"; };
 
@@ -245,200 +236,7 @@ in rec {
     };
   */
 
-
+  tests = allTests;
-  # Run the tests for each platform.  You can run a test by doing
-  # e.g. ‘nix-build -A tests.login.x86_64-linux’, or equivalently,
-  # ‘nix-build tests/login.nix -A result’.
-  tests.atd = callTest tests/atd.nix {};
-  tests.acme = callTest tests/acme.nix {};
-  tests.avahi = callTest tests/avahi.nix {};
-  tests.beegfs = callTest tests/beegfs.nix {};
-  tests.upnp = callTest tests/upnp.nix {};
-  tests.bittorrent = callTest tests/bittorrent.nix {};
-  tests.bind = callTest tests/bind.nix {};
-  #tests.blivet = callTest tests/blivet.nix {};   # broken since 2017-07024
-  tests.boot = callSubTests tests/boot.nix {};
-  tests.boot-stage1 = callTest tests/boot-stage1.nix {};
-  tests.borgbackup = callTest tests/borgbackup.nix {};
-  tests.buildbot = callSubTests tests/buildbot.nix {};
-  tests.cadvisor = callTestOnMatchingSystems ["x86_64-linux"] tests/cadvisor.nix {};
-  tests.ceph = callTestOnMatchingSystems ["x86_64-linux"] tests/ceph.nix {};
-  tests.certmgr = callSubTests tests/certmgr.nix {};
-  tests.cfssl = callTestOnMatchingSystems ["x86_64-linux"] tests/cfssl.nix {};
-  tests.chromium = (callSubTestsOnMatchingSystems ["x86_64-linux"] tests/chromium.nix {}).stable or {};
-  tests.cjdns = callTest tests/cjdns.nix {};
-  tests.cloud-init = callTest tests/cloud-init.nix {};
-  tests.codimd = callTest tests/codimd.nix {};
-  tests.containers-ipv4 = callTest tests/containers-ipv4.nix {};
-  tests.containers-ipv6 = callTest tests/containers-ipv6.nix {};
-  tests.containers-bridge = callTest tests/containers-bridge.nix {};
-  tests.containers-imperative = callTest tests/containers-imperative.nix {};
-  tests.containers-extra_veth = callTest tests/containers-extra_veth.nix {};
-  tests.containers-physical_interfaces = callTest tests/containers-physical_interfaces.nix {};
-  tests.containers-restart_networking = callTest tests/containers-restart_networking.nix {};
-  tests.containers-tmpfs = callTest tests/containers-tmpfs.nix {};
-  tests.containers-hosts = callTest tests/containers-hosts.nix {};
-  tests.containers-macvlans = callTest tests/containers-macvlans.nix {};
-  tests.couchdb = callTest tests/couchdb.nix {};
-  tests.deluge = callTest tests/deluge.nix {};
-  tests.dhparams = callTest tests/dhparams.nix {};
-  tests.docker = callTestOnMatchingSystems ["x86_64-linux"] tests/docker.nix {};
-  tests.docker-tools = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-tools.nix {};
-  tests.docker-tools-overlay = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-tools-overlay.nix {};
-  tests.docker-edge = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-edge.nix {};
-  tests.docker-preloader = callTestOnMatchingSystems ["x86_64-linux"] tests/docker-preloader.nix {};
-  tests.docker-registry = callTest tests/docker-registry.nix {};
-  tests.dovecot = callTest tests/dovecot.nix {};
-  tests.dnscrypt-proxy = callTestOnMatchingSystems ["x86_64-linux"] tests/dnscrypt-proxy.nix {};
-  tests.ecryptfs = callTest tests/ecryptfs.nix {};
-  tests.etcd = callTestOnMatchingSystems ["x86_64-linux"] tests/etcd.nix {};
-  tests.ec2-nixops = (callSubTestsOnMatchingSystems ["x86_64-linux"] tests/ec2.nix {}).boot-ec2-nixops or {};
-  # ec2-config doesn't work in a sandbox as the simulated ec2 instance needs network access
-  #tests.ec2-config = (callSubTestsOnMatchingSystems ["x86_64-linux"] tests/ec2.nix {}).boot-ec2-config or {};
-  tests.elk = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/elk.nix {};
-  tests.env = callTest tests/env.nix {};
-  tests.ferm = callTest tests/ferm.nix {};
-  tests.firefox = callTest tests/firefox.nix {};
-  tests.flatpak = callTest tests/flatpak.nix {};
-  tests.firewall = callTest tests/firewall.nix {};
-  tests.fsck = callTest tests/fsck.nix {};
-  tests.fwupd = callTest tests/fwupd.nix {};
-  tests.gdk-pixbuf = callTest tests/gdk-pixbuf.nix {};
-  tests.gitea = callSubTests tests/gitea.nix {};
-  tests.gitlab = callTest tests/gitlab.nix {};
-  tests.gitolite = callTest tests/gitolite.nix {};
-  tests.gjs = callTest tests/gjs.nix {};
-  tests.gocd-agent = callTest tests/gocd-agent.nix {};
-  tests.gocd-server = callTest tests/gocd-server.nix {};
-  tests.gnome3 = callTest tests/gnome3.nix {};
-  tests.gnome3-gdm = callTest tests/gnome3-gdm.nix {};
-  tests.grafana = callTest tests/grafana.nix {};
-  tests.graphite = callTest tests/graphite.nix {};
-  tests.hadoop.hdfs = callTestOnMatchingSystems [ "x86_64-linux" ] tests/hadoop/hdfs.nix {};
-  tests.hadoop.yarn = callTestOnMatchingSystems [ "x86_64-linux" ] tests/hadoop/yarn.nix {};
-  tests.hardened = callTest tests/hardened.nix { };
-  tests.haproxy = callTest tests/haproxy.nix {};
-  tests.hibernate = callTest tests/hibernate.nix {};
-  tests.hitch = callTest tests/hitch {};
-  tests.home-assistant = callTest tests/home-assistant.nix { };
-  tests.hound = callTest tests/hound.nix {};
-  tests.hocker-fetchdocker = callTest tests/hocker-fetchdocker {};
-  tests.hydra = callTest tests/hydra {};
-  tests.i3wm = callTest tests/i3wm.nix {};
-  tests.iftop = callTest tests/iftop.nix {};
-  tests.initrd-network-ssh = callTest tests/initrd-network-ssh {};
-  tests.installer = callSubTests tests/installer.nix {};
-  tests.influxdb = callTest tests/influxdb.nix {};
-  tests.ipv6 = callTest tests/ipv6.nix {};
-  tests.jenkins = callTest tests/jenkins.nix {};
-  tests.ostree = callTest tests/ostree.nix {};
-  tests.osquery = callTest tests/osquery.nix {};
-  tests.plasma5 = callTest tests/plasma5.nix {};
-  tests.plotinus = callTest tests/plotinus.nix {};
-  tests.keymap = callSubTests tests/keymap.nix {};
-  tests.incron = callTest tests/incron.nix {};
-  tests.initrdNetwork = callTest tests/initrd-network.nix {};
-  tests.kafka = callSubTests tests/kafka.nix {};
-  tests.kernel-latest = callTest tests/kernel-latest.nix {};
-  tests.kernel-lts = callTest tests/kernel-lts.nix {};
-  tests.kubernetes.dns = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/dns.nix {};
-  ## kubernetes.e2e should eventually replace kubernetes.rbac when it works
-  #tests.kubernetes.e2e = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/e2e.nix {};
-  tests.kubernetes.rbac = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/kubernetes/rbac.nix {};
-  tests.latestKernel.login = callTest tests/login.nix { latestKernel = true; };
-  tests.ldap = callTest tests/ldap.nix {};
-  #tests.lightdm = callTest tests/lightdm.nix {};
-  tests.login = callTest tests/login.nix {};
-  #tests.logstash = callTest tests/logstash.nix {};
-  tests.mathics = callTest tests/mathics.nix {};
-  tests.matrix-synapse = callTest tests/matrix-synapse.nix {};
-  tests.memcached = callTest tests/memcached.nix {};
-  tests.mesos = callTest tests/mesos.nix {};
-  tests.misc = callTest tests/misc.nix {};
-  tests.mongodb = callTest tests/mongodb.nix {};
-  tests.mpd = callTest tests/mpd.nix {};
-  tests.mumble = callTest tests/mumble.nix {};
-  tests.munin = callTest tests/munin.nix {};
-  tests.mutableUsers = callTest tests/mutable-users.nix {};
-  tests.mysql = callTest tests/mysql.nix {};
-  tests.mysqlBackup = callTest tests/mysql-backup.nix {};
-  tests.mysqlReplication = callTest tests/mysql-replication.nix {};
-  tests.nat.firewall = callTest tests/nat.nix { withFirewall = true; };
-  tests.nat.firewall-conntrack = callTest tests/nat.nix { withFirewall = true; withConntrackHelpers = true; };
-  tests.nat.standalone = callTest tests/nat.nix { withFirewall = false; };
-  tests.netdata = callTest tests/netdata.nix { };
-  tests.networking.networkd = callSubTests tests/networking.nix { networkd = true; };
-  tests.networking.scripted = callSubTests tests/networking.nix { networkd = false; };
-  tests.nextcloud = callSubTests tests/nextcloud { };
-  # TODO: put in networking.nix after the test becomes more complete
-  tests.networkingProxy = callTest tests/networking-proxy.nix {};
-  tests.nexus = callTest tests/nexus.nix { };
-  tests.nfs3 = callTest tests/nfs.nix { version = 3; };
-  tests.nfs4 = callTest tests/nfs.nix { version = 4; };
-  tests.nginx = callTest tests/nginx.nix { };
-  tests.nghttpx = callTest tests/nghttpx.nix { };
-  tests.nix-ssh-serve = callTest tests/nix-ssh-serve.nix { };
-  tests.novacomd = callTestOnMatchingSystems ["x86_64-linux"] tests/novacomd.nix { };
-  tests.leaps = callTest tests/leaps.nix { };
-  tests.nsd = callTest tests/nsd.nix {};
-  tests.openssh = callTest tests/openssh.nix {};
-  tests.openldap = callTest tests/openldap.nix {};
-  tests.opensmtpd = callTest tests/opensmtpd.nix {};
-  tests.owncloud = callTest tests/owncloud.nix {};
-  tests.pam-oath-login = callTest tests/pam-oath-login.nix {};
-  tests.peerflix = callTest tests/peerflix.nix {};
-  tests.php-pcre = callTest tests/php-pcre.nix {};
-  tests.postgresql = callSubTests tests/postgresql.nix {};
-  tests.pgmanage = callTest tests/pgmanage.nix {};
-  tests.postgis = callTest tests/postgis.nix {};
-  tests.powerdns = callTest tests/powerdns.nix {};
-  tests.pgjwt = callTest tests/pgjwt.nix {};
-  tests.predictable-interface-names = callSubTests tests/predictable-interface-names.nix {};
-  tests.printing = callTest tests/printing.nix {};
-  tests.prometheus = callTest tests/prometheus.nix {};
-  tests.prometheus-exporters = callTest tests/prometheus-exporters.nix {};
-  tests.prosody = callTest tests/prosody.nix {};
-  tests.proxy = callTest tests/proxy.nix {};
-  tests.quagga = callTest tests/quagga.nix {};
-  tests.quake3 = callTest tests/quake3.nix {};
-  tests.rabbitmq = callTest tests/rabbitmq.nix {};
-  tests.radicale = callTest tests/radicale.nix {};
-  tests.redmine = callTest tests/redmine.nix {};
-  tests.rspamd = callSubTests tests/rspamd.nix {};
-  tests.rsyslogd = callSubTests tests/rsyslogd.nix {};
-  tests.runInMachine = callTest tests/run-in-machine.nix {};
-  tests.rxe = callTest tests/rxe.nix {};
-  tests.samba = callTest tests/samba.nix {};
-  tests.sddm = callSubTests tests/sddm.nix {};
-  tests.simple = callTest tests/simple.nix {};
-  tests.slim = callTest tests/slim.nix {};
-  tests.slurm = callTest tests/slurm.nix {};
-  tests.smokeping = callTest tests/smokeping.nix {};
-  tests.snapper = callTest tests/snapper.nix {};
-  tests.solr = callTest tests/solr.nix {};
-  #tests.statsd = callTest tests/statsd.nix {}; # statsd is broken: #45946
-  tests.strongswan-swanctl = callTest tests/strongswan-swanctl.nix {};
-  tests.sudo = callTest tests/sudo.nix {};
-  tests.systemd = callTest tests/systemd.nix {};
-  tests.switchTest = callTest tests/switch-test.nix {};
-  tests.taskserver = callTest tests/taskserver.nix {};
-  tests.tomcat = callTest tests/tomcat.nix {};
-  tests.tor = callTest tests/tor.nix {};
-  tests.transmission = callTest tests/transmission.nix {};
-  tests.udisks2 = callTest tests/udisks2.nix {};
-  tests.vault = callTest tests/vault.nix {};
-  tests.virtualbox = callSubTestsOnMatchingSystems ["x86_64-linux"] tests/virtualbox.nix {};
-  tests.wordpress = callTest tests/wordpress.nix {};
-  tests.xautolock = callTest tests/xautolock.nix {};
-  tests.xdg-desktop-portal = callTest tests/xdg-desktop-portal.nix {};
-  tests.xfce = callTest tests/xfce.nix {};
-  tests.xmonad = callTest tests/xmonad.nix {};
-  tests.xrdp = callTest tests/xrdp.nix {};
-  tests.xss-lock = callTest tests/xss-lock.nix {};
-  tests.yabar = callTest tests/yabar.nix {};
-  tests.zookeeper = callTest tests/zookeeper.nix {};
-  tests.morty = callTest tests/morty.nix { };
-  tests.bcachefs = callTest tests/bcachefs.nix { };
 
   /* Build a bunch of typical closures so that Hydra can keep track of
      the evolution of closure sizes. */

nixos/tests/all-tests.nix

@@ -0,0 +1,214 @@
+{ system, pkgs, callTest }:
+# The return value of this function will be an attrset with arbitrary depth and
+# the `anything` returned by callTest at its test leafs.
+# The tests not supported by `system` will be replaced with `{}`, so that
+# `passthru.tests` can contain links to those without breaking on architectures
+# where said tests are unsupported.
+# Example callTest that just extracts the derivation from the test:
+#   callTest = t: t.test;
+
+with pkgs.lib;
+
+let
+  discoverTests = val:
+    if !isAttrs val then val
+    else if hasAttr "test" val then callTest val
+    else mapAttrs (n: s: discoverTests s) val;
+  handleTest = path: args:
+    discoverTests (import path ({ inherit system pkgs; } // args));
+  handleTestOn = systems: path: args:
+    if elem system systems then handleTest path args
+    else {};
+in
+{
+  acme = handleTestOn ["x86_64-linux"] ./acme.nix {};
+  atd = handleTest ./atd.nix {};
+  avahi = handleTest ./avahi.nix {};
+  bcachefs = handleTestOn ["x86_64-linux"] ./bcachefs.nix {}; # linux-4.18.2018.10.12 is unsupported on aarch64
+  beegfs = handleTestOn ["x86_64-linux"] ./beegfs.nix {}; # beegfs is unsupported on aarch64
+  bind = handleTest ./bind.nix {};
+  bittorrent = handleTest ./bittorrent.nix {};
+  #blivet = handleTest ./blivet.nix {};   # broken since 2017-07024
+  boot = handleTestOn ["x86_64-linux"] ./boot.nix {}; # syslinux is unsupported on aarch64
+  boot-stage1 = handleTest ./boot-stage1.nix {};
+  borgbackup = handleTest ./borgbackup.nix {};
+  buildbot = handleTest ./buildbot.nix {};
+  cadvisor = handleTestOn ["x86_64-linux"] ./cadvisor.nix {};
+  ceph = handleTestOn ["x86_64-linux"] ./ceph.nix {};
+  certmgr = handleTest ./certmgr.nix {};
+  cfssl = handleTestOn ["x86_64-linux"] ./cfssl.nix {};
+  chromium = (handleTestOn ["x86_64-linux"] ./chromium.nix {}).stable or {};
+  cjdns = handleTest ./cjdns.nix {};
+  cloud-init = handleTest ./cloud-init.nix {};
+  codimd = handleTest ./codimd.nix {};
+  containers-bridge = handleTest ./containers-bridge.nix {};
+  containers-extra_veth = handleTest ./containers-extra_veth.nix {};
+  containers-hosts = handleTest ./containers-hosts.nix {};
+  containers-imperative = handleTest ./containers-imperative.nix {};
+  containers-ipv4 = handleTest ./containers-ipv4.nix {};
+  containers-ipv6 = handleTest ./containers-ipv6.nix {};
+  containers-macvlans = handleTest ./containers-macvlans.nix {};
+  containers-physical_interfaces = handleTest ./containers-physical_interfaces.nix {};
+  containers-restart_networking = handleTest ./containers-restart_networking.nix {};
+  containers-tmpfs = handleTest ./containers-tmpfs.nix {};
+  #couchdb = handleTest ./couchdb.nix {}; # spidermonkey-1.8.5 is marked as broken
+  deluge = handleTest ./deluge.nix {};
+  dhparams = handleTest ./dhparams.nix {};
+  dnscrypt-proxy = handleTestOn ["x86_64-linux"] ./dnscrypt-proxy.nix {};
+  docker = handleTestOn ["x86_64-linux"] ./docker.nix {};
+  docker-edge = handleTestOn ["x86_64-linux"] ./docker-edge.nix {};
+  docker-preloader = handleTestOn ["x86_64-linux"] ./docker-preloader.nix {};
+  docker-registry = handleTest ./docker-registry.nix {};
+  docker-tools = handleTestOn ["x86_64-linux"] ./docker-tools.nix {};
+  docker-tools-overlay = handleTestOn ["x86_64-linux"] ./docker-tools-overlay.nix {};
+  dovecot = handleTest ./dovecot.nix {};
+  # ec2-config doesn't work in a sandbox as the simulated ec2 instance needs network access
+  #ec2-config = (handleTestOn ["x86_64-linux"] ./ec2.nix {}).boot-ec2-config or {};
+  ec2-nixops = (handleTestOn ["x86_64-linux"] ./ec2.nix {}).boot-ec2-nixops or {};
+  ecryptfs = handleTest ./ecryptfs.nix {};
+  elk = handleTestOn ["x86_64-linux"] ./elk.nix {};
+  env = handleTest ./env.nix {};
+  etcd = handleTestOn ["x86_64-linux"] ./etcd.nix {};
+  ferm = handleTest ./ferm.nix {};
+  firefox = handleTest ./firefox.nix {};
+  firewall = handleTest ./firewall.nix {};
+  flatpak = handleTest ./flatpak.nix {};
+  fsck = handleTest ./fsck.nix {};
+  fwupd = handleTestOn ["x86_64-linux"] ./fwupd.nix {}; # libsmbios is unsupported on aarch64
+  gdk-pixbuf = handleTest ./gdk-pixbuf.nix {};
+  gitea = handleTest ./gitea.nix {};
+  gitlab = handleTest ./gitlab.nix {};
+  gitolite = handleTest ./gitolite.nix {};
+  gjs = handleTest ./gjs.nix {};
+  gnome3 = handleTestOn ["x86_64-linux"] ./gnome3.nix {}; # libsmbios is unsupported on aarch64
+  gnome3-gdm = handleTestOn ["x86_64-linux"] ./gnome3-gdm.nix {}; # libsmbios is unsupported on aarch64
+  gocd-agent = handleTest ./gocd-agent.nix {};
+  gocd-server = handleTest ./gocd-server.nix {};
+  grafana = handleTest ./grafana.nix {};
+  graphite = handleTest ./graphite.nix {};
+  hadoop.hdfs = handleTestOn [ "x86_64-linux" ] ./hadoop/hdfs.nix {};
+  hadoop.yarn = handleTestOn [ "x86_64-linux" ] ./hadoop/yarn.nix {};
+  haproxy = handleTest ./haproxy.nix {};
+  #hardened = handleTest ./hardened.nix {}; # broken due useSandbox = true
+  hibernate = handleTest ./hibernate.nix {};
+  hitch = handleTest ./hitch {};
+  hocker-fetchdocker = handleTest ./hocker-fetchdocker {};
+  home-assistant = handleTest ./home-assistant.nix {};
+  hound = handleTest ./hound.nix {};
+  hydra = handleTest ./hydra {};
+  i3wm = handleTest ./i3wm.nix {};
+  iftop = handleTest ./iftop.nix {};
+  incron = handleTest tests/incron.nix {};
+  influxdb = handleTest ./influxdb.nix {};
+  initrd-network-ssh = handleTest ./initrd-network-ssh {};
+  initrdNetwork = handleTest ./initrd-network.nix {};
+  installer = handleTest ./installer.nix {};
+  ipv6 = handleTest ./ipv6.nix {};
+  jenkins = handleTest ./jenkins.nix {};
+  kafka = handleTest ./kafka.nix {};
+  kernel-latest = handleTest ./kernel-latest.nix {};
+  kernel-lts = handleTest ./kernel-lts.nix {};
+  keymap = handleTest ./keymap.nix {};
+  kubernetes.dns = handleTestOn ["x86_64-linux"] ./kubernetes/dns.nix {};
+  # kubernetes.e2e should eventually replace kubernetes.rbac when it works
+  #kubernetes.e2e = handleTestOn ["x86_64-linux"] ./kubernetes/e2e.nix {};
+  kubernetes.rbac = handleTestOn ["x86_64-linux"] ./kubernetes/rbac.nix {};
+  latestKernel.login = handleTest ./login.nix { latestKernel = true; };
+  ldap = handleTest ./ldap.nix {};
+  leaps = handleTest ./leaps.nix {};
+  #lightdm = handleTest ./lightdm.nix {};
+  login = handleTest ./login.nix {};
+  #logstash = handleTest ./logstash.nix {};
+  mathics = handleTest ./mathics.nix {};
+  matrix-synapse = handleTest ./matrix-synapse.nix {};
+  memcached = handleTest ./memcached.nix {};
+  mesos = handleTest ./mesos.nix {};
+  misc = handleTest ./misc.nix {};
+  mongodb = handleTest ./mongodb.nix {};
+  morty = handleTest ./morty.nix {};
+  mpd = handleTest ./mpd.nix {};
+  mumble = handleTest ./mumble.nix {};
+  munin = handleTest ./munin.nix {};
+  mutableUsers = handleTest ./mutable-users.nix {};
+  mysql = handleTest ./mysql.nix {};
+  mysqlBackup = handleTest ./mysql-backup.nix {};
+  mysqlReplication = handleTest ./mysql-replication.nix {};
+  nat.firewall = handleTest ./nat.nix { withFirewall = true; };
+  nat.firewall-conntrack = handleTest ./nat.nix { withFirewall = true; withConntrackHelpers = true; };
+  nat.standalone = handleTest ./nat.nix { withFirewall = false; };
+  netdata = handleTest ./netdata.nix {};
+  networking.networkd = handleTest ./networking.nix { networkd = true; };
+  networking.scripted = handleTest ./networking.nix { networkd = false; };
+  # TODO: put in networking.nix after the test becomes more complete
+  networkingProxy = handleTest ./networking-proxy.nix {};
+  nextcloud = handleTest ./nextcloud {};
+  nexus = handleTest ./nexus.nix {};
+  nfs3 = handleTest ./nfs.nix { version = 3; };
+  nfs4 = handleTest ./nfs.nix { version = 4; };
+  nghttpx = handleTest ./nghttpx.nix {};
+  nginx = handleTest ./nginx.nix {};
+  nix-ssh-serve = handleTest ./nix-ssh-serve.nix {};
+  novacomd = handleTestOn ["x86_64-linux"] ./novacomd.nix {};
+  nsd = handleTest ./nsd.nix {};
+  openldap = handleTest ./openldap.nix {};
+  opensmtpd = handleTest ./opensmtpd.nix {};
+  openssh = handleTest ./openssh.nix {};
+  osquery = handleTest ./osquery.nix {};
+  ostree = handleTest ./ostree.nix {};
+  owncloud = handleTest ./owncloud.nix {};
+  pam-oath-login = handleTest ./pam-oath-login.nix {};
+  peerflix = handleTest ./peerflix.nix {};
+  pgjwt = handleTest ./pgjwt.nix {};
+  pgmanage = handleTest ./pgmanage.nix {};
+  php-pcre = handleTest ./php-pcre.nix {};
+  plasma5 = handleTest ./plasma5.nix {};
+  plotinus = handleTest ./plotinus.nix {};
+  postgis = handleTest ./postgis.nix {};
+  postgresql = handleTest ./postgresql.nix {};
+  powerdns = handleTest ./powerdns.nix {};
+  predictable-interface-names = handleTest ./predictable-interface-names.nix {};
+  printing = handleTest ./printing.nix {};
+  prometheus = handleTest ./prometheus.nix {};
+  prometheus-exporters = handleTest ./prometheus-exporters.nix {};
+  prosody = handleTest ./prosody.nix {};
+  proxy = handleTest ./proxy.nix {};
+  quagga = handleTest ./quagga.nix {};
+  quake3 = handleTest ./quake3.nix {};
+  rabbitmq = handleTest ./rabbitmq.nix {};
+  radicale = handleTest ./radicale.nix {};
+  redmine = handleTest ./redmine.nix {};
+  rspamd = handleTest ./rspamd.nix {};
+  rsyslogd = handleTest ./rsyslogd.nix {};
+  runInMachine = handleTest ./run-in-machine.nix {};
+  rxe = handleTest ./rxe.nix {};
+  samba = handleTest ./samba.nix {};
+  sddm = handleTest ./sddm.nix {};
+  simple = handleTest ./simple.nix {};
+  slim = handleTest ./slim.nix {};
+  slurm = handleTest ./slurm.nix {};
+  smokeping = handleTest ./smokeping.nix {};
+  snapper = handleTest ./snapper.nix {};
+  solr = handleTest ./solr.nix {};
+  #statsd = handleTest ./statsd.nix {}; # statsd is broken: #45946
+  strongswan-swanctl = handleTest ./strongswan-swanctl.nix {};
+  sudo = handleTest ./sudo.nix {};
+  switchTest = handleTest ./switch-test.nix {};
+  systemd = handleTest ./systemd.nix {};
+  taskserver = handleTest ./taskserver.nix {};
+  tomcat = handleTest ./tomcat.nix {};
+  tor = handleTest ./tor.nix {};
+  transmission = handleTest ./transmission.nix {};
+  udisks2 = handleTest ./udisks2.nix {};
+  upnp = handleTest ./upnp.nix {};
+  vault = handleTest ./vault.nix {};
+  virtualbox = handleTestOn ["x86_64-linux"] ./virtualbox.nix {};
+  wordpress = handleTest ./wordpress.nix {};
+  xautolock = handleTest ./xautolock.nix {};
+  xdg-desktop-portal = handleTest ./xdg-desktop-portal.nix {};
+  xfce = handleTest ./xfce.nix {};
+  xmonad = handleTest ./xmonad.nix {};
+  xrdp = handleTest ./xrdp.nix {};
+  xss-lock = handleTest ./xss-lock.nix {};
+  yabar = handleTest ./yabar.nix {};
+  zookeeper = handleTest ./zookeeper.nix {};
+}

nixos/tests/boot.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
 
 let

nixos/tests/buildbot.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 
 let
   # Test ensures buildbot master comes up correctly and workers can connect

nixos/tests/certmgr.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 let
   mkSpec = { host, service ? null, action }: {
     inherit action;

nixos/tests/chromium.nix

@@ -1,5 +1,6 @@
 { system ? builtins.currentSystem
-, pkgs ? import ../.. { inherit system; }
+, config ? {}
+, pkgs ? import ../.. { inherit system config; }
 , channelMap ? {
     stable = pkgs.chromium;
     beta   = pkgs.chromiumBeta;
@@ -7,7 +8,7 @@
   }
 }:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
 
 mapAttrs (channel: chromiumPkg: makeTest rec {

nixos/tests/cloud-init.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
 
 let

nixos/tests/ec2.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
 
 let

nixos/tests/elk.nix

@@ -1,6 +1,12 @@
-{ system ? builtins.currentSystem, enableUnfree ? false }:
+{ system ? builtins.currentSystem,
-with import ../lib/testing.nix { inherit system; };
+  config ? {},
+  pkgs ? import ../.. { inherit system config; },
+  enableUnfree ? false
+}:
+
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
+
 let
   esUrl = "http://localhost:9200";
 

nixos/tests/gitea.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
 
 {

nixos/tests/installer.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
 
 let

nixos/tests/kafka.nix

@@ -1,5 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
-with import ../lib/testing.nix { inherit system; };
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
 
 let

nixos/tests/keymap.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 
 let
   readyFile  = "/tmp/readerReady";

nixos/tests/kubernetes/base.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../../lib/testing.nix { inherit system; };
+with import ../../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
 
 let

nixos/tests/make-test.nix

@@ -1,5 +1,9 @@
-f: { system ? builtins.currentSystem, ... } @ args:
+f: {
+  system ? builtins.currentSystem,
+  pkgs ? import ../.. { inherit system; config = {}; },
+  ...
+} @ args:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 
 makeTest (if pkgs.lib.isFunction f then f (args // { inherit pkgs; inherit (pkgs) lib; }) else f)

nixos/tests/networking.nix

@@ -1,8 +1,10 @@
 { system ? builtins.currentSystem
+, config ? {}
+, pkgs ? import ../.. { inherit system config; },
 # bool: whether to use networkd in the tests
 , networkd }:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
 
 let

nixos/tests/nextcloud/default.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../../.. { inherit system config; }
+}:
 {
-  basic = import ./basic.nix { inherit system; };
+  basic = import ./basic.nix { inherit system pkgs; };
-  with-postgresql-and-redis = import ./with-postgresql-and-redis.nix { inherit system; };
+  with-postgresql-and-redis = import ./with-postgresql-and-redis.nix { inherit system pkgs; };
-  with-mysql-and-memcached = import ./with-mysql-and-memcached.nix { inherit system; };
+  with-mysql-and-memcached = import ./with-mysql-and-memcached.nix { inherit system pkgs; };
 }

nixos/tests/postgresql.nix

@@ -1,6 +1,11 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
-with import ../lib/testing.nix { inherit system; };
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
+
 let
   postgresql-versions = pkgs.callPackages ../../pkgs/servers/sql/postgresql { };
   test-sql = pkgs.writeText "postgresql-test" ''

nixos/tests/predictable-interface-names.nix

@@ -1,7 +1,10 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
 let
-  inherit (import ../lib/testing.nix { inherit system; }) makeTest pkgs;
+  inherit (import ../lib/testing.nix { inherit system pkgs; }) makeTest;
 in pkgs.lib.listToAttrs (pkgs.lib.crossLists (predictable: withNetworkd: {
   name = pkgs.lib.optionalString (!predictable) "un" + "predictable"
        + pkgs.lib.optionalString withNetworkd "Networkd";

nixos/tests/rspamd.nix

@@ -1,6 +1,11 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
-with import ../lib/testing.nix { inherit system; };
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
+
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
+
 let
   initMachine = ''
     startAll

nixos/tests/rsyslogd.nix

@@ -1,7 +1,11 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
+
 {
   test1 = makeTest {
     name = "rsyslogd-test1";

nixos/tests/run-in-machine.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 
 let
   output = runInMachine {

nixos/tests/sddm.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 
 let
   inherit (pkgs) lib;

nixos/tests/virtualbox.nix

@@ -1,6 +1,11 @@
-{ system ? builtins.currentSystem, debug ? false, enableUnfree ? false }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; },
+  debug ? false,
+  enableUnfree ? false
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 with pkgs.lib;
 
 let

nixos/tests/zfs.nix

@@ -1,6 +1,9 @@
-{ system ? builtins.currentSystem }:
+{ system ? builtins.currentSystem,
+  config ? {},
+  pkgs ? import ../.. { inherit system config; }
+}:
 
-with import ../lib/testing.nix { inherit system; };
+with import ../lib/testing.nix { inherit system pkgs; };
 
 let
 

pkgs/servers/mail/dovecot/default.nix

@@ -68,11 +68,8 @@ stdenv.mkDerivation rec {
     description = "Open source IMAP and POP3 email server written with security primarily in mind";
     maintainers = with stdenv.lib.maintainers; [ peti rickynils fpletz ];
     platforms = stdenv.lib.platforms.unix;
-    # https://github.com/NixOS/nixpkgs/issues/50230 must be resolved before
+  };
-    # this can be enabled.
+  passthru.tests = {
-    #
+    opensmtpd-interaction = nixosTests.opensmtpd;
-    # tests = {
-    #   opensmtpd-interaction = nixosTests.opensmtpd;
-    # };
   };
 }

pkgs/servers/mail/opensmtpd/default.nix

@@ -61,11 +61,8 @@ stdenv.mkDerivation rec {
     license = licenses.isc;
     platforms = platforms.linux;
     maintainers = with maintainers; [ rickynils obadz ekleog ];
-    # https://github.com/NixOS/nixpkgs/issues/50230 must be resolved before
+  };
-    # this can be enabled.
+  passthru.tests = {
-    #
+    basic-functionality-and-dovecot-interaction = nixosTests.opensmtpd;
-    # tests = {
-    #   basic-functionality-and-dovecot-interaction = nixosTests.opensmtpd;
-    # };
   };
 }

pkgs/stdenv/generic/check-meta.nix

@@ -166,14 +166,15 @@ let
     hydraPlatforms = listOf str;
     broken = bool;
     # TODO: refactor once something like Profpatsch's types-simple will land
+    # This is currently dead code due to https://github.com/NixOS/nix/issues/2532
     tests = attrsOf (mkOptionType {
       name = "test";
-      check = x: isDerivation x &&
+      check = x: x == {} || ( # Accept {} for tests that are unsupported
-        x ? meta.timeout &&
+        isDerivation x &&
-        x ? meta.needsVMSupport;
+        x ? meta.timeout
+      );
       merge = lib.options.mergeOneOption;
     });
-    needsVMSupport = bool;
     timeout = int;
 
     # Weirder stuff that doesn't appear in the documentation?

pkgs/top-level/all-packages.nix

@@ -73,23 +73,11 @@ with pkgs;
 
   ### Push NixOS tests inside the fixed point
 
-  nixosTests =
+  nixosTests = import ../../nixos/tests/all-tests.nix {
-    let
+    inherit pkgs;
-      # TODO(Ericson2314,ekleog): Check this will work correctly with cross-
+    system = stdenv.hostPlatform.system;
-      system = builtins.currentSystem;
+    callTest = t: t.test;
-      rawTests = (import ../../nixos/release.nix {
+  };
-        nixpkgs = pkgs;
-      }).tests;
-      testNames = builtins.attrNames rawTests;
-      filteredList = builtins.filter
-        (test: rawTests.${test} ? ${system})
-        testNames;
-      finalList = map
-        (test: { name = test; value = rawTests.${test}.${system}; })
-        filteredList;
-      finalTests = builtins.listToAttrs finalList;
-    in
-    finalTests;
 
   ### BUILD SUPPORT