* Moved test-related stuff from lib/build-vms.nix to lib/testing.nix.

* Factored out some commonality between tests to make them a bit
  simpler to write.  A test is a function { pkgs, ... }: -> { nodes,
  testScript } or { machine, testScript }.  So it's no longer
  necessary to have a "vms" attribute in every test.

svn path=/nixos/trunk/; revision=19220

lib/build-vms.nix

@@ -1,8 +1,4 @@
-{ nixos ? ./..
+{ nixpkgs, services, system }:
-, nixpkgs ? ../../nixpkgs
-, services ? ../../nixos/services
-, system ? builtins.currentSystem
-}:
 
 let pkgs = import nixpkgs { config = {}; inherit system; }; in
 
@@ -55,12 +51,12 @@ rec {
   buildVM =
     nodes: configurations:
 
-    import "${nixos}/lib/eval-config.nix" {
+    import ./eval-config.nix {
       inherit nixpkgs services system;
-      modules = configurations ++ [
+      modules = configurations ++
-        "${nixos}/modules/virtualisation/qemu-vm.nix" # !!!
+        [ ../modules/virtualisation/qemu-vm.nix # !!!
-        "${nixos}/modules/testing/test-instrumentation.nix" # !!! should only get added for automated test runs
+          ../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs
-      ];
+        ];
       extraArgs = { inherit nodes; };
       /* !!! bug in the module/option handling: this ignores the
          config from assignIPAddresses.  Too much magic. 
@@ -117,74 +113,4 @@ rec {
     else [];
 
 
-  # Run an automated test suite in the given virtual network.
-  # `network' must be the result of a call to the
-  # `buildVirtualNetwork' function.  `tests' is a Perl fragment
-  # describing the tests.
-  runTests = network: tests:
-    stdenv.mkDerivation {
-      name = "vm-test-run";
-      inherit tests;
-      buildCommand =
-        ''
-          mkdir $out
-          cp ${./test-driver/Machine.pm} Machine.pm
-          ensureDir $out/nix-support
-          
-          ${perl}/bin/perl ${./test-driver/test-driver.pl} ${network}/vms/*/bin/run-*-vm
-          
-          for i in */coverage-data; do
-            ensureDir $out/coverage-data
-            mv $i $out/coverage-data/$(dirname $i)
-          done
-
-          for i in $out/*.png; do
-            echo "report screenshot $i" >> $out/nix-support/hydra-build-products
-          done
-        ''; # */
-    };
-
-
-  # Generate a coverage report from the coverage data produced by
-  # runTests.
-  makeReport = x: runCommand "report" { buildInputs = [rsync]; }
-    ''
-      for d in ${x}/coverage-data/*; do
-
-          echo "doing $d"
-
-          ensureDir $TMPDIR/gcov/
-
-          for i in $(cd $d/nix/store && ls); do
-              if ! test -e $TMPDIR/gcov/nix/store/$i; then
-                  echo "copying $i"
-                  mkdir -p $TMPDIR/gcov/$(echo $i | cut -c34-)
-                  rsync -rv /nix/store/$i/.build/* $TMPDIR/gcov/
-              fi
-          done
-
-          chmod -R u+w $TMPDIR/gcov
-
-          find $TMPDIR/gcov -name "*.gcda" -exec rm {} \;
-
-          for i in $(cd $d/nix/store && ls); do
-              rsync -rv $d/nix/store/$i/.build/* $TMPDIR/gcov/
-          done
-
-          find $TMPDIR/gcov -name "*.gcda" -exec chmod 644 {} \;
-          
-          echo "producing info..."
-          ${pkgs.lcov}/bin/geninfo --ignore-errors source,gcov $TMPDIR/gcov --output-file $TMPDIR/app.info
-          cat $TMPDIR/app.info >> $TMPDIR/full.info
-      done
-            
-      echo "making report..."
-      ensureDir $out/coverage
-      ${pkgs.lcov}/bin/genhtml --show-details $TMPDIR/full.info -o $out/coverage
-      cp $TMPDIR/full.info $out/coverage/
-
-      ensureDir $out/nix-support
-      echo "report coverage $out/coverage" >> $out/nix-support/hydra-build-products
-    ''; # */
-    
 }

lib/testing.nix

@@ -0,0 +1,79 @@
+{ nixpkgs, services, system }:
+
+with import ./build-vms.nix { inherit nixpkgs services system; };
+with pkgs;
+
+rec {
+
+
+  # Run an automated test suite in the given virtual network.
+  # `network' must be the result of a call to the
+  # `buildVirtualNetwork' function.  `tests' is a Perl fragment
+  # describing the tests.
+  runTests = network: tests:
+    stdenv.mkDerivation {
+      name = "vm-test-run";
+      inherit tests;
+      buildCommand =
+        ''
+          mkdir $out
+          cp ${./test-driver/Machine.pm} Machine.pm
+          ensureDir $out/nix-support
+          
+          ${perl}/bin/perl ${./test-driver/test-driver.pl} ${network}/vms/*/bin/run-*-vm
+          
+          for i in */coverage-data; do
+            ensureDir $out/coverage-data
+            mv $i $out/coverage-data/$(dirname $i)
+          done
+
+          for i in $out/*.png; do
+            echo "report screenshot $i" >> $out/nix-support/hydra-build-products
+          done
+        ''; # */
+    };
+
+
+  # Generate a coverage report from the coverage data produced by
+  # runTests.
+  makeReport = x: runCommand "report" { buildInputs = [rsync]; }
+    ''
+      for d in ${x}/coverage-data/*; do
+
+          echo "doing $d"
+
+          ensureDir $TMPDIR/gcov/
+
+          for i in $(cd $d/nix/store && ls); do
+              if ! test -e $TMPDIR/gcov/nix/store/$i; then
+                  echo "copying $i"
+                  mkdir -p $TMPDIR/gcov/$(echo $i | cut -c34-)
+                  rsync -rv /nix/store/$i/.build/* $TMPDIR/gcov/
+              fi
+          done
+
+          chmod -R u+w $TMPDIR/gcov
+
+          find $TMPDIR/gcov -name "*.gcda" -exec rm {} \;
+
+          for i in $(cd $d/nix/store && ls); do
+              rsync -rv $d/nix/store/$i/.build/* $TMPDIR/gcov/
+          done
+
+          find $TMPDIR/gcov -name "*.gcda" -exec chmod 644 {} \;
+          
+          echo "producing info..."
+          ${pkgs.lcov}/bin/geninfo --ignore-errors source,gcov $TMPDIR/gcov --output-file $TMPDIR/app.info
+          cat $TMPDIR/app.info >> $TMPDIR/full.info
+      done
+            
+      echo "making report..."
+      ensureDir $out/coverage
+      ${pkgs.lcov}/bin/genhtml --show-details $TMPDIR/full.info -o $out/coverage
+      cp $TMPDIR/full.info $out/coverage/
+
+      ensureDir $out/nix-support
+      echo "report coverage $out/coverage" >> $out/nix-support/hydra-build-products
+    ''; # */
+    
+}

release.nix

@@ -111,30 +111,20 @@ let
     minimal_install_archive = {system ? "i686-linux"}: (iso_minimal {inherit system;})
       .config.system.build.minimalInstallArchive;
 
-    tests.subversion =
+    tests = 
       { services ? ../services }:
+      let
+        t = import ./tests { 
+          inherit nixpkgs services;
+          system = "i686-linux";
+        };
+      in {
+        subversion = t.subversion.report;
+        kde4 = t.kde4.test;
+        quake3 = t.quake3.test;
+      };
 
-      (import ./tests/subversion.nix {
+    /*
-        inherit nixpkgs services;
-        system = "i686-linux";
-      }).report;
-    
-    tests.kde4 =
-      { services ? ../services }:
-
-      (import ./tests/kde4.nix {
-        inherit nixpkgs services;
-        system = "i686-linux";
-      }).test;
-
-    tests.quake3 =
-      { services ? ../services }:
-
-      (import ./tests/quake3.nix {
-        inherit nixpkgs services;
-        system = "i686-linux";
-      }).test;
-
     ### tests about installing NixOS
 
     # installs NixOs in a qemu_kvm instance using a tweaked iso.
@@ -142,6 +132,7 @@ let
       (import ./tests/test-nixos-install-from-cd/test.nix {
         inherit nixpkgs;
       }).test;
+    */
 
     # the archive installer can't be tested without chroot which requires being root
     # options: run in kvm or uml ?

tests/default.nix

@@ -0,0 +1,29 @@
+{ nixpkgs ? ../../nixpkgs
+, services ? ../../services
+, system ? builtins.currentSystem
+}:
+
+let
+
+  testLib = 
+    (import ../lib/build-vms.nix { inherit nixpkgs services system; }) //
+    (import ../lib/testing.nix { inherit nixpkgs services system; });
+
+  apply = testFun:
+    with testLib;
+    let
+      t = testFun { inherit pkgs testLib; };
+    in t // rec {
+      nodes = if t ? nodes then t.nodes else { machine = t.machine; };
+      vms = buildVirtualNetwork { inherit nodes; };
+      test = runTests vms t.testScript;
+      report = makeReport test;
+    };
+
+in
+
+{
+  kde4 = apply (import ./kde4.nix);
+  quake3 = apply (import ./quake3.nix);
+  subversion = apply (import ./subversion.nix);
+}

tests/kde4.nix

@@ -1,68 +1,56 @@
-{ nixos ? ./..
+{ pkgs, ... }:
-, nixpkgs ? ../../nixpkgs
-, services ? ../../nixos/services
-, system ? builtins.currentSystem
-}:
 
-with import ../lib/build-vms.nix { inherit nixos nixpkgs services system; };
+{
 
-rec {
+  machine = 
+    { config, pkgs, ... }:
 
-  nodes =
+    { services.xserver.enable = true;
-    { client =
-        { config, pkgs, ... }:
 
-        { services.xserver.enable = true;
+      services.httpd.enable = true;
-        
+      services.httpd.adminAddr = "foo@example.org";
-          services.httpd.enable = true;
+      services.httpd.documentRoot = "${pkgs.valgrind}/share/doc/valgrind/html";
-          services.httpd.adminAddr = "foo@example.org";
-          services.httpd.documentRoot = "${pkgs.valgrind}/share/doc/valgrind/html";
-  
-          services.xserver.displayManager.slim.enable = false;
-          services.xserver.displayManager.kdm.enable = true;
-          services.xserver.displayManager.kdm.extraConfig =
-            ''
-              [X-:0-Core]
-              AutoLoginEnable=true
-              AutoLoginUser=alice
-              AutoLoginPass=foobar
-            '';
-            
-          services.xserver.desktopManager.default = "kde4";
-          services.xserver.desktopManager.kde4.enable = true;
 
-          users.extraUsers = pkgs.lib.singleton
+      services.xserver.displayManager.slim.enable = false;
-            { name = "alice";
+      services.xserver.displayManager.kdm.enable = true;
-              description = "Alice Foobar";
+      services.xserver.displayManager.kdm.extraConfig =
-              home = "/home/alice";
+        ''
-              createHome = true;
+          [X-:0-Core]
-              useDefaultShell = true;
+          AutoLoginEnable=true
-              password = "foobar";
+          AutoLoginUser=alice
-            };
+          AutoLoginPass=foobar
+        '';
 
-          environment.systemPackages = [ pkgs.scrot ];
+      services.xserver.desktopManager.default = "kde4";
+      services.xserver.desktopManager.kde4.enable = true;
+
+      users.extraUsers = pkgs.lib.singleton
+        { name = "alice";
+          description = "Alice Foobar";
+          home = "/home/alice";
+          createHome = true;
+          useDefaultShell = true;
+          password = "foobar";
         };
+
+      environment.systemPackages = [ pkgs.scrot ];
     };
 
-  vms = buildVirtualNetwork { inherit nodes; };
+  testScript =
-
-  test = runTests vms
     ''
-      startAll;
+      $machine->waitForFile("/tmp/.X11-unix/X0");
-
-      $client->waitForFile("/tmp/.X11-unix/X0");
 
       sleep 70;
 
-      print STDERR $client->execute("su - alice -c 'DISPLAY=:0.0 kwrite /var/log/messages &'");
+      print STDERR $machine->execute("su - alice -c 'DISPLAY=:0.0 kwrite /var/log/messages &'");
 
       sleep 10;
       
-      print STDERR $client->execute("su - alice -c 'DISPLAY=:0.0 konqueror http://localhost/ &'");
+      print STDERR $machine->execute("su - alice -c 'DISPLAY=:0.0 konqueror http://localhost/ &'");
 
       sleep 10;
       
-      print STDERR $client->mustSucceed("DISPLAY=:0.0 scrot /hostfs/$ENV{out}/screen.png");
+      print STDERR $machine->mustSucceed("DISPLAY=:0.0 scrot /hostfs/$ENV{out}/screen.png");
     '';
   
 }

tests/quake3.nix

@@ -1,10 +1,4 @@
-{ nixos ? ./..
+{ pkgs, ... }:
-, nixpkgs ? /etc/nixos/nixpkgs
-, services ? /etc/nixos/services
-, system ? builtins.currentSystem
-}:
-
-with import ../lib/build-vms.nix { inherit nixos nixpkgs services system; };
 
 rec {
 
@@ -34,9 +28,7 @@ rec {
       client2 = client;
     };
 
-  vms = buildVirtualNetwork { inherit nodes; };
+  testScript =
-
-  test = runTests vms
     ''
       startAll;
 
@@ -57,8 +49,8 @@ rec {
       $server->mustSucceed("grep -q 'Foo.*entered the game' /tmp/log");
       $server->mustSucceed("grep -q 'Bar.*entered the game' /tmp/log");
       
-      print STDERR $client1->execute("DISPLAY=:0.0 scrot /hostfs/$ENV{out}/screen1.png");
+      print STDERR $client1->mustSucceed("DISPLAY=:0.0 scrot /hostfs/$ENV{out}/screen1.png");
-      print STDERR $client2->execute("DISPLAY=:0.0 scrot /hostfs/$ENV{out}/screen2.png");
+      print STDERR $client2->mustSucceed("DISPLAY=:0.0 scrot /hostfs/$ENV{out}/screen2.png");
     '';
   
 }

tests/subversion.nix

@@ -1,10 +1,4 @@
-{ nixos ? ./..
+{ pkgs, ... }:
-, nixpkgs ? ../../nixpkgs
-, services ? ../../services
-, system ? builtins.currentSystem
-}:
-
-with import ../lib/build-vms.nix { inherit nixos nixpkgs services system; };
 
 let
 
@@ -37,7 +31,7 @@ let
 
 in
       
-rec {
+{
 
   nodes =
     { webserver =
@@ -66,9 +60,7 @@ rec {
 
     };
 
-  vms = buildVirtualNetwork { inherit nodes; };
+  testScript =
-
-  test = runTests vms
     ''
       startAll;
 
@@ -125,6 +117,4 @@ rec {
       $webserver->execute("sleep 10"); # !!!
     '';
 
-  report = makeReport test;
-    
 }