Merge pull request #74898 from tfc/nixos-test-retry

nixos/test: Use retry() in all looping functions that need timeouts

nixos/lib/test-driver/test-driver.py

@@ -312,8 +312,13 @@ class Machine:
         self.monitor.send(message)
         return self.wait_for_monitor_prompt()
 
-    def wait_for_unit(self, unit: str, user: Optional[str] = None) -> bool:
+    def wait_for_unit(self, unit: str, user: Optional[str] = None) -> None:
-        while True:
+        """Wait for a systemd unit to get into "active" state.
+        Throws exceptions on "failed" and "inactive" states as well as
+        after timing out.
+        """
+
+        def check_active(_: Any) -> bool:
             info = self.get_unit_info(unit, user)
             state = info["ActiveState"]
             if state == "failed":
@@ -329,8 +334,10 @@ class Machine:
                                 'unit "{}" is inactive and there ' "are no pending jobs"
                             ).format(unit)
                         )
-            if state == "active":
+
-                return True
+            return state == "active"
+
+        retry(check_active)
 
     def get_unit_info(self, unit: str, user: Optional[str] = None) -> Dict[str, str]:
         status, lines = self.systemctl('--no-pager show "{}"'.format(unit), user)
@@ -421,17 +428,33 @@ class Machine:
                     )
 
     def wait_until_succeeds(self, command: str) -> str:
-        with self.nested("waiting for success: {}".format(command)):
+        """Wait until a command returns success and return its output.
-            while True:
+        Throws an exception on timeout.
+        """
+        output = ""
+
+        def check_success(_: Any) -> bool:
+            nonlocal output
             status, output = self.execute(command)
-                if status == 0:
+            return status == 0
+
+        with self.nested("waiting for success: {}".format(command)):
+            retry(check_success)
             return output
 
     def wait_until_fails(self, command: str) -> str:
-        with self.nested("waiting for failure: {}".format(command)):
+        """Wait until a command returns failure.
-            while True:
+        Throws an exception on timeout.
+        """
+        output = ""
+
+        def check_failure(_: Any) -> bool:
+            nonlocal output
             status, output = self.execute(command)
-                if status != 0:
+            return status != 0
+
+        with self.nested("waiting for failure: {}".format(command)):
+            retry(check_failure)
             return output
 
     def wait_for_shutdown(self) -> None:
@@ -453,25 +476,38 @@ class Machine:
         )
         return output
 
-    def wait_until_tty_matches(self, tty: str, regexp: str) -> bool:
+    def wait_until_tty_matches(self, tty: str, regexp: str) -> None:
+        """Wait until the visible output on the chosen TTY matches regular
+        expression. Throws an exception on timeout.
+        """
         matcher = re.compile(regexp)
-        with self.nested("waiting for {} to appear on tty {}".format(regexp, tty)):
+
-            while True:
+        def tty_matches(last: bool) -> bool:
             text = self.get_tty_text(tty)
-                if len(matcher.findall(text)) > 0:
+            if last:
-                    return True
+                self.log(
+                    f"Last chance to match /{regexp}/ on TTY{tty}, "
+                    f"which currently contains: {text}"
+                )
+            return len(matcher.findall(text)) > 0
+
+        with self.nested("waiting for {} to appear on tty {}".format(regexp, tty)):
+            retry(tty_matches)
 
     def send_chars(self, chars: List[str]) -> None:
         with self.nested("sending keys ‘{}‘".format(chars)):
             for char in chars:
                 self.send_key(char)
 
-    def wait_for_file(self, filename: str) -> bool:
+    def wait_for_file(self, filename: str) -> None:
-        with self.nested("waiting for file ‘{}‘".format(filename)):
+        """Waits until the file exists in machine's file system."""
-            while True:
+
+        def check_file(_: Any) -> bool:
             status, _ = self.execute("test -e {}".format(filename))
-                if status == 0:
+            return status == 0
-                    return True
+
+        with self.nested("waiting for file ‘{}‘".format(filename)):
+            retry(check_file)
 
     def wait_for_open_port(self, port: int) -> None:
         def port_is_open(_: Any) -> bool:
@@ -494,8 +530,8 @@ class Machine:
     def stop_job(self, jobname: str, user: Optional[str] = None) -> Tuple[int, str]:
         return self.systemctl("stop {}".format(jobname), user)
 
-    def wait_for_job(self, jobname: str) -> bool:
+    def wait_for_job(self, jobname: str) -> None:
-        return self.wait_for_unit(jobname)
+        self.wait_for_unit(jobname)
 
     def connect(self) -> None:
         if self.connected:
@@ -700,18 +736,20 @@ class Machine:
         """Wait until it is possible to connect to the X server.  Note that
         testing the existence of /tmp/.X11-unix/X0 is insufficient.
         """
-        with self.nested("waiting for the X11 server"):
+
-            while True:
+        def check_x(_: Any) -> bool:
             cmd = (
                 "journalctl -b SYSLOG_IDENTIFIER=systemd | "
                 + 'grep "Reached target Current graphical"'
             )
             status, _ = self.execute(cmd)
             if status != 0:
-                    continue
+                return False
             status, _ = self.execute("[ -e /tmp/.X11-unix/X0 ]")
-                if status == 0:
+            return status == 0
-                    return
+
+        with self.nested("waiting for the X11 server"):
+            retry(check_x)
 
     def get_window_names(self) -> List[str]:
         return self.succeed(

nixos/tests/pam-oath-login.nix

@@ -1,4 +1,4 @@
-import ./make-test.nix ({ ... }:
+import ./make-test-python.nix ({ ... }:
 
 let
   oathSnakeoilSecret = "cdd4083ef8ff1fa9178c6d46bfb1a3";
@@ -55,70 +55,54 @@ in
       };
     };
 
-  testScript =
+  testScript = ''
-    ''
+    def switch_to_tty(tty_number):
-      $machine->waitForUnit('multi-user.target');
+        machine.fail(f"pgrep -f 'agetty.*tty{tty_number}'")
-      $machine->waitUntilSucceeds("pgrep -f 'agetty.*tty1'");
+        machine.send_key(f"alt-f{tty_number}")
-      $machine->screenshot("postboot");
+        machine.wait_until_succeeds(f"[ $(fgconsole) = {tty_number} ]")
+        machine.wait_for_unit(f"getty@tty{tty_number}.service")
+        machine.wait_until_succeeds(f"pgrep -f 'agetty.*tty{tty_number}'")
 
 
-      subtest "Invalid password", sub {
+    def enter_user_alice(tty_number):
-        $machine->fail("pgrep -f 'agetty.*tty2'");
+        machine.wait_until_tty_matches(tty_number, "login: ")
-        $machine->sendKeys("alt-f2");
+        machine.send_chars("alice\n")
-        $machine->waitUntilSucceeds("[ \$(fgconsole) = 2 ]");
+        machine.wait_until_tty_matches(tty_number, "login: alice")
-        $machine->waitForUnit('getty@tty2.service');
+        machine.wait_until_succeeds("pgrep login")
-        $machine->waitUntilSucceeds("pgrep -f 'agetty.*tty2'");
+        machine.wait_until_tty_matches(tty_number, "One-time password")
 
-        $machine->waitUntilTTYMatches(2, "login: ");
-        $machine->sendChars("alice\n");
-        $machine->waitUntilTTYMatches(2, "login: alice");
-        $machine->waitUntilSucceeds("pgrep login");
 
-        $machine->waitUntilTTYMatches(2, "One-time password");
+    machine.wait_for_unit("multi-user.target")
-        $machine->sendChars("${oathSnakeOilPassword1}\n");
+    machine.wait_until_succeeds("pgrep -f 'agetty.*tty1'")
-        $machine->waitUntilTTYMatches(2, "Password: ");
+    machine.screenshot("postboot")
-        $machine->sendChars("blorg\n");
-        $machine->waitUntilTTYMatches(2, "Login incorrect");
-      };
 
-      subtest "Invalid oath token", sub {
+    with subtest("Invalid password"):
-        $machine->fail("pgrep -f 'agetty.*tty3'");
+        switch_to_tty(2)
-        $machine->sendKeys("alt-f3");
+        enter_user_alice(2)
-        $machine->waitUntilSucceeds("[ \$(fgconsole) = 3 ]");
-        $machine->waitForUnit('getty@tty3.service');
-        $machine->waitUntilSucceeds("pgrep -f 'agetty.*tty3'");
 
-        $machine->waitUntilTTYMatches(3, "login: ");
+        machine.send_chars("${oathSnakeOilPassword1}\n")
-        $machine->sendChars("alice\n");
+        machine.wait_until_tty_matches(2, "Password: ")
-        $machine->waitUntilTTYMatches(3, "login: alice");
+        machine.send_chars("blorg\n")
-        $machine->waitUntilSucceeds("pgrep login");
+        machine.wait_until_tty_matches(2, "Login incorrect")
-        $machine->waitUntilTTYMatches(3, "One-time password");
-        $machine->sendChars("000000\n");
-        $machine->waitUntilTTYMatches(3, "Login incorrect");
-        $machine->waitUntilTTYMatches(3, "login:");
-      };
 
-      subtest "Happy path (both passwords are mandatory to get us in)", sub {
+    with subtest("Invalid oath token"):
-        $machine->fail("pgrep -f 'agetty.*tty4'");
+        switch_to_tty(3)
-        $machine->sendKeys("alt-f4");
+        enter_user_alice(3)
-        $machine->waitUntilSucceeds("[ \$(fgconsole) = 4 ]");
-        $machine->waitForUnit('getty@tty4.service');
-        $machine->waitUntilSucceeds("pgrep -f 'agetty.*tty4'");
 
-        $machine->waitUntilTTYMatches(4, "login: ");
+        machine.send_chars("000000\n")
-        $machine->sendChars("alice\n");
+        machine.wait_until_tty_matches(3, "Login incorrect")
-        $machine->waitUntilTTYMatches(4, "login: alice");
+        machine.wait_until_tty_matches(3, "login:")
-        $machine->waitUntilSucceeds("pgrep login");
-        $machine->waitUntilTTYMatches(4, "One-time password");
-        $machine->sendChars("${oathSnakeOilPassword2}\n");
-        $machine->waitUntilTTYMatches(4, "Password: ");
-        $machine->sendChars("${alicePassword}\n");
 
-        $machine->waitUntilSucceeds("pgrep -u alice bash");
+    with subtest("Happy path: Both passwords are mandatory to get us in"):
-        $machine->sendChars("touch  done4\n");
+        switch_to_tty(4)
-        $machine->waitForFile("/home/alice/done4");
+        enter_user_alice(4)
-      };
 
+        machine.send_chars("${oathSnakeOilPassword2}\n")
+        machine.wait_until_tty_matches(4, "Password: ")
+        machine.send_chars("${alicePassword}\n")
+
+        machine.wait_until_succeeds("pgrep -u alice bash")
+        machine.send_chars("touch  done4\n")
+        machine.wait_for_file("/home/alice/done4")
     '';
-
 })