nixos/ceph: use ConditionPathExists to delay ceph daemon start

This prevents services to be started before they're initialized, and
renders the `systemd.targets.ceph.wantedBy = lib.mkForce [];` hack in
the vm tests obsolete - The config now starts up ceph after a reboot,
too.

Let's take advantage of that, crash all VMs, and boot them up again.

nixos/modules/services/network-filesystems/ceph.nix

@@ -14,7 +14,9 @@ let
       { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph; })
       daemonIds));
 
-  makeService = (daemonType: daemonId: clusterName: ceph: {
+  makeService = (daemonType: daemonId: clusterName: ceph:
+    let
+      stateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"; in {
     enable = true;
     description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
     after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
@@ -24,6 +26,9 @@ let
 
     path = [ pkgs.getopt ];
 
+    # Don't start services that are not yet initialized
+    unitConfig.ConditionPathExists = "/var/lib/${stateDirectory}/keyring";
+
     serviceConfig = {
       LimitNOFILE = 1048576;
       LimitNPROC = 1048576;
@@ -36,7 +41,7 @@ let
       Restart = "on-failure";
       StartLimitBurst = "5";
       StartLimitInterval = "30min";
-      StateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}";
+      StateDirectory = stateDirectory;
       User = "ceph";
       Group = if daemonType == "osd" then "disk" else "ceph";
       ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
@@ -60,6 +65,7 @@ let
         partOf = [ "ceph.target" ];
         wantedBy = [ "ceph.target" ];
         before = [ "ceph.target" ];
+        unitConfig.StopWhenUnneeded = true;
       };
     }
   );
@@ -389,9 +395,12 @@ in
 
     systemd.targets = let
       targets = [
-        { ceph = { description = "Ceph target allowing to start/stop all ceph service instances at once";
+        { ceph = {
-                     wantedBy = [ "multi-user.target" ]; }; }
+          description = "Ceph target allowing to start/stop all ceph service instances at once";
-      ] ++ optional cfg.mon.enable (makeTarget "mon")
+          wantedBy = [ "multi-user.target" ];
+          unitConfig.StopWhenUnneeded = true;
+        }; } ]
+        ++ optional cfg.mon.enable (makeTarget "mon")
         ++ optional cfg.mds.enable (makeTarget "mds")
         ++ optional cfg.osd.enable (makeTarget "osd")
         ++ optional cfg.rgw.enable (makeTarget "rgw")

nixos/tests/ceph-multi-node.nix

@@ -49,9 +49,6 @@ let
     boot.kernelModules = [ "xfs" ];
 
     services.ceph = cephConfig;
-
-    # So that we don't have to battle systemd when bootstraping
-    systemd.targets.ceph.wantedBy = lib.mkForce [];
   };
 
   networkMonA = {
@@ -191,22 +188,17 @@ let
       "ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it"
     );
 
-    # As we disable the target in the config, we still want to test that it works as intended
+    # Shut down ceph on all machines in a very unpolite way
-    $osd0->mustSucceed("systemctl stop ceph-osd-${cfg.osd0.name}");
+    $monA->crash;
-    $osd1->mustSucceed("systemctl stop ceph-osd-${cfg.osd1.name}");
+    $osd0->crash;
-    $monA->mustSucceed(
+    $osd1->crash;
-      "systemctl stop ceph-mgr-${cfg.monA.name}",
-      "systemctl stop ceph-mon-${cfg.monA.name}"
-    );
 
-    $monA->succeed("systemctl start ceph.target");
+    # Start it up
-    $monA->waitForUnit("ceph-mon-${cfg.monA.name}");
+    $osd0->start;
-    $monA->waitForUnit("ceph-mgr-${cfg.monA.name}");
+    $osd1->start;
-    $osd0->succeed("systemctl start ceph.target");
+    $monA->start;
-    $osd0->waitForUnit("ceph-osd-${cfg.osd0.name}");
-    $osd1->succeed("systemctl start ceph.target");
-    $osd1->waitForUnit("ceph-osd-${cfg.osd1.name}");
 
+    # Ensure the cluster comes back up again
     $monA->succeed("ceph -s | grep 'mon: 1 daemons'");
     $monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'");
     $monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");

nixos/tests/ceph-single-node.nix

@@ -46,9 +46,6 @@ let
     boot.kernelModules = [ "xfs" ];
 
     services.ceph = cephConfig;
-
-    # So that we don't have to battle systemd when bootstraping
-    systemd.targets.ceph.wantedBy = lib.mkForce [];
   };
 
   networkMonA = {
@@ -151,20 +148,17 @@ let
       "ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it"
     );
 
-    # As we disable the target in the config, we still want to test that it works as intended
+    # Shut down ceph by stopping ceph.target.
-    $monA->mustSucceed(
+    $monA->mustSucceed("systemctl stop ceph.target");
-      "systemctl stop ceph-osd-${cfg.osd0.name}",
-      "systemctl stop ceph-osd-${cfg.osd1.name}",
-      "systemctl stop ceph-mgr-${cfg.monA.name}",
-      "systemctl stop ceph-mon-${cfg.monA.name}"
-    );
 
+    # Start it up
     $monA->succeed("systemctl start ceph.target");
     $monA->waitForUnit("ceph-mon-${cfg.monA.name}");
     $monA->waitForUnit("ceph-mgr-${cfg.monA.name}");
     $monA->waitForUnit("ceph-osd-${cfg.osd0.name}");
     $monA->waitForUnit("ceph-osd-${cfg.osd1.name}");
 
+    # Ensure the cluster comes back up again
     $monA->succeed("ceph -s | grep 'mon: 1 daemons'");
     $monA->waitUntilSucceeds("ceph -s | grep 'quorum ${cfg.monA.name}'");
     $monA->waitUntilSucceeds("ceph osd stat | grep -e '2 osds: 2 up[^,]*, 2 in'");