diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index dafdae0c327..9f1b443b89b 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -66,6 +66,20 @@ in description = '' Configuration for Nomad. See the documentation for supported values. + + Notes about data_dir: + + If data_dir is set to a value other than the + default value of "/var/lib/nomad" it is the Nomad + cluster manager's responsibility to make sure that this directory + exists and has the appropriate permissions. + + Additionally, if dropPrivileges is + true then data_dir + cannot be customized. Setting + dropPrivileges to true enables + the DynamicUser feature of systemd which directly + manages and operates on StateDirectory. ''; example = literalExample '' { @@ -109,25 +123,28 @@ in iptables ]); - serviceConfig = { - DynamicUser = cfg.dropPrivileges; - ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; - ExecStart = "${cfg.package}/bin/nomad agent -config=/etc/nomad.json" + - concatMapStrings (path: " -config=${path}") cfg.extraSettingsPaths; - KillMode = "process"; - KillSignal = "SIGINT"; - LimitNOFILE = 65536; - LimitNPROC = "infinity"; - OOMScoreAdjust = -1000; - Restart = "on-failure"; - RestartSec = 2; - # Agrees with the default `data_dir = "/var/lib/nomad"` in `settings` above. - StateDirectory = "nomad"; - TasksMax = "infinity"; - User = optionalString cfg.dropPrivileges "nomad"; - } // (optionalAttrs cfg.enableDocker { - SupplementaryGroups = "docker"; # space-separated string - }); + serviceConfig = mkMerge [ + { + DynamicUser = cfg.dropPrivileges; + ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + ExecStart = "${cfg.package}/bin/nomad agent -config=/etc/nomad.json" + + concatMapStrings (path: " -config=${path}") cfg.extraSettingsPaths; + KillMode = "process"; + KillSignal = "SIGINT"; + LimitNOFILE = 65536; + LimitNPROC = "infinity"; + OOMScoreAdjust = -1000; + Restart = "on-failure"; + RestartSec = 2; + TasksMax = "infinity"; + } + (mkIf cfg.enableDocker { + SupplementaryGroups = "docker"; # space-separated string + }) + (mkIf (cfg.settings.data_dir == "/var/lib/nomad") { + StateDirectory = "nomad"; + }) + ]; unitConfig = { StartLimitIntervalSec = 10; @@ -135,6 +152,13 @@ in }; }; + assertions = [ + { + assertion = cfg.dropPrivileges -> cfg.settings.data_dir == "/var/lib/nomad"; + message = "settings.data_dir must be equal to \"/var/lib/nomad\" if dropPrivileges is true"; + } + ]; + # Docker support requires the Docker daemon to be running. virtualisation.docker.enable = mkIf cfg.enableDocker true; }; diff --git a/nixos/tests/nomad.nix b/nixos/tests/nomad.nix index bd052152bd6..51b11a8fef9 100644 --- a/nixos/tests/nomad.nix +++ b/nixos/tests/nomad.nix @@ -2,7 +2,7 @@ import ./make-test-python.nix ( { lib, ... }: { name = "nomad"; nodes = { - server = { pkgs, lib, ... }: { + default_server = { pkgs, lib, ... }: { networking = { interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{ address = "192.168.1.1"; @@ -30,24 +30,68 @@ import ./make-test-python.nix ( enableDocker = false; }; }; + + custom_state_dir_server = { pkgs, lib, ... }: { + networking = { + interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{ + address = "192.168.1.1"; + prefixLength = 16; + }]; + }; + + environment.etc."nomad.custom.json".source = + (pkgs.formats.json { }).generate "nomad.custom.json" { + region = "universe"; + datacenter = "earth"; + }; + + services.nomad = { + enable = true; + dropPrivileges = false; + + settings = { + data_dir = "/nomad/data/dir"; + server = { + enabled = true; + bootstrap_expect = 1; + }; + }; + + extraSettingsPaths = [ "/etc/nomad.custom.json" ]; + enableDocker = false; + }; + + systemd.services.nomad.serviceConfig.ExecStartPre = "${pkgs.writeShellScript "mk_data_dir" '' + set -euxo pipefail + + ${pkgs.coreutils}/bin/mkdir -p /nomad/data/dir + ''}"; + }; }; testScript = '' - server.wait_for_unit("nomad.service") + def test_nomad_server(server): + server.wait_for_unit("nomad.service") - # wait for healthy server - server.wait_until_succeeds( - "[ $(nomad operator raft list-peers | grep true | wc -l) == 1 ]" - ) + # wait for healthy server + server.wait_until_succeeds( + "[ $(nomad operator raft list-peers | grep true | wc -l) == 1 ]" + ) - # wait for server liveness - server.succeed("[ $(nomad server members | grep -o alive | wc -l) == 1 ]") + # wait for server liveness + server.succeed("[ $(nomad server members | grep -o alive | wc -l) == 1 ]") - # check the region - server.succeed("nomad server members | grep -o universe") + # check the region + server.succeed("nomad server members | grep -o universe") - # check the datacenter - server.succeed("[ $(nomad server members | grep -o earth | wc -l) == 1 ]") + # check the datacenter + server.succeed("[ $(nomad server members | grep -o earth | wc -l) == 1 ]") + + + servers = [default_server, custom_state_dir_server] + + for server in servers: + test_nomad_server(server) ''; } )