From 3e00482ba879e3b061d0dcf3f37d3263c48bfa34 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 18 Jan 2021 09:02:05 -0500 Subject: [PATCH 01/12] nixos/nomad: add assertion for the value of dropPrivileges and its relation to data_dir --- nixos/modules/services/networking/nomad.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index dafdae0c327..04f15fe1366 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -135,6 +135,13 @@ in }; }; + assertions = [ + { + assertion = cfg.dropPrivileges -> cfg.settings.data_dir == "/var/lib/nomad"; + message = "settings.data_dir must be equal to \"/var/lib/nomad\" if dropPrivileges is true"; + } + ]; + # Docker support requires the Docker daemon to be running. virtualisation.docker.enable = mkIf cfg.enableDocker true; }; From f3aa71b7ec19f380192e9e31c6f208988777e341 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 18 Jan 2021 09:03:52 -0500 Subject: [PATCH 02/12] nixos/nomad: describe the nomad cluster manager responsibilities --- nixos/modules/services/networking/nomad.nix | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 04f15fe1366..b58b5c9182c 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -66,6 +66,19 @@ in description = '' Configuration for Nomad. See the documentation for supported values. + + Notes about `data_dir`: + + If `data_dir` is set to a value other than the default value of + `"/var/lib/nomad"` it is the Nomad cluster manager's responsibility + to make sure that this directory exist and has the appropriate + permissions. One way to do this is with the `ExecStartPre` feature of + systemd. + + Additionally, if `dropPrivileges` is `true` then `data_dir` + **cannot** be customized. Setting `dropPrivileges` to `true` enables + the `DynamicUser` feature of systemd which directly manages and + operates on `StateDirectory`. ''; example = literalExample '' { From 58fe45936e89afef1e3ac35f80d07eb6b0a7ab20 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 18 Jan 2021 09:05:59 -0500 Subject: [PATCH 03/12] nixos/nomad: move serviceConfig into mkMerge and mkIf for docker SupplementaryGroups --- nixos/modules/services/networking/nomad.nix | 39 +++++++++++---------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index b58b5c9182c..10fe63f8425 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -122,25 +122,26 @@ in iptables ]); - serviceConfig = { - DynamicUser = cfg.dropPrivileges; - ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; - ExecStart = "${cfg.package}/bin/nomad agent -config=/etc/nomad.json" + - concatMapStrings (path: " -config=${path}") cfg.extraSettingsPaths; - KillMode = "process"; - KillSignal = "SIGINT"; - LimitNOFILE = 65536; - LimitNPROC = "infinity"; - OOMScoreAdjust = -1000; - Restart = "on-failure"; - RestartSec = 2; - # Agrees with the default `data_dir = "/var/lib/nomad"` in `settings` above. - StateDirectory = "nomad"; - TasksMax = "infinity"; - User = optionalString cfg.dropPrivileges "nomad"; - } // (optionalAttrs cfg.enableDocker { - SupplementaryGroups = "docker"; # space-separated string - }); + serviceConfig = mkMerge [ + { + DynamicUser = cfg.dropPrivileges; + ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; + ExecStart = "${cfg.package}/bin/nomad agent -config=/etc/nomad.json" + + concatMapStrings (path: " -config=${path}") cfg.extraSettingsPaths; + KillMode = "process"; + KillSignal = "SIGINT"; + LimitNOFILE = 65536; + LimitNPROC = "infinity"; + OOMScoreAdjust = -1000; + Restart = "on-failure"; + RestartSec = 2; + # Agrees with the default `data_dir = "/var/lib/nomad"` in `settings` above. + StateDirectory = "nomad"; + TasksMax = "infinity"; + User = optionalString cfg.dropPrivileges "nomad"; + } + (mkIf cfg.enableDocker { SupplementaryGroups = "docker"; }) # space-separated string + ]; unitConfig = { StartLimitIntervalSec = 10; From b80c4544592fe8cf665c4e76b91f912c50877c51 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 18 Jan 2021 09:07:09 -0500 Subject: [PATCH 04/12] nixos/nomad: add mkIf for StateDirectory --- nixos/modules/services/networking/nomad.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 10fe63f8425..0845b4f97b8 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -135,12 +135,11 @@ in OOMScoreAdjust = -1000; Restart = "on-failure"; RestartSec = 2; - # Agrees with the default `data_dir = "/var/lib/nomad"` in `settings` above. - StateDirectory = "nomad"; TasksMax = "infinity"; User = optionalString cfg.dropPrivileges "nomad"; } (mkIf cfg.enableDocker { SupplementaryGroups = "docker"; }) # space-separated string + (mkIf (cfg.settings.data_dir == "/var/lib/nomad") { StateDirectory = "nomad"; }) ]; unitConfig = { From b72a46713f12a311a9e5455a8fc446cd981850bb Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 18 Jan 2021 09:07:56 -0500 Subject: [PATCH 05/12] nixos/nomad: reformat SupplementaryGroups expression --- nixos/modules/services/networking/nomad.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 0845b4f97b8..3f06490faa7 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -138,7 +138,9 @@ in TasksMax = "infinity"; User = optionalString cfg.dropPrivileges "nomad"; } - (mkIf cfg.enableDocker { SupplementaryGroups = "docker"; }) # space-separated string + (mkIf cfg.enableDocker { + SupplementaryGroups = "docker"; # space-separated string + }) (mkIf (cfg.settings.data_dir == "/var/lib/nomad") { StateDirectory = "nomad"; }) ]; From bddb7ac4066574c868091a5bbd10f770bdae0c32 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Mon, 18 Jan 2021 09:17:32 -0500 Subject: [PATCH 06/12] nixos/nomad: fix typo and spell out ExecStartPre usage --- nixos/modules/services/networking/nomad.nix | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 3f06490faa7..80dcbe16f12 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -71,9 +71,10 @@ in If `data_dir` is set to a value other than the default value of `"/var/lib/nomad"` it is the Nomad cluster manager's responsibility - to make sure that this directory exist and has the appropriate - permissions. One way to do this is with the `ExecStartPre` feature of - systemd. + to make sure that this directory exists and has the appropriate + permissions. One way to ensure this is the case to create the + directory and adjust its permissions as needed using the + `ExecStartPre` feature of systemd. Additionally, if `dropPrivileges` is `true` then `data_dir` **cannot** be customized. Setting `dropPrivileges` to `true` enables From 5d0b3b7228b6a89033adb7df07db82efb7951b6f Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Tue, 19 Jan 2021 07:21:31 -0500 Subject: [PATCH 07/12] nixos/nomad: fix markup and remove suggestion --- nixos/modules/services/networking/nomad.nix | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 80dcbe16f12..7951930c3a1 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -67,19 +67,19 @@ in Configuration for Nomad. See the documentation for supported values. - Notes about `data_dir`: + Notes about data_dir: - If `data_dir` is set to a value other than the default value of - `"/var/lib/nomad"` it is the Nomad cluster manager's responsibility - to make sure that this directory exists and has the appropriate - permissions. One way to ensure this is the case to create the - directory and adjust its permissions as needed using the - `ExecStartPre` feature of systemd. + If data_dir is set to a value other than the + default value of "/var/lib/nomad" it is the Nomad + cluster manager's responsibility to make sure that this directory + exists and has the appropriate permissions. - Additionally, if `dropPrivileges` is `true` then `data_dir` - **cannot** be customized. Setting `dropPrivileges` to `true` enables - the `DynamicUser` feature of systemd which directly manages and - operates on `StateDirectory`. + Additionally, if dropPrivileges is + true then data_dir + cannot be customized. Setting + dropPrivileges to true enables + the DynamicUser feature of systemd which directly + manages and operates on StateDirectory. ''; example = literalExample '' { From 12b9249cf3cf4024b9ac4956e577126163110755 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Tue, 19 Jan 2021 07:24:04 -0500 Subject: [PATCH 08/12] nixos/nomad: unconditionally set user to nomad --- nixos/modules/services/networking/nomad.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 7951930c3a1..571baf67154 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -137,7 +137,7 @@ in Restart = "on-failure"; RestartSec = 2; TasksMax = "infinity"; - User = optionalString cfg.dropPrivileges "nomad"; + User = "nomad"; } (mkIf cfg.enableDocker { SupplementaryGroups = "docker"; # space-separated string From 5ce4ce61746c19f09ae1504a49f3216de6770055 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 23 Jan 2021 18:04:26 -0500 Subject: [PATCH 09/12] nixos/nomad: make formatting consistent in mkMerge call --- nixos/modules/services/networking/nomad.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 571baf67154..87b4ff18818 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -142,7 +142,9 @@ in (mkIf cfg.enableDocker { SupplementaryGroups = "docker"; # space-separated string }) - (mkIf (cfg.settings.data_dir == "/var/lib/nomad") { StateDirectory = "nomad"; }) + (mkIf (cfg.settings.data_dir == "/var/lib/nomad") { + StateDirectory = "nomad"; + }) ]; unitConfig = { From de71f5b50670908026842672aebe2565d239cda0 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 23 Jan 2021 18:19:50 -0500 Subject: [PATCH 10/12] nixos/tests: add test for custom nomad state directory --- nixos/tests/nomad.nix | 68 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/nixos/tests/nomad.nix b/nixos/tests/nomad.nix index bd052152bd6..51b11a8fef9 100644 --- a/nixos/tests/nomad.nix +++ b/nixos/tests/nomad.nix @@ -2,7 +2,7 @@ import ./make-test-python.nix ( { lib, ... }: { name = "nomad"; nodes = { - server = { pkgs, lib, ... }: { + default_server = { pkgs, lib, ... }: { networking = { interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{ address = "192.168.1.1"; @@ -30,24 +30,68 @@ import ./make-test-python.nix ( enableDocker = false; }; }; + + custom_state_dir_server = { pkgs, lib, ... }: { + networking = { + interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{ + address = "192.168.1.1"; + prefixLength = 16; + }]; + }; + + environment.etc."nomad.custom.json".source = + (pkgs.formats.json { }).generate "nomad.custom.json" { + region = "universe"; + datacenter = "earth"; + }; + + services.nomad = { + enable = true; + dropPrivileges = false; + + settings = { + data_dir = "/nomad/data/dir"; + server = { + enabled = true; + bootstrap_expect = 1; + }; + }; + + extraSettingsPaths = [ "/etc/nomad.custom.json" ]; + enableDocker = false; + }; + + systemd.services.nomad.serviceConfig.ExecStartPre = "${pkgs.writeShellScript "mk_data_dir" '' + set -euxo pipefail + + ${pkgs.coreutils}/bin/mkdir -p /nomad/data/dir + ''}"; + }; }; testScript = '' - server.wait_for_unit("nomad.service") + def test_nomad_server(server): + server.wait_for_unit("nomad.service") - # wait for healthy server - server.wait_until_succeeds( - "[ $(nomad operator raft list-peers | grep true | wc -l) == 1 ]" - ) + # wait for healthy server + server.wait_until_succeeds( + "[ $(nomad operator raft list-peers | grep true | wc -l) == 1 ]" + ) - # wait for server liveness - server.succeed("[ $(nomad server members | grep -o alive | wc -l) == 1 ]") + # wait for server liveness + server.succeed("[ $(nomad server members | grep -o alive | wc -l) == 1 ]") - # check the region - server.succeed("nomad server members | grep -o universe") + # check the region + server.succeed("nomad server members | grep -o universe") - # check the datacenter - server.succeed("[ $(nomad server members | grep -o earth | wc -l) == 1 ]") + # check the datacenter + server.succeed("[ $(nomad server members | grep -o earth | wc -l) == 1 ]") + + + servers = [default_server, custom_state_dir_server] + + for server in servers: + test_nomad_server(server) ''; } ) From 2a3cb407b04d322fd72bedc74b2751c40c36b4f1 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 23 Jan 2021 18:20:17 -0500 Subject: [PATCH 11/12] nixos/nomad: only set User if privileges are dropped --- nixos/modules/services/networking/nomad.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 87b4ff18818..60fefa2a1da 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -137,7 +137,7 @@ in Restart = "on-failure"; RestartSec = 2; TasksMax = "infinity"; - User = "nomad"; + User = optionalString cfg.dropPrivileges "nomad"; } (mkIf cfg.enableDocker { SupplementaryGroups = "docker"; # space-separated string From c7c3b9e4ae1396ee7c8291794a69141ff40508e3 Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Sat, 23 Jan 2021 19:44:28 -0500 Subject: [PATCH 12/12] nixos/nomad: remove User setting entirely --- nixos/modules/services/networking/nomad.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nixos/modules/services/networking/nomad.nix b/nixos/modules/services/networking/nomad.nix index 60fefa2a1da..9f1b443b89b 100644 --- a/nixos/modules/services/networking/nomad.nix +++ b/nixos/modules/services/networking/nomad.nix @@ -137,7 +137,6 @@ in Restart = "on-failure"; RestartSec = 2; TasksMax = "infinity"; - User = optionalString cfg.dropPrivileges "nomad"; } (mkIf cfg.enableDocker { SupplementaryGroups = "docker"; # space-separated string