public/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #109768 from cpcloud/nomad-datadir-cleanup

Browse Source 
nixos/nomad: enforce specific data_dir semantics
This commit is contained in:
Bernardo Meurer 2021-01-24 18:20:08 +00:00 committed by GitHub
commit 105b9eb1b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 99 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
nixos/modules/services/networking/nomad.nix
View File

@ -66,6 +66,20 @@ in
description = '' description = ''
Configuration for Nomad. See the <link xlink:href="https://www.nomadproject.io/docs/configuration">documentation</link> Configuration for Nomad. See the <link xlink:href="https://www.nomadproject.io/docs/configuration">documentation</link>
for supported values. for supported values.
Notes about <literal>data_dir</literal>:
If <literal>data_dir</literal> is set to a value other than the
default value of <literal>"/var/lib/nomad"</literal> it is the Nomad
cluster manager's responsibility to make sure that this directory
exists and has the appropriate permissions.
Additionally, if <literal>dropPrivileges</literal> is
<literal>true</literal> then <literal>data_dir</literal>
<emphasis>cannot</emphasis> be customized. Setting
<literal>dropPrivileges</literal> to <literal>true</literal> enables
the <literal>DynamicUser</literal> feature of systemd which directly
manages and operates on <literal>StateDirectory</literal>.
''; '';
example = literalExample '' example = literalExample ''
{ {
@ -109,25 +123,28 @@ in
iptables iptables
]); ]);
serviceConfig = { serviceConfig = mkMerge [
DynamicUser = cfg.dropPrivileges; {
ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID"; DynamicUser = cfg.dropPrivileges;
ExecStart = "${cfg.package}/bin/nomad agent -config=/etc/nomad.json" + ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
concatMapStrings (path: " -config=${path}") cfg.extraSettingsPaths; ExecStart = "${cfg.package}/bin/nomad agent -config=/etc/nomad.json" +
KillMode = "process"; concatMapStrings (path: " -config=${path}") cfg.extraSettingsPaths;
KillSignal = "SIGINT"; KillMode = "process";
LimitNOFILE = 65536; KillSignal = "SIGINT";
LimitNPROC = "infinity"; LimitNOFILE = 65536;
OOMScoreAdjust = -1000; LimitNPROC = "infinity";
Restart = "on-failure"; OOMScoreAdjust = -1000;
RestartSec = 2; Restart = "on-failure";
# Agrees with the default `data_dir = "/var/lib/nomad"` in `settings` above. RestartSec = 2;
StateDirectory = "nomad"; TasksMax = "infinity";
TasksMax = "infinity"; }
User = optionalString cfg.dropPrivileges "nomad"; (mkIf cfg.enableDocker {
} // (optionalAttrs cfg.enableDocker { SupplementaryGroups = "docker"; # space-separated string
SupplementaryGroups = "docker"; # space-separated string })
}); (mkIf (cfg.settings.data_dir == "/var/lib/nomad") {
StateDirectory = "nomad";
})
];
unitConfig = { unitConfig = {
StartLimitIntervalSec = 10; StartLimitIntervalSec = 10;
@ -135,6 +152,13 @@ in
}; };
}; };
assertions = [
{
assertion = cfg.dropPrivileges -> cfg.settings.data_dir == "/var/lib/nomad";
message = "settings.data_dir must be equal to \"/var/lib/nomad\" if dropPrivileges is true";
}
];
# Docker support requires the Docker daemon to be running. # Docker support requires the Docker daemon to be running.
virtualisation.docker.enable = mkIf cfg.enableDocker true; virtualisation.docker.enable = mkIf cfg.enableDocker true;
}; };

68
nixos/tests/nomad.nix
View File

@ -2,7 +2,7 @@ import ./make-test-python.nix (
{ lib, ... }: { { lib, ... }: {
name = "nomad"; name = "nomad";
nodes = { nodes = {
server = { pkgs, lib, ... }: { default_server = { pkgs, lib, ... }: {
networking = { networking = {
interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{ interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{
address = "192.168.1.1"; address = "192.168.1.1";
@ -30,24 +30,68 @@ import ./make-test-python.nix (
enableDocker = false; enableDocker = false;
}; };
}; };
custom_state_dir_server = { pkgs, lib, ... }: {
networking = {
interfaces.eth1.ipv4.addresses = lib.mkOverride 0 [{
address = "192.168.1.1";
prefixLength = 16;
}];
};
environment.etc."nomad.custom.json".source =
(pkgs.formats.json { }).generate "nomad.custom.json" {
region = "universe";
datacenter = "earth";
};
services.nomad = {
enable = true;
dropPrivileges = false;
settings = {
data_dir = "/nomad/data/dir";
server = {
enabled = true;
bootstrap_expect = 1;
};
};
extraSettingsPaths = [ "/etc/nomad.custom.json" ];
enableDocker = false;
};
systemd.services.nomad.serviceConfig.ExecStartPre = "${pkgs.writeShellScript "mk_data_dir" ''
set -euxo pipefail
${pkgs.coreutils}/bin/mkdir -p /nomad/data/dir
''}";
};
}; };
testScript = '' testScript = ''
server.wait_for_unit("nomad.service") def test_nomad_server(server):
server.wait_for_unit("nomad.service")
# wait for healthy server # wait for healthy server
server.wait_until_succeeds( server.wait_until_succeeds(
"[ $(nomad operator raft list-peers | grep true | wc -l) == 1 ]" "[ $(nomad operator raft list-peers | grep true | wc -l) == 1 ]"
) )
# wait for server liveness # wait for server liveness
server.succeed("[ $(nomad server members | grep -o alive | wc -l) == 1 ]") server.succeed("[ $(nomad server members | grep -o alive | wc -l) == 1 ]")
# check the region # check the region
server.succeed("nomad server members | grep -o universe") server.succeed("nomad server members | grep -o universe")
# check the datacenter # check the datacenter
server.succeed("[ $(nomad server members | grep -o earth | wc -l) == 1 ]") server.succeed("[ $(nomad server members | grep -o earth | wc -l) == 1 ]")
servers = [default_server, custom_state_dir_server]
for server in servers:
test_nomad_server(server)
''; '';
} }
) )