diff --git a/nixos/modules/misc/ids.nix b/nixos/modules/misc/ids.nix index 73231edf077..aac86087f9e 100644 --- a/nixos/modules/misc/ids.nix +++ b/nixos/modules/misc/ids.nix @@ -317,6 +317,10 @@ restic = 291; openvpn = 292; meguca = 293; + yarn = 294; + hdfs = 295; + mapred = 296; + hadoop = 297; # When adding a uid, make sure it doesn't match an existing gid. And don't use uids above 399! @@ -594,6 +598,10 @@ restic = 291; openvpn = 292; meguca = 293; + yarn = 294; + hdfs = 295; + mapred = 296; + hadoop = 297; # When adding a gid, make sure it doesn't match an existing # uid. Users and groups with the same name should have equal diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index ea996acebb2..c70d52c94cd 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -178,6 +178,7 @@ ./services/backup/rsnapshot.nix ./services/backup/tarsnap.nix ./services/backup/znapzend.nix + ./services/cluster/hadoop/default.nix ./services/cluster/kubernetes/default.nix ./services/cluster/kubernetes/dns.nix ./services/cluster/kubernetes/dashboard.nix diff --git a/nixos/modules/services/cluster/hadoop/conf.nix b/nixos/modules/services/cluster/hadoop/conf.nix new file mode 100644 index 00000000000..38db10406b9 --- /dev/null +++ b/nixos/modules/services/cluster/hadoop/conf.nix @@ -0,0 +1,31 @@ +{ hadoop, pkgs }: +let + propertyXml = name: value: '' + + ${name} + ${builtins.toString value} + + ''; + siteXml = fileName: properties: pkgs.writeTextDir fileName '' + + + + ${builtins.concatStringsSep "\n" (pkgs.lib.mapAttrsToList propertyXml properties)} + + ''; + userFunctions = '' + hadoop_verify_logdir() { + echo Skipping verification of log directory + } + ''; +in +pkgs.buildEnv { + name = "hadoop-conf"; + paths = [ + (siteXml "core-site.xml" hadoop.coreSite) + (siteXml "hdfs-site.xml" hadoop.hdfsSite) + (siteXml "mapred-site.xml" hadoop.mapredSite) + (siteXml "yarn-site.xml" hadoop.yarnSite) + (pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions) + ]; +} diff --git a/nixos/modules/services/cluster/hadoop/default.nix b/nixos/modules/services/cluster/hadoop/default.nix new file mode 100644 index 00000000000..53c13fd0603 --- /dev/null +++ b/nixos/modules/services/cluster/hadoop/default.nix @@ -0,0 +1,63 @@ +{ config, lib, pkgs, ...}: +let + cfg = config.services.hadoop; + hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; }; +in +with lib; +{ + imports = [ ./yarn.nix ./hdfs.nix ]; + + options.services.hadoop = { + coreSite = mkOption { + default = {}; + example = { + "fs.defaultFS" = "hdfs://localhost"; + }; + description = "Hadoop core-site.xml definition"; + }; + + hdfsSite = mkOption { + default = {}; + example = { + "dfs.nameservices" = "namenode1"; + }; + description = "Hadoop hdfs-site.xml definition"; + }; + + mapredSite = mkOption { + default = {}; + example = { + "mapreduce.map.cpu.vcores" = "1"; + }; + description = "Hadoop mapred-site.xml definition"; + }; + + yarnSite = mkOption { + default = {}; + example = { + "yarn.resourcemanager.ha.id" = "resourcemanager1"; + }; + description = "Hadoop yarn-site.xml definition"; + }; + + package = mkOption { + type = types.package; + default = pkgs.hadoop; + defaultText = "pkgs.hadoop"; + example = literalExample "pkgs.hadoop"; + description = '' + ''; + }; + }; + + + config = mkMerge [ + (mkIf (builtins.hasAttr "yarn" config.users.extraUsers || + builtins.hasAttr "hdfs" config.users.extraUsers ) { + users.extraGroups.hadoop = { + gid = config.ids.gids.hadoop; + }; + }) + + ]; +} diff --git a/nixos/modules/services/cluster/hadoop/hdfs.nix b/nixos/modules/services/cluster/hadoop/hdfs.nix new file mode 100644 index 00000000000..48020e6139c --- /dev/null +++ b/nixos/modules/services/cluster/hadoop/hdfs.nix @@ -0,0 +1,73 @@ +{ config, lib, pkgs, ...}: +let + cfg = config.services.hadoop; + hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; }; +in +with lib; +{ + options.services.hadoop.hdfs = { + namenode.enabled = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run the Hadoop YARN NameNode + ''; + }; + datanode.enabled = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run the Hadoop YARN DataNode + ''; + }; + }; + + config = mkMerge [ + (mkIf cfg.hdfs.namenode.enabled { + systemd.services."hdfs-namenode" = { + description = "Hadoop HDFS NameNode"; + wantedBy = [ "multi-user.target" ]; + + environment = { + HADOOP_HOME = "${cfg.package}"; + }; + + preStart = '' + ${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true + ''; + + serviceConfig = { + User = "hdfs"; + SyslogIdentifier = "hdfs-namenode"; + ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode"; + }; + }; + }) + (mkIf cfg.hdfs.datanode.enabled { + systemd.services."hdfs-datanode" = { + description = "Hadoop HDFS DataNode"; + wantedBy = [ "multi-user.target" ]; + + environment = { + HADOOP_HOME = "${cfg.package}"; + }; + + serviceConfig = { + User = "hdfs"; + SyslogIdentifier = "hdfs-datanode"; + ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} datanode"; + }; + }; + }) + (mkIf ( + cfg.hdfs.namenode.enabled || cfg.hdfs.datanode.enabled + ) { + users.extraUsers.hdfs = { + description = "Hadoop HDFS user"; + group = "hadoop"; + uid = config.ids.uids.hdfs; + }; + }) + + ]; +} diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix new file mode 100644 index 00000000000..ce5b04a331c --- /dev/null +++ b/nixos/modules/services/cluster/hadoop/yarn.nix @@ -0,0 +1,74 @@ +{ config, lib, pkgs, ...}: +let + cfg = config.services.hadoop; + hadoopConf = import ./conf.nix { hadoop = cfg; pkgs = pkgs; }; +in +with lib; +{ + options.services.hadoop.yarn = { + resourcemanager.enabled = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run the Hadoop YARN ResourceManager + ''; + }; + nodemanager.enabled = mkOption { + type = types.bool; + default = false; + description = '' + Whether to run the Hadoop YARN NodeManager + ''; + }; + }; + + config = mkMerge [ + (mkIf ( + cfg.yarn.resourcemanager.enabled || cfg.yarn.nodemanager.enabled + ) { + + users.extraUsers.yarn = { + description = "Hadoop YARN user"; + group = "hadoop"; + uid = config.ids.uids.yarn; + }; + }) + + (mkIf cfg.yarn.resourcemanager.enabled { + systemd.services."yarn-resourcemanager" = { + description = "Hadoop YARN ResourceManager"; + wantedBy = [ "multi-user.target" ]; + + environment = { + HADOOP_HOME = "${cfg.package}"; + }; + + serviceConfig = { + User = "yarn"; + SyslogIdentifier = "yarn-resourcemanager"; + ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " + + " resourcemanager"; + }; + }; + }) + + (mkIf cfg.yarn.nodemanager.enabled { + systemd.services."yarn-nodemanager" = { + description = "Hadoop YARN NodeManager"; + wantedBy = [ "multi-user.target" ]; + + environment = { + HADOOP_HOME = "${cfg.package}"; + }; + + serviceConfig = { + User = "yarn"; + SyslogIdentifier = "yarn-nodemanager"; + ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " + + " nodemanager"; + }; + }; + }) + + ]; +} diff --git a/nixos/release.nix b/nixos/release.nix index 0d3ae997469..67820149437 100644 --- a/nixos/release.nix +++ b/nixos/release.nix @@ -299,6 +299,8 @@ in rec { tests.gnome3-gdm = callTest tests/gnome3-gdm.nix {}; tests.grafana = callTest tests/grafana.nix {}; tests.graphite = callTest tests/graphite.nix {}; + tests.hadoop.hdfs = callTestOnMatchingSystems [ "x86_64-linux" ] tests/hadoop/hdfs.nix {}; + tests.hadoop.yarn = callTestOnMatchingSystems [ "x86_64-linux" ] tests/hadoop/yarn.nix {}; tests.hardened = callTest tests/hardened.nix { }; tests.haproxy = callTest tests/haproxy.nix {}; tests.hibernate = callTest tests/hibernate.nix {}; diff --git a/nixos/tests/hadoop/hdfs.nix b/nixos/tests/hadoop/hdfs.nix new file mode 100644 index 00000000000..4206c940c1a --- /dev/null +++ b/nixos/tests/hadoop/hdfs.nix @@ -0,0 +1,54 @@ +import ../make-test.nix ({pkgs, ...}: { + nodes = { + namenode = {pkgs, config, ...}: { + services.hadoop = { + package = pkgs.hadoop_3_1; + hdfs.namenode.enabled = true; + coreSite = { + "fs.defaultFS" = "hdfs://namenode:8020"; + }; + hdfsSite = { + "dfs.replication" = 1; + "dfs.namenode.rpc-bind-host" = "0.0.0.0"; + "dfs.namenode.http-bind-host" = "0.0.0.0"; + }; + }; + networking.firewall.allowedTCPPorts = [ + 9870 # namenode.http-address + 8020 # namenode.rpc-address + ]; + }; + datanode = {pkgs, config, ...}: { + services.hadoop = { + package = pkgs.hadoop_3_1; + hdfs.datanode.enabled = true; + coreSite = { + "fs.defaultFS" = "hdfs://namenode:8020"; + }; + }; + networking.firewall.allowedTCPPorts = [ + 9864 # datanode.http.address + 9866 # datanode.address + 9867 # datanode.ipc.address + ]; + }; + }; + + testScript = '' + startAll + + $namenode->waitForUnit("hdfs-namenode"); + $namenode->waitForUnit("network.target"); + $namenode->waitForOpenPort(8020); + $namenode->waitForOpenPort(9870); + + $datanode->waitForUnit("hdfs-datanode"); + $datanode->waitForUnit("network.target"); + $datanode->waitForOpenPort(9864); + $datanode->waitForOpenPort(9866); + $datanode->waitForOpenPort(9867); + + $namenode->succeed("curl http://namenode:9870"); + $datanode->succeed("curl http://datanode:9864"); + ''; +}) diff --git a/nixos/tests/hadoop/yarn.nix b/nixos/tests/hadoop/yarn.nix new file mode 100644 index 00000000000..e97cc1acc90 --- /dev/null +++ b/nixos/tests/hadoop/yarn.nix @@ -0,0 +1,46 @@ +import ../make-test.nix ({pkgs, ...}: { + nodes = { + resourcemanager = {pkgs, config, ...}: { + services.hadoop.package = pkgs.hadoop_3_1; + services.hadoop.yarn.resourcemanager.enabled = true; + services.hadoop.yarnSite = { + "yarn.resourcemanager.scheduler.class" = "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler"; + }; + networking.firewall.allowedTCPPorts = [ + 8088 # resourcemanager.webapp.address + 8031 # resourcemanager.resource-tracker.address + ]; + }; + nodemanager = {pkgs, config, ...}: { + services.hadoop.package = pkgs.hadoop_3_1; + services.hadoop.yarn.nodemanager.enabled = true; + services.hadoop.yarnSite = { + "yarn.resourcemanager.hostname" = "resourcemanager"; + "yarn.nodemanager.log-dirs" = "/tmp/userlogs"; + "yarn.nodemanager.address" = "0.0.0.0:8041"; + }; + networking.firewall.allowedTCPPorts = [ + 8042 # nodemanager.webapp.address + 8041 # nodemanager.address + ]; + }; + + }; + + testScript = '' + startAll; + + $resourcemanager->waitForUnit("yarn-resourcemanager"); + $resourcemanager->waitForUnit("network.target"); + $resourcemanager->waitForOpenPort(8031); + $resourcemanager->waitForOpenPort(8088); + + $nodemanager->waitForUnit("yarn-nodemanager"); + $nodemanager->waitForUnit("network.target"); + $nodemanager->waitForOpenPort(8042); + $nodemanager->waitForOpenPort(8041); + + $resourcemanager->succeed("curl http://localhost:8088"); + $nodemanager->succeed("curl http://localhost:8042"); + ''; +})