From 2906718bccd16eac7a197d6eca32be6931a707a9 Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Wed, 21 Aug 2019 00:25:41 +0200 Subject: [PATCH 1/3] orangefs: init at 2.9.7 --- pkgs/tools/filesystems/orangefs/default.nix | 61 +++++++++++++++++++++ pkgs/top-level/all-packages.nix | 2 + 2 files changed, 63 insertions(+) create mode 100644 pkgs/tools/filesystems/orangefs/default.nix diff --git a/pkgs/tools/filesystems/orangefs/default.nix b/pkgs/tools/filesystems/orangefs/default.nix new file mode 100644 index 00000000000..37f4026799b --- /dev/null +++ b/pkgs/tools/filesystems/orangefs/default.nix @@ -0,0 +1,61 @@ +{ stdenv, fetchurl, bison, flex, autoreconfHook +, openssl, db, attr, perl, tcsh +} : + +stdenv.mkDerivation rec { + pname = "orangefs"; + version = "2.9.7"; + + src = fetchurl { + url = "http://download.orangefs.org/current/source/orangefs-${version}.tar.gz"; + sha256 = "15669f5rcvn44wkas0mld0qmyclrmhbrw4bbbp66sw3a12vgn4sm"; + }; + + nativeBuildInputs = [ bison flex perl autoreconfHook ]; + buildInputs = [ openssl db attr tcsh ]; + + postPatch = '' + # Issue introduced by attr-2.4.48 + substituteInPlace src/apps/user/ofs_setdirhint.c --replace attr/xattr.h sys/xattr.h + + # Do not try to install empty sysconfdir + substituteInPlace Makefile.in --replace 'install -d $(sysconfdir)' "" + + # perl interpreter needs to be fixed or build fails + patchShebangs ./src/apps/admin/pvfs2-genconfig + + # symlink points to a location in /usr + rm ./src/client/webpack/ltmain.sh + ''; + + configureFlags = [ + "--sysconfdir=/etc/orangefs" + "--enable-shared" + "--enable-fast" + "--with-ssl=${stdenv.lib.getDev openssl}" + ]; + + + enableParallelBuilding = true; + + postInstall = '' + # install useful helper scripts + install examples/keys/pvfs2-gen-keys.sh $out/bin + ''; + + postFixup = '' + for f in pvfs2-getmattr pvfs2-setmattr; do + substituteInPlace $out/bin/$f --replace '#!/bin/csh' '#!${tcsh}/bin/tcsh' + done + + sed -i 's:openssl:${openssl}/bin/openssl:' $out/bin/pvfs2-gen-keys.sh + ''; + + meta = with stdenv.lib; { + description = "Scale-out network file system for use on high-end computing systems"; + homepage = "http://www.orangefs.org/"; + license = with licenses; [ asl20 bsd3 gpl2 lgpl21 lgpl21Plus openldap ]; + platforms = [ "x86_64-linux" ]; + maintainers = with maintainers; [ markuskowa ]; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index d4211308c6f..d7a47ec6f0b 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -5261,6 +5261,8 @@ in opn2bankeditor = callPackage ../tools/audio/opl3bankeditor/opn2bankeditor.nix { }; + orangefs = callPackage ../tools/filesystems/orangefs { }; + os-prober = callPackage ../tools/misc/os-prober {}; osl = callPackage ../development/compilers/osl { }; From 8b4ce06e63eb6e52a2fcfd655db46226dc364c0e Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Wed, 21 Aug 2019 00:26:36 +0200 Subject: [PATCH 2/3] nixos/orangefs: add modules for server and client --- nixos/modules/module-list.nix | 2 + .../network-filesystems/orangefs/client.nix | 97 ++++++++ .../network-filesystems/orangefs/server.nix | 225 ++++++++++++++++++ 3 files changed, 324 insertions(+) create mode 100644 nixos/modules/services/network-filesystems/orangefs/client.nix create mode 100644 nixos/modules/services/network-filesystems/orangefs/server.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index fb8453f1d53..6d8162b4c6e 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -547,6 +547,8 @@ ./services/network-filesystems/nfsd.nix ./services/network-filesystems/openafs/client.nix ./services/network-filesystems/openafs/server.nix + ./services/network-filesystems/orangefs/server.nix + ./services/network-filesystems/orangefs/client.nix ./services/network-filesystems/rsyncd.nix ./services/network-filesystems/samba.nix ./services/network-filesystems/tahoe.nix diff --git a/nixos/modules/services/network-filesystems/orangefs/client.nix b/nixos/modules/services/network-filesystems/orangefs/client.nix new file mode 100644 index 00000000000..b69d9e713c3 --- /dev/null +++ b/nixos/modules/services/network-filesystems/orangefs/client.nix @@ -0,0 +1,97 @@ +{ config, lib, pkgs, ...} : + +with lib; + +let + cfg = config.services.orangefs.client; + +in { + ###### interface + + options = { + services.orangefs.client = { + enable = mkEnableOption "OrangeFS client daemon"; + + extraOptions = mkOption { + type = with types; listOf str; + default = []; + description = "Extra command line options for pvfs2-client."; + }; + + fileSystems = mkOption { + description = '' + The orangefs file systems to be mounted. + This option is prefered over using directly since + the pvfs client service needs to be running for it to be mounted. + ''; + + example = [{ + mountPoint = "/orangefs"; + target = "tcp://server:3334/orangefs"; + }]; + + type = with types; listOf (submodule ({ ... } : { + options = { + + mountPoint = mkOption { + type = types.str; + default = "/orangefs"; + description = "Mount point."; + }; + + options = mkOption { + type = with types; listOf str; + default = []; + description = "Mount options"; + }; + + target = mkOption { + type = types.str; + default = null; + example = "tcp://server:3334/orangefs"; + description = "Target URL"; + }; + }; + })); + }; + }; + }; + + + ###### implementation + + config = mkIf cfg.enable { + environment.systemPackages = [ pkgs.orangefs ]; + + boot.supportedFilesystems = [ "pvfs2" ]; + boot.kernelModules = [ "orangefs" ]; + + systemd.services.orangefs-client = { + requires = [ "network-online.target" ]; + after = [ "network-online.target" ]; + + serviceConfig = { + Type = "simple"; + + ExecStart = '' + ${pkgs.orangefs}/bin/pvfs2-client-core \ + --logtype=syslog ${concatStringsSep " " cfg.extraOptions} + ''; + + TimeoutStopSec = "120"; + }; + }; + + systemd.mounts = map (fs: { + requires = [ "orangefs-client.service" ]; + after = [ "orangefs-client.service" ]; + bindsTo = [ "orangefs-client.service" ]; + wantedBy = [ "remote-fs.target" ]; + type = "pvfs2"; + options = concatStringsSep "," fs.options; + what = fs.target; + where = fs.mountPoint; + }) cfg.fileSystems; + }; +} + diff --git a/nixos/modules/services/network-filesystems/orangefs/server.nix b/nixos/modules/services/network-filesystems/orangefs/server.nix new file mode 100644 index 00000000000..74ebdc13402 --- /dev/null +++ b/nixos/modules/services/network-filesystems/orangefs/server.nix @@ -0,0 +1,225 @@ +{ config, lib, pkgs, ...} : + +with lib; + +let + cfg = config.services.orangefs.server; + + aliases = mapAttrsToList (alias: url: alias) cfg.servers; + + # Maximum handle number is 2^63 + maxHandle = 9223372036854775806; + + # One range of handles for each meta/data instance + handleStep = maxHandle / (length aliases) / 2; + + fileSystems = mapAttrsToList (name: fs: '' + + Name ${name} + ID ${toString fs.id} + RootHandle ${toString fs.rootHandle} + + ${fs.extraConfig} + + + ${concatStringsSep "\n" ( + imap0 (i: alias: + let + begin = i * handleStep + 3; + end = begin + handleStep - 1; + in "Range ${alias} ${toString begin}-${toString end}") aliases + )} + + + + ${concatStringsSep "\n" ( + imap0 (i: alias: + let + begin = i * handleStep + 3 + (length aliases) * handleStep; + end = begin + handleStep - 1; + in "Range ${alias} ${toString begin}-${toString end}") aliases + )} + + + + TroveSyncMeta ${if fs.troveSyncMeta then "yes" else "no"} + TroveSyncData ${if fs.troveSyncData then "yes" else "no"} + ${fs.extraStorageHints} + + + + '') cfg.fileSystems; + + configFile = '' + + LogType ${cfg.logType} + DataStorageSpace ${cfg.dataStorageSpace} + MetaDataStorageSpace ${cfg.metadataStorageSpace} + + BMIModules ${concatStringsSep "," cfg.BMIModules} + ${cfg.extraDefaults} + + + ${cfg.extraConfig} + + + ${concatStringsSep "\n" (mapAttrsToList (alias: url: "Alias ${alias} ${url}") cfg.servers)} + + + ${concatStringsSep "\n" fileSystems} + ''; + +in { + ###### interface + + options = { + services.orangefs.server = { + enable = mkEnableOption "OrangeFS server"; + + logType = mkOption { + type = with types; enum [ "file" "syslog" ]; + default = "syslog"; + description = "Destination for log messages."; + }; + + dataStorageSpace = mkOption { + type = types.str; + default = null; + example = "/data/storage"; + description = "Directory for data storage."; + }; + + metadataStorageSpace = mkOption { + type = types.str; + default = null; + example = "/data/meta"; + description = "Directory for meta data storage."; + }; + + BMIModules = mkOption { + type = with types; listOf str; + default = [ "bmi_tcp" ]; + example = [ "bmi_tcp" "bmi_ib"]; + description = "List of BMI modules to load."; + }; + + extraDefaults = mkOption { + type = types.lines; + default = ""; + description = "Extra config for <Defaults> section."; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = "Extra config for the global section."; + }; + + servers = mkOption { + type = with types; attrsOf types.str; + default = {}; + example = '' + { + node1="tcp://node1:3334"; + node2="tcp://node2:3334"; + } + ''; + description = "URLs for storage server including port. The attribute names define the server alias."; + }; + + fileSystems = mkOption { + description = '' + These options will create the <FileSystem> sections of config file. + ''; + default = { orangefs = {}; }; + defaultText = literalExample "{ orangefs = {}; }"; + example = literalExample '' + { + fs1 = { + id = 101; + }; + + fs2 = { + id = 102; + }; + } + ''; + type = with types; attrsOf (submodule ({ ... } : { + options = { + id = mkOption { + type = types.int; + default = 1; + description = "File system ID (must be unique within configuration)."; + }; + + rootHandle = mkOption { + type = types.int; + default = 3; + description = "File system root ID."; + }; + + extraConfig = mkOption { + type = types.lines; + default = ""; + description = "Extra config for <FileSystem> section."; + }; + + troveSyncMeta = mkOption { + type = types.bool; + default = true; + description = "Sync meta data."; + }; + + troveSyncData = mkOption { + type = types.bool; + default = false; + description = "Sync data."; + }; + + extraStorageHints = mkOption { + type = types.lines; + default = ""; + description = "Extra config for <StorageHints> section."; + }; + }; + })); + }; + }; + }; + + ###### implementation + + config = mkIf cfg.enable { + environment.systemPackages = [ pkgs.orangefs ]; + + # orangefs daemon will run as user + users.users.orangefs.isSystemUser = true; + users.groups.orangefs = {}; + + # To format the file system the config file is needed. + environment.etc."orangefs/server.conf" = { + text = configFile; + user = "orangefs"; + group = "orangefs"; + }; + + systemd.services.orangefs-server = { + wantedBy = [ "multi-user.target" ]; + requires = [ "network-online.target" ]; + after = [ "network-online.target" ]; + + serviceConfig = { + # Run as "simple" in forground mode. + # This is more reliable + ExecStart = '' + ${pkgs.orangefs}/bin/pvfs2-server -d \ + /etc/orangefs/server.conf + ''; + TimeoutStopSec = "120"; + User = "orangefs"; + Group = "orangefs"; + }; + }; + }; + +} From 7ced973114e6a2a7d2d9cf94765b85cf10cba554 Mon Sep 17 00:00:00 2001 From: Markus Kowalewski Date: Wed, 21 Aug 2019 00:27:14 +0200 Subject: [PATCH 3/3] nixos/orangefs: add test --- nixos/tests/all-tests.nix | 1 + nixos/tests/orangefs.nix | 88 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 nixos/tests/orangefs.nix diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 5643da99e55..51af43e6c69 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -205,6 +205,7 @@ in # openstack-image-userdata doesn't work in a sandbox as the simulated openstack instance needs network access #openstack-image-userdata = (handleTestOn ["x86_64-linux"] ./openstack-image.nix {}).userdata or {}; openstack-image-metadata = (handleTestOn ["x86_64-linux"] ./openstack-image.nix {}).metadata or {}; + orangefs = handleTest ./orangefs.nix {}; os-prober = handleTestOn ["x86_64-linux"] ./os-prober.nix {}; osquery = handleTest ./osquery.nix {}; osrm-backend = handleTest ./osrm-backend.nix {}; diff --git a/nixos/tests/orangefs.nix b/nixos/tests/orangefs.nix new file mode 100644 index 00000000000..bdf4fc10c44 --- /dev/null +++ b/nixos/tests/orangefs.nix @@ -0,0 +1,88 @@ +import ./make-test.nix ({ ... } : + +let + server = { pkgs, ... } : { + networking.firewall.allowedTCPPorts = [ 3334 ]; + boot.initrd.postDeviceCommands = '' + ${pkgs.e2fsprogs}/bin/mkfs.ext4 -L data /dev/vdb + ''; + + virtualisation.emptyDiskImages = [ 4096 ]; + + fileSystems = pkgs.lib.mkVMOverride + [ { mountPoint = "/data"; + device = "/dev/disk/by-label/data"; + fsType = "ext4"; + } + ]; + + services.orangefs.server = { + enable = true; + dataStorageSpace = "/data/storage"; + metadataStorageSpace = "/data/meta"; + servers = { + server1 = "tcp://server1:3334"; + server2 = "tcp://server2:3334"; + }; + }; + }; + + client = { lib, ... } : { + networking.firewall.enable = true; + + services.orangefs.client = { + enable = true; + fileSystems = [{ + target = "tcp://server1:3334/orangefs"; + mountPoint = "/orangefs"; + }]; + }; + }; + +in { + name = "orangefs"; + + nodes = { + server1 = server; + server2 = server; + + client1 = client; + client2 = client; + }; + + testScript = '' + # format storage + foreach my $server (($server1,$server2)) + { + $server->start(); + $server->waitForUnit("multi-user.target"); + $server->succeed("mkdir -p /data/storage /data/meta"); + $server->succeed("chown orangefs:orangefs /data/storage /data/meta"); + $server->succeed("chmod 0770 /data/storage /data/meta"); + $server->succeed("sudo -g orangefs -u orangefs pvfs2-server -f /etc/orangefs/server.conf"); + } + + # start services after storage is formated on all machines + foreach my $server (($server1,$server2)) + { + $server->succeed("systemctl start orangefs-server.service"); + } + + # Check if clients can reach and mount the FS + foreach my $client (($client1,$client2)) + { + $client->start(); + $client->waitForUnit("orangefs-client.service"); + # Both servers need to be reachable + $client->succeed("pvfs2-check-server -h server1 -f orangefs -n tcp -p 3334"); + $client->succeed("pvfs2-check-server -h server2 -f orangefs -n tcp -p 3334"); + $client->waitForUnit("orangefs.mount"); + + } + + # R/W test between clients + $client1->succeed("echo test > /orangefs/file1"); + $client2->succeed("grep test /orangefs/file1"); + + ''; +})