ceph: 12.2.7 -> 13.2.4

* maintain only one version
* ceph-client: init
* include ceph-volume python tool in output

nixos/ceph: extraConfig, fix test, wait for ceph-mgr to become active

* run ceph with disk group permission
* add extraConfig option for the global section
needed per cluster
* clear up how ceph.conf is generated
* fix ceph testcase

nixos/modules/services/network-filesystems/ceph.nix

@@ -3,18 +3,18 @@
 with lib;
 
 let
-  ceph = pkgs.ceph;
   cfg  = config.services.ceph;
+
   # function that translates "camelCaseOptions" to "camel case options", credits to tilpner in #nixos@freenode
-  translateOption = replaceStrings upperChars (map (s: " ${s}") lowerChars);
+  expandCamelCase = replaceStrings upperChars (map (s: " ${s}") lowerChars);
-  generateDaemonList = (daemonType: daemons: extraServiceConfig:
+  expandCamelCaseAttrs = mapAttrs' (name: value: nameValuePair (expandCamelCase name) value);
-    mkMerge (
+
-      map (daemon:
+  makeServices = (daemonType: daemonIds: extraServiceConfig:
-        { "ceph-${daemonType}-${daemon}" = generateServiceFile daemonType daemon cfg.global.clusterName ceph extraServiceConfig; }
+    mkMerge (map (daemonId:
-      ) daemons
+      { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph extraServiceConfig; })
-    )
+      daemonIds));
-  );
+
-  generateServiceFile = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: {
+  makeService = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: {
     enable = true;
     description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}";
     after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target";
@@ -34,23 +34,29 @@ let
       Restart = "on-failure";
       StartLimitBurst = "5";
       StartLimitInterval = "30min";
-      ExecStart = "${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} -f --cluster ${clusterName} --id ${if daemonType == "rgw" then "client.${daemonId}" else daemonId} --setuser ceph --setgroup ceph";
+      ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \
+                    -f --cluster ${clusterName} --id ${daemonId} --setuser ceph \
+                    --setgroup ${if daemonType == "osd" then "disk" else "ceph"}'';
     } // extraServiceConfig
-      // optionalAttrs (daemonType == "osd") { ExecStartPre = "${ceph.out}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}"; };
+      // optionalAttrs (daemonType == "osd") { ExecStartPre = ''${ceph.out}/libexec/ceph/ceph-osd-prestart.sh \
-    } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) { preStart = ''
+                                                              --id ${daemonId} --cluster ${clusterName}''; };
+    } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) {
+      preStart = ''
         daemonPath="/var/lib/ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"
-        if [ ! -d ''$daemonPath ]; then
+        if [ ! -d $daemonPath ]; then
-          mkdir -m 755 -p ''$daemonPath
+          mkdir -m 755 -p $daemonPath
-          chown -R ceph:ceph ''$daemonPath
+          chown -R ceph:ceph $daemonPath
         fi
       '';
     } // optionalAttrs (daemonType == "osd") { path = [ pkgs.getopt ]; }
   );
-  generateTargetFile = (daemonType:
+
+  makeTarget = (daemonType:
     {
       "ceph-${daemonType}" = {
         description = "Ceph target allowing to start/stop all ceph-${daemonType} services at once";
         partOf = [ "ceph.target" ];
+        wantedBy = [ "ceph.target" ];
         before = [ "ceph.target" ];
       };
     }
@@ -157,6 +163,27 @@ in
           A comma-separated list of subnets that will be used as cluster networks in the cluster.
         '';
       };
+
+      rgwMimeTypesFile = mkOption {
+        type = with types; nullOr path;
+        default = "${pkgs.mime-types}/etc/mime.types";
+        description = ''
+          Path to mime types used by radosgw.
+        '';
+      };
+    };
+
+    extraConfig = mkOption {
+      type = with types; attrsOf str;
+      default = {};
+      example = ''
+        {
+          "ms bind ipv6" = "true";
+        };
+      '';
+      description = ''
+        Extra configuration to add to the global section. Use for setting values that are common for all daemons in the cluster.
+      '';
     };
 
     mgr = {
@@ -216,6 +243,7 @@ in
           to the id part in ceph i.e. [ "name1" ] would result in osd.name1
         '';
       };
+
       extraConfig = mkOption {
         type = with types; attrsOf str;
         default = {
@@ -296,9 +324,6 @@ in
       { assertion = cfg.global.fsid != "";
         message = "fsid has to be set to a valid uuid for the cluster to function";
       }
-      { assertion = cfg.mgr.enable == true;
-        message = "ceph 12.x requires atleast 1 MGR daemon enabled for the cluster to function";
-      }
       { assertion = cfg.mon.enable == true -> cfg.mon.daemons != [];
         message = "have to set id of atleast one MON if you're going to enable Monitor";
       }
@@ -317,14 +342,12 @@ in
       ''Not setting up a list of members in monInitialMembers requires that you set the host variable for each mon daemon or else the cluster won't function'';
 
     environment.etc."ceph/ceph.conf".text = let
-      # Translate camelCaseOptions to the expected camel case option for ceph.conf
-      translatedGlobalConfig = mapAttrs' (name: value: nameValuePair (translateOption name) value) cfg.global;
       # Merge the extraConfig set for mgr daemons, as mgr don't have their own section
-      globalAndMgrConfig = translatedGlobalConfig // optionalAttrs cfg.mgr.enable cfg.mgr.extraConfig;
+      globalSection = expandCamelCaseAttrs (cfg.global // cfg.extraConfig // optionalAttrs cfg.mgr.enable cfg.mgr.extraConfig);
       # Remove all name-value pairs with null values from the attribute set to avoid making empty sections in the ceph.conf
-      globalConfig = mapAttrs' (name: value: nameValuePair (translateOption name) value) (filterAttrs (name: value: value != null) globalAndMgrConfig);
+      globalSection' = filterAttrs (name: value: value != null) globalSection;
       totalConfig = {
-          "global" = globalConfig;
+          "global" = globalSection';
         } // optionalAttrs (cfg.mon.enable && cfg.mon.extraConfig != {}) { "mon" = cfg.mon.extraConfig; }
           // optionalAttrs (cfg.mds.enable && cfg.mds.extraConfig != {}) { "mds" = cfg.mds.extraConfig; }
           // optionalAttrs (cfg.osd.enable && cfg.osd.extraConfig != {}) { "osd" = cfg.osd.extraConfig; }
@@ -336,8 +359,9 @@ in
       name = "ceph";
       uid = config.ids.uids.ceph;
       description = "Ceph daemon user";
+      group = "ceph";
+      extraGroups = [ "disk" ];
     };
-
     users.groups = singleton {
       name = "ceph";
       gid = config.ids.gids.ceph;
@@ -345,22 +369,26 @@ in
 
     systemd.services = let
       services = []
-        ++ optional cfg.mon.enable (generateDaemonList "mon" cfg.mon.daemons { RestartSec = "10"; })
+        ++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons { RestartSec = "10"; })
-        ++ optional cfg.mds.enable (generateDaemonList "mds" cfg.mds.daemons { StartLimitBurst = "3"; })
+        ++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons { StartLimitBurst = "3"; })
-        ++ optional cfg.osd.enable (generateDaemonList "osd" cfg.osd.daemons { StartLimitBurst = "30"; RestartSec = "20s"; })
+        ++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons { StartLimitBurst = "30";
-        ++ optional cfg.rgw.enable (generateDaemonList "rgw" cfg.rgw.daemons { })
+                                                                         RestartSec = "20s";
-        ++ optional cfg.mgr.enable (generateDaemonList "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; });
+                                                                         PrivateDevices = "no"; # osd needs disk access
+                                                                       })
+        ++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons { })
+        ++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; });
       in
         mkMerge services;
 
     systemd.targets = let
       targets = [
-        { "ceph" = { description = "Ceph target allowing to start/stop all ceph service instances at once"; }; }
+        { "ceph" = { description = "Ceph target allowing to start/stop all ceph service instances at once";
-      ] ++ optional cfg.mon.enable (generateTargetFile "mon")
+                     wantedBy = [ "multi-user.target" ]; }; }
-        ++ optional cfg.mds.enable (generateTargetFile "mds")
+      ] ++ optional cfg.mon.enable (makeTarget "mon")
-        ++ optional cfg.osd.enable (generateTargetFile "osd")
+        ++ optional cfg.mds.enable (makeTarget "mds")
-        ++ optional cfg.rgw.enable (generateTargetFile "rgw")
+        ++ optional cfg.osd.enable (makeTarget "osd")
-        ++ optional cfg.mgr.enable (generateTargetFile "mgr");
+        ++ optional cfg.rgw.enable (makeTarget "rgw")
+        ++ optional cfg.mgr.enable (makeTarget "mgr");
       in
         mkMerge targets;
 

nixos/tests/ceph.nix

@@ -1,4 +1,4 @@
-import ./make-test.nix ({pkgs, ...}: rec {
+import ./make-test.nix ({pkgs, lib, ...}: rec {
   name = "All-in-one-basic-ceph-cluster";
   meta = with pkgs.stdenv.lib.maintainers; {
     maintainers = [ lejonet ];
@@ -40,6 +40,9 @@ import ./make-test.nix ({pkgs, ...}: rec {
       services.ceph.mon = {
         enable = true;
         daemons = [ "aio" ];
+        extraConfig = {
+          "mgr initial modules" = "dashboard";
+        };
       };
 
       services.ceph.mgr = {
@@ -51,6 +54,9 @@ import ./make-test.nix ({pkgs, ...}: rec {
         enable = true;
         daemons = [ "0" "1" ];
       };
+
+      # So that we don't have to battle systemd when bootstraping
+      systemd.targets.ceph.wantedBy = lib.mkForce [];
     };
   };
 
@@ -69,7 +75,6 @@ import ./make-test.nix ({pkgs, ...}: rec {
 
     # Bootstrap ceph-mon daemon
     $aio->mustSucceed(
-      "mkdir -p /var/lib/ceph/bootstrap-osd && chown ceph:ceph /var/lib/ceph/bootstrap-osd",
       "sudo -u ceph ceph-authtool --create-keyring /tmp/ceph.mon.keyring --gen-key -n mon. --cap mon 'allow *'",
       "ceph-authtool --create-keyring /etc/ceph/ceph.client.admin.keyring --gen-key -n client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *'",
       "ceph-authtool /tmp/ceph.mon.keyring --import-keyring /etc/ceph/ceph.client.admin.keyring",
@@ -90,6 +95,7 @@ import ./make-test.nix ({pkgs, ...}: rec {
     );
     $aio->waitForUnit("ceph-mgr-aio");
     $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'");
+    $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'");
 
     # Bootstrap both OSDs
     $aio->mustSucceed(
@@ -135,5 +141,23 @@ import ./make-test.nix ({pkgs, ...}: rec {
       "ceph osd pool ls | grep 'aio-test'",
       "ceph osd pool delete aio-other-test aio-other-test --yes-i-really-really-mean-it"
     );
+
+    # As we disable the target in the config, we still want to test that it works as intended
+    $aio->mustSucceed(
+      "systemctl stop ceph-osd-0",
+      "systemctl stop ceph-osd-1",
+      "systemctl stop ceph-mgr-aio",
+      "systemctl stop ceph-mon-aio"
+    );
+    $aio->succeed("systemctl start ceph.target");
+    $aio->waitForUnit("ceph-mon-aio");
+    $aio->waitForUnit("ceph-mgr-aio");
+    $aio->waitForUnit("ceph-osd-0");
+    $aio->waitForUnit("ceph-osd-1");
+    $aio->succeed("ceph -s | grep 'mon: 1 daemons'");
+    $aio->waitUntilSucceeds("ceph -s | grep 'quorum aio'");
+    $aio->waitUntilSucceeds("ceph osd stat | grep '2 osds: 2 up, 2 in'");
+    $aio->waitUntilSucceeds("ceph -s | grep 'mgr: aio(active)'");
+    $aio->waitUntilSucceeds("ceph -s | grep 'HEALTH_OK'");
   '';
 })

pkgs/tools/filesystems/ceph/0000-ceph-volume-allow-loop.patch

@@ -0,0 +1,35 @@
+diff --git a/src/ceph-disk/ceph_disk/main.py b/src/ceph-disk/ceph_disk/main.py
+index f8ef35503e..2cc8a75d2b 100644
+--- a/src/ceph-disk/ceph_disk/main.py
++++ b/src/ceph-disk/ceph_disk/main.py
+@@ -881,13 +881,14 @@ def is_partition(dev):
+     if not stmode_is_diskdevice(st.st_mode):
+         raise Error('not a block device', dev)
+ 
++    major = os.major(st.st_rdev)
++    minor = os.minor(st.st_rdev)
++
+     name = get_dev_name(dev)
+-    if os.path.exists(os.path.join(BLOCKDIR, name)):
++    if os.path.exists(os.path.join(BLOCKDIR, name)) or os.path.exists(os.path.join(BLOCKDIR, "loop%d" % minor)):
+         return False
+ 
+     # make sure it is a partition of something else
+-    major = os.major(st.st_rdev)
+-    minor = os.minor(st.st_rdev)
+     if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)):
+         return True
+ 
+diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py
+index c3a7915400..9a708b05f2 100644
+--- a/src/ceph-volume/ceph_volume/util/disk.py
++++ b/src/ceph-volume/ceph_volume/util/disk.py
+@@ -203,7 +203,7 @@ def is_device(dev):
+     # use lsblk first, fall back to using stat
+     TYPE = lsblk(dev).get('TYPE')
+     if TYPE:
+-        return TYPE == 'disk'
++        return TYPE == 'disk' or TYPE == 'loop'
+ 
+     # fallback to stat
+     return _stat_is_device(os.lstat(dev).st_mode)

pkgs/tools/filesystems/ceph/0000-dont-check-cherrypy-version.patch

@@ -0,0 +1,33 @@
+diff --git a/src/pybind/mgr/dashboard/module.py b/src/pybind/mgr/dashboard/module.py
+index 0b53743d74..2e17ce1168 100644
+--- a/src/pybind/mgr/dashboard/module.py
++++ b/src/pybind/mgr/dashboard/module.py
+@@ -28,28 +28,6 @@ except ImportError:
+     # To be picked up and reported by .can_run()
+     cherrypy = None
+ 
+-
+-# The SSL code in CherryPy 3.5.0 is buggy.  It was fixed long ago,
+-# but 3.5.0 is still shipping in major linux distributions
+-# (Fedora 27, Ubuntu Xenial), so we must monkey patch it to get SSL working.
+-if cherrypy is not None:
+-    v = StrictVersion(cherrypy.__version__)
+-    # It was fixed in 3.7.0.  Exact lower bound version is probably earlier,
+-    # but 3.5.0 is what this monkey patch is tested on.
+-    if v >= StrictVersion("3.5.0") and v < StrictVersion("3.7.0"):
+-        from cherrypy.wsgiserver.wsgiserver2 import HTTPConnection,\
+-                                                    CP_fileobject
+-
+-        def fixed_init(hc_self, server, sock, makefile=CP_fileobject):
+-            hc_self.server = server
+-            hc_self.socket = sock
+-            hc_self.rfile = makefile(sock, "rb", hc_self.rbufsize)
+-            hc_self.wfile = makefile(sock, "wb", hc_self.wbufsize)
+-            hc_self.requests_seen = 0
+-
+-        HTTPConnection.__init__ = fixed_init
+-
+-
+ if 'COVERAGE_ENABLED' in os.environ:
+     import coverage
+     _cov = coverage.Coverage(config_file="{}/.coveragerc".format(os.path.dirname(__file__)))

pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch

@@ -0,0 +1,11 @@
+--- a/cmake/modules/BuildSPDK.cmake	2018-08-09 09:22:34.950684960 +0200
++++ b/cmake/modules/BuildSPDK.cmake	2018-08-09 09:21:59.986964224 +0200
+@@ -16,7 +16,7 @@
+     # unset $CFLAGS, otherwise it will interfere with how SPDK sets
+     # its include directory.
+     # unset $LDFLAGS, otherwise SPDK will fail to mock some functions.
+-    BUILD_COMMAND env -i PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} $(MAKE) EXTRA_CFLAGS="-fPIC"
++    BUILD_COMMAND env PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} $(MAKE) EXTRA_CFLAGS="-fPIC" C_OPT="-mssse3"
+     BUILD_IN_SOURCE 1
+     INSTALL_COMMAND "true")
+   ExternalProject_Get_Property(spdk-ext source_dir)

pkgs/tools/filesystems/ceph/0001-kv-RocksDBStore-API-break-additional.patch

@@ -1,58 +0,0 @@
-Seulement dans ceph: ceph.old
-diff -ur ceph.old/src/kv/RocksDBStore.cc ceph/src/kv/RocksDBStore.cc
---- ceph.old/src/kv/RocksDBStore.cc	1980-01-02 00:00:00.000000000 +0100
-+++ ceph/src/kv/RocksDBStore.cc	2018-01-24 14:08:35.017553372 +0100
-@@ -505,7 +505,7 @@
-   // considering performance overhead, default is disabled
-   if (g_conf->rocksdb_perf) {
-     rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex);
--    rocksdb::perf_context.Reset();
-+    rocksdb::get_perf_context()->Reset();
-   }
- 
-   RocksDBTransactionImpl * _t =
-@@ -532,13 +532,13 @@
-     utime_t write_wal_time;
-     utime_t write_pre_and_post_process_time;
-     write_wal_time.set_from_double(
--	static_cast<double>(rocksdb::perf_context.write_wal_time)/1000000000);
-+	static_cast<double>(rocksdb::get_perf_context()->write_wal_time)/1000000000);
-     write_memtable_time.set_from_double(
--	static_cast<double>(rocksdb::perf_context.write_memtable_time)/1000000000);
-+	static_cast<double>(rocksdb::get_perf_context()->write_memtable_time)/1000000000);
-     write_delay_time.set_from_double(
--	static_cast<double>(rocksdb::perf_context.write_delay_time)/1000000000);
-+	static_cast<double>(rocksdb::get_perf_context()->write_delay_time)/1000000000);
-     write_pre_and_post_process_time.set_from_double(
--	static_cast<double>(rocksdb::perf_context.write_pre_and_post_process_time)/1000000000);
-+	static_cast<double>(rocksdb::get_perf_context()->write_pre_and_post_process_time)/1000000000);
-     logger->tinc(l_rocksdb_write_memtable_time, write_memtable_time);
-     logger->tinc(l_rocksdb_write_delay_time, write_delay_time);
-     logger->tinc(l_rocksdb_write_wal_time, write_wal_time);
-@@ -558,7 +558,7 @@
-   // considering performance overhead, default is disabled
-   if (g_conf->rocksdb_perf) {
-     rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex);
--    rocksdb::perf_context.Reset();
-+    rocksdb::get_perf_context()->Reset();
-   }
- 
-   RocksDBTransactionImpl * _t =
-@@ -586,13 +586,13 @@
-     utime_t write_wal_time;
-     utime_t write_pre_and_post_process_time;
-     write_wal_time.set_from_double(
--	static_cast<double>(rocksdb::perf_context.write_wal_time)/1000000000);
-+	static_cast<double>(rocksdb::get_perf_context()->write_wal_time)/1000000000);
-     write_memtable_time.set_from_double(
--	static_cast<double>(rocksdb::perf_context.write_memtable_time)/1000000000);
-+	static_cast<double>(rocksdb::get_perf_context()->write_memtable_time)/1000000000);
-     write_delay_time.set_from_double(
--	static_cast<double>(rocksdb::perf_context.write_delay_time)/1000000000);
-+	static_cast<double>(rocksdb::get_perf_context()->write_delay_time)/1000000000);
-     write_pre_and_post_process_time.set_from_double(
--	static_cast<double>(rocksdb::perf_context.write_pre_and_post_process_time)/1000000000);
-+	static_cast<double>(rocksdb::get_perf_context()->write_pre_and_post_process_time)/1000000000);
-     logger->tinc(l_rocksdb_write_memtable_time, write_memtable_time);
-     logger->tinc(l_rocksdb_write_delay_time, write_delay_time);
-     logger->tinc(l_rocksdb_write_wal_time, write_wal_time);

pkgs/tools/filesystems/ceph/0002-fix-absolute-include-path.patch

@@ -1,19 +0,0 @@
-diff -ru ceph/src/key_value_store/kv_flat_btree_async.cc ceph-copy/src/key_value_store/kv_flat_btree_async.cc
---- ceph/src/key_value_store/kv_flat_btree_async.cc	1980-01-02 00:00:00.000000000 +0100
-+++ ceph-copy/src/key_value_store/kv_flat_btree_async.cc	2018-02-13 21:49:59.232860487 +0100
-@@ -15,13 +15,13 @@
- #include "key_value_store/kv_flat_btree_async.h"
- #include "key_value_store/kvs_arg_types.h"
- #include "include/rados/librados.hpp"
--#include "/usr/include/asm-generic/errno.h"
--#include "/usr/include/asm-generic/errno-base.h"
- #include "common/ceph_context.h"
- #include "common/Clock.h"
- #include "include/types.h"
- 
- 
-+#include <asm-generic/errno.h>
-+#include <asm-generic/errno-base.h>
- #include <string>
- #include <iostream>
- #include <cassert>

pkgs/tools/filesystems/ceph/default.nix

@@ -1,12 +1,189 @@
-{ callPackage, fetchgit, fetchpatch, ... } @ args:
+{ stdenv, runCommand, fetchurl
+, ensureNewerSourcesHook
+, cmake, pkgconfig
+, which, git
+, boost, python2Packages
+, libxml2, zlib, lz4
+, openldap, lttng-ust
+, babeltrace, gperf
+, cunit, snappy
+, rocksdb, makeWrapper
+, leveldb, oathToolkit
 
-callPackage ./generic.nix (args // rec {
+# Optional Dependencies
-  version = "12.2.7";
+, yasm ? null, fcgi ? null, expat ? null
+, curl ? null, fuse ? null
+, libedit ? null, libatomic_ops ? null, kinetic-cpp-client ? null
+, libs3 ? null
 
-  src = fetchgit {
+# Mallocs
-    url = "https://github.com/ceph/ceph.git";
+, jemalloc ? null, gperftools ? null
-    rev = "refs/tags/v${version}";
+
-    sha256 = "031nfw2g2fdpxxx39g862phgmdx68hj9r54axazandghfhc1bzrl";
+# Crypto Dependencies
+, cryptopp ? null
+, nss ? null, nspr ? null
+
+# Linux Only Dependencies
+, linuxHeaders, utillinux, libuuid, udev, keyutils
+, libaio ? null, libxfs ? null, zfs ? null
+, ...
+}:
+
+# We must have one crypto library
+assert cryptopp != null || (nss != null && nspr != null);
+
+with stdenv; with stdenv.lib;
+let
+  shouldUsePkg = pkg: if pkg != null && pkg.meta.available then pkg else null;
+
+  optYasm = shouldUsePkg yasm;
+  optFcgi = shouldUsePkg fcgi;
+  optExpat = shouldUsePkg expat;
+  optCurl = shouldUsePkg curl;
+  optFuse = shouldUsePkg fuse;
+  optLibedit = shouldUsePkg libedit;
+  optLibatomic_ops = shouldUsePkg libatomic_ops;
+  optKinetic-cpp-client = shouldUsePkg kinetic-cpp-client;
+  optLibs3 = shouldUsePkg libs3;
+
+  optJemalloc = shouldUsePkg jemalloc;
+  optGperftools = shouldUsePkg gperftools;
+
+  optCryptopp = shouldUsePkg cryptopp;
+  optNss = shouldUsePkg nss;
+  optNspr = shouldUsePkg nspr;
+
+  optLibaio = shouldUsePkg libaio;
+  optLibxfs = shouldUsePkg libxfs;
+  optZfs = shouldUsePkg zfs;
+
+  hasRadosgw = optFcgi != null && optExpat != null && optCurl != null && optLibedit != null;
+
+
+  # Malloc implementation (can be jemalloc, tcmalloc or null)
+  malloc = if optJemalloc != null then optJemalloc else optGperftools;
+
+  # We prefer nss over cryptopp
+  cryptoStr = if optNss != null && optNspr != null then "nss" else
+    if optCryptopp != null then "cryptopp" else "none";
+
+  cryptoLibsMap = {
+    nss = [ optNss optNspr ];
+    cryptopp = [ optCryptopp ];
+    none = [ ];
   };
 
-})
+  ceph-python-env = python2Packages.python.withPackages (ps: [
+    ps.sphinx
+    ps.flask
+    ps.cython
+    ps.setuptools
+    ps.virtualenv
+    # Libraries needed by the python tools
+    ps.Mako
+    ps.cherrypy
+    ps.pecan
+    ps.prettytable
+    ps.webob
+    ps.bcrypt
+  ]);
+
+  version = "13.2.4";
+in rec {
+  ceph = stdenv.mkDerivation {
+    name="ceph-${version}";
+
+    src = fetchurl {
+      url = "http://download.ceph.com/tarballs/ceph-${version}.tar.gz";
+      sha256 = "0g2mc6rp84ia44vz8kl449820m9hmfavzfmwn8fy6li14xr8a00w";
+    };
+
+    patches = [
+      ./0000-fix-SPDK-build-env.patch
+      # TODO: remove when https://github.com/ceph/ceph/pull/21289 is merged
+      ./0000-ceph-volume-allow-loop.patch
+      ./0000-dont-check-cherrypy-version.patch
+    ];
+
+    nativeBuildInputs = [
+      cmake
+      pkgconfig which git python2Packages.wrapPython makeWrapper
+      (ensureNewerSourcesHook { year = "1980"; })
+    ];
+
+    buildInputs = cryptoLibsMap.${cryptoStr} ++ [
+      boost ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3
+      malloc zlib openldap lttng-ust babeltrace gperf cunit
+      snappy rocksdb lz4 oathToolkit leveldb
+      optKinetic-cpp-client
+    ] ++ optionals stdenv.isLinux [
+      linuxHeaders utillinux libuuid udev keyutils optLibaio optLibxfs optZfs
+    ] ++ optionals hasRadosgw [
+      optFcgi optExpat optCurl optFuse optLibedit
+    ];
+
+    preConfigure =''
+      substituteInPlace src/common/module.c --replace "/sbin/modinfo"  "modinfo"
+      substituteInPlace src/common/module.c --replace "/sbin/modprobe" "modprobe"
+      # Since Boost 1.67 this seems to have changed
+      substituteInPlace CMakeLists.txt --replace "list(APPEND BOOST_COMPONENTS python)" "list(APPEND BOOST_COMPONENTS python27)"
+      substituteInPlace src/CMakeLists.txt --replace "Boost::python " "Boost::python27 "
+
+      # for pybind/rgw to find internal dep
+      export LD_LIBRARY_PATH="$PWD/build/lib:$LD_LIBRARY_PATH"
+      # install target needs to be in PYTHONPATH for "*.pth support" check to succeed
+      export PYTHONPATH=$lib/lib/python2.7/site-packages/:$out/lib/python2.7/site-packages/
+
+      patchShebangs src/spdk
+    '';
+
+    cmakeFlags = [
+      "-DWITH_SYSTEM_ROCKSDB=ON"
+      "-DROCKSDB_INCLUDE_DIR=${rocksdb}/include/rocksdb"
+      "-DWITH_SYSTEM_BOOST=ON"
+      "-DWITH_SYSTEMD=OFF"
+      "-DWITH_TESTS=OFF"
+      # TODO breaks with sandbox, tries to download stuff with npm
+      "-DWITH_MGR_DASHBOARD_FRONTEND=OFF"
+    ];
+
+    postFixup = ''
+      wrapPythonPrograms
+      wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$lib/lib/ceph/mgr:$out/lib/python2.7/site-packages/"
+    '';
+
+    enableParallelBuilding = true;
+
+    outputs = [ "out" "lib" "dev" "doc" "man" ];
+
+    meta = {
+      homepage = https://ceph.com/;
+      description = "Distributed storage system";
+      license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
+      maintainers = with maintainers; [ adev ak krav ];
+      platforms = platforms.unix;
+    };
+
+    passthru.version = version;
+  };
+
+  ceph-client = runCommand "ceph-client-${version}" {
+     meta = {
+        homepage = https://ceph.com/;
+        description = "Tools needed to mount Ceph's RADOS Block Devices";
+        license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
+        maintainers = with maintainers; [ adev ak krav ];
+        platforms = platforms.unix;
+      };
+    } ''
+      mkdir -p $out/{bin,etc,lib/python2.7/site-packages}
+      cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin
+      cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin
+      cp -r ${ceph}/bin/rbd-replay* $out/bin
+      cp -r ${ceph}/lib/python2.7/site-packages $out/lib/python2.7/
+      cp -r ${ceph}/etc/bash_completion.d $out/etc
+      # wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths
+      substituteInPlace $out/bin/ceph          --replace ${ceph} $out
+      substituteInPlace $out/bin/.ceph-wrapped --replace ${ceph} $out
+   '';
+}

pkgs/tools/filesystems/ceph/generic.nix

@@ -1,175 +0,0 @@
-{ stdenv, ensureNewerSourcesHook, cmake, pkgconfig
-, which, git
-, boost, python2Packages
-, libxml2, zlib
-, openldap, lttng-ust
-, babeltrace, gperf
-, cunit, snappy
-, rocksdb, makeWrapper
-
-# Optional Dependencies
-, yasm ? null, fcgi ? null, expat ? null
-, curl ? null, fuse ? null
-, libedit ? null, libatomic_ops ? null, kinetic-cpp-client ? null
-, libs3 ? null
-
-# Mallocs
-, jemalloc ? null, gperftools ? null
-
-# Crypto Dependencies
-, cryptopp ? null
-, nss ? null, nspr ? null
-
-# Linux Only Dependencies
-, linuxHeaders, libuuid, udev, keyutils, libaio ? null, libxfs ? null
-, zfs ? null
-
-# Version specific arguments
-, version, src ? [], buildInputs ? []
-, ...
-}:
-
-# We must have one crypto library
-assert cryptopp != null || (nss != null && nspr != null);
-
-with stdenv;
-with stdenv.lib;
-let
-
-  shouldUsePkg = pkg_: let pkg = (builtins.tryEval pkg_).value;
-    in if lib.any (lib.meta.platformMatch stdenv.hostPlatform) pkg.meta.platforms
-      then pkg else null;
-
-  optYasm = shouldUsePkg yasm;
-  optFcgi = shouldUsePkg fcgi;
-  optExpat = shouldUsePkg expat;
-  optCurl = shouldUsePkg curl;
-  optFuse = shouldUsePkg fuse;
-  optLibedit = shouldUsePkg libedit;
-  optLibatomic_ops = shouldUsePkg libatomic_ops;
-  optKinetic-cpp-client = shouldUsePkg kinetic-cpp-client;
-  optLibs3 = if versionAtLeast version "10.0.0" then null else shouldUsePkg libs3;
-
-  optJemalloc = shouldUsePkg jemalloc;
-  optGperftools = shouldUsePkg gperftools;
-
-  optCryptopp = shouldUsePkg cryptopp;
-  optNss = shouldUsePkg nss;
-  optNspr = shouldUsePkg nspr;
-
-  optLibaio = shouldUsePkg libaio;
-  optLibxfs = shouldUsePkg libxfs;
-  optZfs = shouldUsePkg zfs;
-
-  hasRadosgw = optFcgi != null && optExpat != null && optCurl != null && optLibedit != null;
-
-
-  # TODO: Reenable when kinetic support is fixed
-  #hasKinetic = versionAtLeast version "9.0.0" && optKinetic-cpp-client != null;
-  hasKinetic = false;
-
-  # Malloc implementation (can be jemalloc, tcmalloc or null)
-  malloc = if optJemalloc != null then optJemalloc else optGperftools;
-
-  # We prefer nss over cryptopp
-  cryptoStr = if optNss != null && optNspr != null then "nss" else
-    if optCryptopp != null then "cryptopp" else "none";
-  cryptoLibsMap = {
-    nss = [ optNss optNspr ];
-    cryptopp = [ optCryptopp ];
-    none = [ ];
-  };
-
-  ceph-python-env = python2Packages.python.withPackages (ps: [
-    ps.sphinx
-    ps.flask
-    ps.cython
-    ps.setuptools
-    ps.pip
-    # Libraries needed by the python tools
-    ps.Mako
-    ps.pecan
-    ps.prettytable
-    ps.webob
-    ps.cherrypy
-  ]);
-
-in
-stdenv.mkDerivation {
-  pname = "ceph";
-  inherit version;
-
-  inherit src;
-
-  patches = [
- #   ./ceph-patch-cmake-path.patch
-    ./0001-kv-RocksDBStore-API-break-additional.patch
-  ] ++ optionals stdenv.isLinux [
-    ./0002-fix-absolute-include-path.patch
-  ];
-
-  nativeBuildInputs = [
-    cmake
-    pkgconfig which git python2Packages.wrapPython makeWrapper
-    (ensureNewerSourcesHook { year = "1980"; })
-  ];
-
-  buildInputs = buildInputs ++ cryptoLibsMap.${cryptoStr} ++ [
-    boost ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3
-    malloc zlib openldap lttng-ust babeltrace gperf cunit
-    snappy rocksdb
-  ] ++ optionals stdenv.isLinux [
-    linuxHeaders libuuid udev keyutils optLibaio optLibxfs optZfs
-  ] ++ optionals hasRadosgw [
-    optFcgi optExpat optCurl optFuse optLibedit
-  ] ++ optionals hasKinetic [
-    optKinetic-cpp-client
-  ];
-
-
-  preConfigure =''
-    # rip off submodule that interfer with system libs
-	rm -rf src/boost
-	rm -rf src/rocksdb
-
-	# require LD_LIBRARY_PATH for cython to find internal dep
-	export LD_LIBRARY_PATH="$PWD/build/lib:$LD_LIBRARY_PATH"
-
-	# requires setuptools due to embedded in-cmake setup.py usage
-	export PYTHONPATH="${python2Packages.setuptools}/lib/python2.7/site-packages/:$PYTHONPATH"
-  '';
-
-  cmakeFlags = [
-    "-DENABLE_GIT_VERSION=OFF"
-    "-DWITH_SYSTEM_BOOST=ON"
-    "-DWITH_SYSTEM_ROCKSDB=ON"
-    "-DWITH_LEVELDB=OFF"
-
-    # enforce shared lib
-    "-DBUILD_SHARED_LIBS=ON"
-
-    # disable cephfs, cmake build broken for now
-    "-DWITH_CEPHFS=OFF"
-    "-DWITH_LIBCEPHFS=OFF"
-  ];
-
-  postFixup = ''
-    wrapPythonPrograms
-    wrapProgram $out/bin/ceph-mgr --set PYTHONPATH $out/${python2Packages.python.sitePackages}
-  '';
-
-  enableParallelBuilding = true;
-
-  outputs = [ "dev" "lib" "out" "doc" ];
-
-  meta = {
-    homepage = https://ceph.com/;
-    description = "Distributed storage system";
-    license = licenses.lgpl21;
-    maintainers = with maintainers; [ adev ak ];
-    platforms = platforms.unix;
-    broken = true;
-  };
-
-  passthru.version = version;
-}

pkgs/top-level/all-packages.nix

@@ -2193,9 +2193,11 @@ in
   nrg2iso = callPackage ../tools/cd-dvd/nrg2iso { };
 
   libceph = ceph.lib;
-  ceph = callPackage ../tools/filesystems/ceph {
+  inherit (callPackages ../tools/filesystems/ceph {
-    boost = boost166.override { enablePython = true; };
+    boost = boost167.override { enablePython = true; };
-  };
+  })
+    ceph
+    ceph-client;
   ceph-dev = ceph;
 
   inherit (callPackages ../tools/security/certmgr { })