Merge pull request #92791 from johanot/ceph-15

ceph: 14.2.10 -> 15.2.4

nixos/tests/ceph-multi-node.nix

@@ -183,15 +183,15 @@ let
     monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
 
     monA.succeed(
-        "ceph osd pool create multi-node-test 128 128",
+        "ceph osd pool create multi-node-test 32 32",
         "ceph osd pool ls | grep 'multi-node-test'",
         "ceph osd pool rename multi-node-test multi-node-other-test",
         "ceph osd pool ls | grep 'multi-node-other-test'",
     )
-    monA.wait_until_succeeds("ceph -s | grep '1 pools, 128 pgs'")
+    monA.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'")
     monA.succeed("ceph osd pool set multi-node-other-test size 2")
     monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
-    monA.wait_until_succeeds("ceph -s | grep '128 active+clean'")
+    monA.wait_until_succeeds("ceph -s | grep '33 active+clean'")
     monA.fail(
         "ceph osd pool ls | grep 'multi-node-test'",
         "ceph osd pool delete multi-node-other-test multi-node-other-test --yes-i-really-really-mean-it",

nixos/tests/ceph-single-node.nix

@@ -143,12 +143,12 @@ let
     monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
 
     monA.succeed(
-        "ceph osd pool create single-node-test 128 128",
+        "ceph osd pool create single-node-test 32 32",
         "ceph osd pool ls | grep 'single-node-test'",
         "ceph osd pool rename single-node-test single-node-other-test",
         "ceph osd pool ls | grep 'single-node-other-test'",
     )
-    monA.wait_until_succeeds("ceph -s | grep '1 pools, 128 pgs'")
+    monA.wait_until_succeeds("ceph -s | grep '2 pools, 33 pgs'")
     monA.succeed(
         "ceph osd getcrushmap -o crush",
         "crushtool -d crush -o decrushed",
@@ -158,7 +158,7 @@ let
         "ceph osd pool set single-node-other-test size 2",
     )
     monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
-    monA.wait_until_succeeds("ceph -s | grep '128 active+clean'")
+    monA.wait_until_succeeds("ceph -s | grep '33 active+clean'")
     monA.fail(
         "ceph osd pool ls | grep 'multi-node-test'",
         "ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it",

pkgs/tools/filesystems/ceph/0000-fix-SPDK-build-env.patch

@@ -1,11 +1,11 @@
---- a/cmake/modules/BuildSPDK.cmake	2018-08-09 09:22:34.950684960 +0200
+--- a/cmake/modules/BuildSPDK.cmake
-+++ b/cmake/modules/BuildSPDK.cmake	2018-08-09 09:21:59.986964224 +0200
++++ b/cmake/modules/BuildSPDK.cmake
-@@ -16,7 +16,7 @@
+@@ -35,7 +35,7 @@ macro(build_spdk)
      # unset $CFLAGS, otherwise it will interfere with how SPDK sets
      # its include directory.
      # unset $LDFLAGS, otherwise SPDK will fail to mock some functions.
--    BUILD_COMMAND env -i PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} $(MAKE) EXTRA_CFLAGS="-fPIC"
+-    BUILD_COMMAND env -i PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} ${make_cmd} EXTRA_CFLAGS="${spdk_CFLAGS}"
-+    BUILD_COMMAND env PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} $(MAKE) EXTRA_CFLAGS="-fPIC" C_OPT="-mssse3"
++    BUILD_COMMAND env -i PATH=$ENV{PATH} CC=${CMAKE_C_COMPILER} ${make_cmd} EXTRA_CFLAGS="${spdk_CFLAGS}" C_OPT="-mssse3"
      BUILD_IN_SOURCE 1
      INSTALL_COMMAND "true")
-   ExternalProject_Get_Property(spdk-ext source_dir)
+   unset(make_cmd)

pkgs/tools/filesystems/ceph/default.nix

@@ -75,6 +75,26 @@ let
     none = [ ];
   };
 
+  getMeta = description: {
+     homepage = "https://ceph.com/";
+     inherit description;
+     license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
+     maintainers = with maintainers; [ adev ak johanot krav ];
+     platforms = [ "x86_64-linux" ];
+   };
+
+  ceph-common = python3Packages.buildPythonPackage rec{
+    pname = "ceph-common";
+    inherit src version;
+
+    sourceRoot = "ceph-${version}/src/python-common";
+
+    checkInputs = [ python3Packages.pytest ];
+    propagatedBuildInputs = with python3Packages; [ pyyaml six ];
+
+    meta = getMeta "Ceph common module for code shared by manager modules";
+  };
+
   ceph-python-env = python3Packages.python.withPackages (ps: [
     ps.sphinx
     ps.flask
@@ -83,27 +103,33 @@ let
     ps.virtualenv
     # Libraries needed by the python tools
     ps.Mako
+    ceph-common
     ps.cherrypy
+    ps.dateutil
+    ps.jsonpatch
     ps.pecan
     ps.prettytable
     ps.pyjwt
     ps.webob
     ps.bcrypt
+    # scipy > 1.3 breaks diskprediction_local, leading to mgr hang on startup
+    # Bump (and get rid of scipy_1_3) once these issues are resolved:
+    # https://tracker.ceph.com/issues/42764 https://tracker.ceph.com/issues/45147
+    ps.scipy_1_3
     ps.six
     ps.pyyaml
   ]);
   sitePackages = ceph-python-env.python.sitePackages;
 
-  version = "14.2.10";
+  version = "15.2.4";
+  src = fetchurl {
+    url = "http://download.ceph.com/tarballs/ceph-${version}.tar.gz";
+    sha256 = "0jy5dp4r1bqk1l7nrv8l8zpl984k61p3vkvf73ygcn03bxyjjlax";
+  };
 in rec {
   ceph = stdenv.mkDerivation {
     pname = "ceph";
-    inherit version;
+    inherit src version;
-
-    src = fetchurl {
-      url = "http://download.ceph.com/tarballs/ceph-${version}.tar.gz";
-      sha256 = "0bbs3ag8zav283qpxrrndhvh2z01ykm6126fmwrbc1c5f9jfjq39";
-    };
 
     patches = [
       ./0000-fix-SPDK-build-env.patch
@@ -173,25 +199,13 @@ in rec {
 
     doCheck = false; # uses pip to install things from the internet
 
-    meta = {
+    meta = getMeta "Distributed storage system";
-      homepage = "https://ceph.com/";
-      description = "Distributed storage system";
-      license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
-      maintainers = with maintainers; [ adev ak krav johanot srhb ];
-      platforms = [ "x86_64-linux" ];
-    };
 
     passthru.version = version;
   };
 
   ceph-client = runCommand "ceph-client-${version}" {
-     meta = {
+      meta = getMeta "Tools needed to mount Ceph's RADOS Block Devices";
-        homepage = "https://ceph.com/";
-        description = "Tools needed to mount Ceph's RADOS Block Devices";
-        license = with licenses; [ lgpl21 gpl2 bsd3 mit publicDomain ];
-        maintainers = with maintainers; [ adev ak johanot krav ];
-        platforms = [ "x86_64-linux" ];
-      };
     } ''
       mkdir -p $out/{bin,etc,${sitePackages}}
       cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin

pkgs/top-level/python-packages.nix

@@ -5919,6 +5919,15 @@ in {
     });
   in if pythonOlder "3.5" then scipy_1_2 else scipy_;
 
+  scipy_1_3 = self.scipy.overridePythonAttrs(oldAttrs: rec {
+    version = "1.3.3";
+    src = oldAttrs.src.override {
+      inherit version;
+      sha256 = "02iqb7ws7fw5fd1a83hx705pzrw1imj7z0bphjsl4bfvw254xgv4";
+    };
+    doCheck = false;
+  });
+
   scikitimage = callPackage ../development/python-modules/scikit-image { };
 
   scikitlearn = let