Merge pull request #33894 from adevress/master

ceph: 9.2.0 -> 12.2.2

pkgs/tools/filesystems/ceph/0001-kv-RocksDBStore-API-break-additional.patch

@@ -0,0 +1,58 @@
+Seulement dans ceph: ceph.old
+diff -ur ceph.old/src/kv/RocksDBStore.cc ceph/src/kv/RocksDBStore.cc
+--- ceph.old/src/kv/RocksDBStore.cc	1980-01-02 00:00:00.000000000 +0100
++++ ceph/src/kv/RocksDBStore.cc	2018-01-24 14:08:35.017553372 +0100
+@@ -505,7 +505,7 @@
+   // considering performance overhead, default is disabled
+   if (g_conf->rocksdb_perf) {
+     rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex);
+-    rocksdb::perf_context.Reset();
++    rocksdb::get_perf_context()->Reset();
+   }
+ 
+   RocksDBTransactionImpl * _t =
+@@ -532,13 +532,13 @@
+     utime_t write_wal_time;
+     utime_t write_pre_and_post_process_time;
+     write_wal_time.set_from_double(
+-	static_cast<double>(rocksdb::perf_context.write_wal_time)/1000000000);
++	static_cast<double>(rocksdb::get_perf_context()->write_wal_time)/1000000000);
+     write_memtable_time.set_from_double(
+-	static_cast<double>(rocksdb::perf_context.write_memtable_time)/1000000000);
++	static_cast<double>(rocksdb::get_perf_context()->write_memtable_time)/1000000000);
+     write_delay_time.set_from_double(
+-	static_cast<double>(rocksdb::perf_context.write_delay_time)/1000000000);
++	static_cast<double>(rocksdb::get_perf_context()->write_delay_time)/1000000000);
+     write_pre_and_post_process_time.set_from_double(
+-	static_cast<double>(rocksdb::perf_context.write_pre_and_post_process_time)/1000000000);
++	static_cast<double>(rocksdb::get_perf_context()->write_pre_and_post_process_time)/1000000000);
+     logger->tinc(l_rocksdb_write_memtable_time, write_memtable_time);
+     logger->tinc(l_rocksdb_write_delay_time, write_delay_time);
+     logger->tinc(l_rocksdb_write_wal_time, write_wal_time);
+@@ -558,7 +558,7 @@
+   // considering performance overhead, default is disabled
+   if (g_conf->rocksdb_perf) {
+     rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableTimeExceptForMutex);
+-    rocksdb::perf_context.Reset();
++    rocksdb::get_perf_context()->Reset();
+   }
+ 
+   RocksDBTransactionImpl * _t =
+@@ -586,13 +586,13 @@
+     utime_t write_wal_time;
+     utime_t write_pre_and_post_process_time;
+     write_wal_time.set_from_double(
+-	static_cast<double>(rocksdb::perf_context.write_wal_time)/1000000000);
++	static_cast<double>(rocksdb::get_perf_context()->write_wal_time)/1000000000);
+     write_memtable_time.set_from_double(
+-	static_cast<double>(rocksdb::perf_context.write_memtable_time)/1000000000);
++	static_cast<double>(rocksdb::get_perf_context()->write_memtable_time)/1000000000);
+     write_delay_time.set_from_double(
+-	static_cast<double>(rocksdb::perf_context.write_delay_time)/1000000000);
++	static_cast<double>(rocksdb::get_perf_context()->write_delay_time)/1000000000);
+     write_pre_and_post_process_time.set_from_double(
+-	static_cast<double>(rocksdb::perf_context.write_pre_and_post_process_time)/1000000000);
++	static_cast<double>(rocksdb::get_perf_context()->write_pre_and_post_process_time)/1000000000);
+     logger->tinc(l_rocksdb_write_memtable_time, write_memtable_time);
+     logger->tinc(l_rocksdb_write_delay_time, write_delay_time);
+     logger->tinc(l_rocksdb_write_wal_time, write_wal_time);

pkgs/tools/filesystems/ceph/default.nix

@@ -1,20 +1,12 @@
 { callPackage, fetchgit, fetchpatch, ... } @ args:
 
 callPackage ./generic.nix (args // rec {
-  version = "9.2.0";
+  version = "12.2.2";
 
   src = fetchgit {
     url = "https://github.com/ceph/ceph.git";
     rev = "refs/tags/v${version}";
-    sha256 = "0a2v3bgkrbkzardcw7ymlhhyjlwi08qmcm7g34y2sjsxk9bd78an";
+    sha256 = "01anqxyffa8l2lzgyb0dj6fjicfjdx2cq9y1klh24x69gxwkdh00";
   };
 
-  patches = [
-    ./fix-pythonpath.patch
-    # For building with xfsprogs 4.5.0:
-    (fetchpatch {
-      url = "https://github.com/ceph/ceph/commit/602425abd5cef741fc1b5d4d1dd70c68e153fc8d.patch";
-      sha256 = "1iyf0ml2n50ki800vjich8lvzmcdviwqwkbs6cdj0vqv2nc5ii1g";
-    })
-  ];
 })

pkgs/tools/filesystems/ceph/generic.nix

@@ -1,12 +1,17 @@
-{ stdenv, ensureNewerSourcesHook, autoconf, automake, makeWrapper, pkgconfig
+{ stdenv, ensureNewerSourcesHook, cmake, pkgconfig
-, libtool, which, git
+, which, git
-, boost, python2Packages, libxml2, zlib
+, boost, python2Packages
+, libxml2, zlib
+, openldap, lttngUst
+, babeltrace, gperf
+, cunit, snappy
+, rocksdb
 
 # Optional Dependencies
-, snappy ? null, leveldb ? null, yasm ? null, fcgi ? null, expat ? null
+, yasm ? null, fcgi ? null, expat ? null
 , curl ? null, fuse ? null, libibverbs ? null, librdmacm ? null
 , libedit ? null, libatomic_ops ? null, kinetic-cpp-client ? null
-, rocksdb ? null, libs3 ? null
+, libs3 ? null
 
 # Mallocs
 , jemalloc ? null, gperftools ? null
@@ -30,21 +35,11 @@ assert cryptopp != null || (nss != null && nspr != null);
 with stdenv;
 with stdenv.lib;
 let
-  inherit (python2Packages) python;
-  mkFlag = trueStr: falseStr: cond: name: val: "--"
-    + (if cond then trueStr else falseStr)
-    + name
-    + optionalString (val != null && cond != false) "=${val}";
-  mkEnable = mkFlag "enable-" "disable-";
-  mkWith = mkFlag "with-" "without-";
-  mkOther = mkFlag "" "" true;
 
   shouldUsePkg = pkg_: let pkg = (builtins.tryEval pkg_).value;
     in if lib.any (x: x == system) (pkg.meta.platforms or [])
       then pkg else null;
 
-  optSnappy = shouldUsePkg snappy;
-  optLeveldb = shouldUsePkg leveldb;
   optYasm = shouldUsePkg yasm;
   optFcgi = shouldUsePkg fcgi;
   optExpat = shouldUsePkg expat;
@@ -55,7 +50,6 @@ let
   optLibedit = shouldUsePkg libedit;
   optLibatomic_ops = shouldUsePkg libatomic_ops;
   optKinetic-cpp-client = shouldUsePkg kinetic-cpp-client;
-  optRocksdb = shouldUsePkg rocksdb;
   optLibs3 = if versionAtLeast version "10.0.0" then null else shouldUsePkg libs3;
 
   optJemalloc = shouldUsePkg jemalloc;
@@ -69,13 +63,11 @@ let
   optLibxfs = shouldUsePkg libxfs;
   optZfs = shouldUsePkg zfs;
 
-  hasServer = optSnappy != null && optLeveldb != null;
+  hasMon = true;
-  hasMon = hasServer;
+  hasMds = true;
-  hasMds = hasServer;
+  hasOsd = true;
-  hasOsd = hasServer;
   hasRadosgw = optFcgi != null && optExpat != null && optCurl != null && optLibedit != null;
 
-  hasRocksdb = versionAtLeast version "9.0.0" && optRocksdb != null;
 
   # TODO: Reenable when kinetic support is fixed
   #hasKinetic = versionAtLeast version "9.0.0" && optKinetic-cpp-client != null;
@@ -93,192 +85,80 @@ let
     none = [ ];
   };
 
-  wrapArgs = "--set PYTHONPATH \"$(toPythonPath $lib)\""
+  ceph-python-env = python2Packages.python.withPackages (ps: [ 
-    + " --prefix PYTHONPATH : \"$(toPythonPath ${python2Packages.flask})\""
+	ps.sphinx
-    + " --set PATH \"$out/bin\"";
+	ps.flask
+	ps.argparse
+	ps.cython 
+	ps.setuptools
+	ps.pip
+	]);
+
 in
 stdenv.mkDerivation {
   name="ceph-${version}";
 
   inherit src;
 
-  patches = patches ++ [
+  patches = [ 
-    ./0001-Makefile-env-Don-t-force-sbin.patch
+ #	 ./ceph-patch-cmake-path.patch
+    ./0001-kv-RocksDBStore-API-break-additional.patch   
   ];
 
   nativeBuildInputs = [
-    autoconf automake makeWrapper pkgconfig libtool which git
+    cmake
+    pkgconfig which git
     (ensureNewerSourcesHook { year = "1980"; })
-  ]
-    ++ optionals (versionAtLeast version "9.0.2") [
-      python2Packages.setuptools python2Packages.argparse
   ];
+  
   buildInputs = buildInputs ++ cryptoLibsMap.${cryptoStr} ++ [
-    boost python libxml2 optYasm optLibatomic_ops optLibs3 malloc python2Packages.flask zlib
+    boost ceph-python-env libxml2 optYasm optLibatomic_ops optLibs3 
-  ] ++ optionals (versionAtLeast version "9.0.0") [
+    malloc zlib openldap lttngUst babeltrace gperf cunit
-    python2Packages.sphinx # Used for docs
+    snappy rocksdb
   ] ++ optionals stdenv.isLinux [
     linuxHeaders libuuid udev keyutils optLibaio optLibxfs optZfs
-  ] ++ optionals hasServer [
-    optSnappy optLeveldb
   ] ++ optionals hasRadosgw [
     optFcgi optExpat optCurl optFuse optLibedit
-  ] ++ optionals hasRocksdb [
-    optRocksdb
   ] ++ optionals hasKinetic [
     optKinetic-cpp-client
   ];
   
-  postPatch = ''
-    # Fix zfs pkgconfig detection
-    sed -i 's,\[zfs\],\[libzfs\],g' configure.ac
-
-    # Fix seagate kinetic linking
-    sed -i 's,libcrypto.a,-lcrypto,g' src/os/Makefile.am
-  '' + optionalString (versionAtLeast version "9.0.0") ''
-    # Fix gmock
-    patchShebangs src/gmock
-  '';
-
   preConfigure =''
-    # Ceph expects the arch command to be usable during configure
+    # rip off submodule that interfer with system libs
-    # for detecting the assembly type
+	rm -rf src/boost
-    mkdir -p mybin
+	rm -rf src/rocksdb
-    echo "#${stdenv.shell} -e" >> mybin/arch
-    echo "uname -m" >> mybin/arch
-    chmod +x mybin/arch
-    PATH="$PATH:$(pwd)/mybin"
 	
-    ./autogen.sh
+	# require LD_LIBRARY_PATH for cython to find internal dep
+	export LD_LIBRARY_PATH="$PWD/build/lib:$LD_LIBRARY_PATH"
 	
-    # Fix the python site-packages install directory
+	# requires setuptools due to embedded in-cmake setup.py usage
-    sed -i "s,\(PYTHON\(\|_EXEC\)_PREFIX=\).*,\1'$lib',g" configure
+	export PYTHONPATH="${python2Packages.setuptools}/lib/python2.7/site-packages/:$PYTHONPATH"
-
-    # Fix the PYTHONPATH for installing ceph-detect-init to $out
-    mkdir -p "$(toPythonPath $out)"
-    export PYTHONPATH="$(toPythonPath $out):$PYTHONPATH"
   '';
 
-  configureFlags = [
+  cmakeFlags = [ 
-    (mkOther                               "exec_prefix"         "\${out}")
+    "-DENABLE_GIT_VERSION=OFF"
-    (mkOther                               "sysconfdir"          "/etc")
+    "-DWITH_SYSTEM_BOOST=ON"
-    (mkOther                               "localstatedir"       "/var")
+    "-DWITH_SYSTEM_ROCKSDB=ON"
-    (mkOther                               "libdir"              "\${lib}/lib")
+    "-DWITH_LEVELDB=OFF"
-    (mkOther                               "includedir"          "\${lib}/include")
+    
-    (mkWith   true                         "rbd"                  null)
+    # enforce shared lib
-    (mkWith   true                         "cephfs"               null)
+    "-DBUILD_SHARED_LIBS=ON"
-    (mkWith   hasRadosgw                   "radosgw"              null)
+    
-    (mkWith   true                         "radosstriper"         null)
+    # disable cephfs, cmake build broken for now
-    (mkWith   hasServer                    "mon"                  null)
+    "-DWITH_CEPHFS=OFF"
-    (mkWith   hasServer                    "osd"                  null)
+    "-DWITH_LIBCEPHFS=OFF"
-    (mkWith   hasServer                    "mds"                  null)
-    (mkEnable true                         "client"               null)
-    (mkEnable hasServer                    "server"               null)
-    (mkWith   (cryptoStr == "cryptopp")    "cryptopp"             null)
-    (mkWith   (cryptoStr == "nss")         "nss"                  null)
-    (mkEnable false                        "root-make-check"      null)
-    (mkWith   false                        "profiler"             null)
-    (mkWith   false                        "debug"                null)
-    (mkEnable false                        "coverage"             null)
-    (mkWith   (optFuse != null)            "fuse"                 null)
-    (mkWith   (malloc == optJemalloc)      "jemalloc"             null)
-    (mkWith   (malloc == optGperftools)    "tcmalloc"             null)
-    (mkEnable false                        "pgrefdebugging"       null)
-    (mkEnable false                        "cephfs-java"          null)
-    (mkWith   (optLibatomic_ops != null)   "libatomic-ops"        null)
-    (mkWith   true                         "ocf"                  null)
-    (mkWith   hasKinetic                   "kinetic"              null)
-    (mkWith   hasRocksdb                   "librocksdb"           null)
-    (mkWith   false                        "librocksdb-static"    null)
-  ] ++ optional stdenv.isLinux [
-    (mkWith   (optLibaio != null)          "libaio"               null)
-    (mkWith   (optLibxfs != null)          "libxfs"               null)
-    (mkWith   (optZfs != null)             "libzfs"               null)
-  ] ++ optional (versionAtLeast version "0.94.3") [
-    (mkWith   false                        "tcmalloc-minimal"     null)
-  ] ++ optional (versionAtLeast version "9.0.1") [
-    (mkWith   false                        "valgrind"             null)
-  ] ++ optional (versionAtLeast version "9.0.2") [
-    (mkWith   true                         "man-pages"            null)
-    (mkWith   true                         "systemd-libexec-dir"  "\${out}/libexec")
-  ] ++ optional (versionOlder version "9.1.0") [
-    (mkWith   (optLibs3 != null)           "system-libs3"         null)
-    (mkWith   true                         "rest-bench"           null)
-  ] ++ optional (versionAtLeast version "9.1.0") [
-    (mkWith   true                         "rgw-user"             "rgw")
-    (mkWith   true                         "rgw-group"            "rgw")
-    (mkWith   true                         "systemd-unit-dir"     "\${out}/etc/systemd/system")
-    (mkWith   false                        "selinux"              null)  # TODO: Implement
   ];
 
-  preBuild = optionalString (versionAtLeast version "9.0.0") ''
-    (cd src/gmock; make -j $NIX_BUILD_CORES)
-  '';
-
-  installFlags = [ "sysconfdir=\${out}/etc" ];
-
-  outputs = [ "out" "lib" ];
-
-  postInstall = ''
-    # Wrap all of the python scripts
-    wrapProgram $out/bin/ceph ${wrapArgs}
-    wrapProgram $out/bin/ceph-brag ${wrapArgs}
-    wrapProgram $out/bin/ceph-rest-api ${wrapArgs}
-    wrapProgram $out/sbin/ceph-create-keys ${wrapArgs}
-    wrapProgram $out/sbin/ceph-disk ${wrapArgs}
-
-    # Bring in lib as a run-time dependency
-    mkdir -p $out/nix-support
-    echo "$lib" > $out/nix-support/propagated-build-inputs
-
-    # Fix the python library loading
-    find $lib/lib -name \*.pyc -or -name \*.pyd -exec rm {} \;
-    for PY in $(find $lib/lib -name \*.py); do
-      LIBS="$(sed -n "s/.*find_library('\([^)]*\)').*/\1/p" "$PY")"
-
-      # Delete any calls to find_library
-      sed -i '/find_library/d' "$PY"
-
-      # Fix each find_library call
-      for LIB in $LIBS; do
-        REALLIB="$lib/lib/lib$LIB.so"
-        sed -i "s,\(lib$LIB = CDLL(\).*,\1'$REALLIB'),g" "$PY"
-      done
-
-      # Reapply compilation optimizations
-      NAME=$(basename -s .py "$PY")
-      rm -f "$PY"{c,o}
-      pushd "$(dirname $PY)"
-      python -c "import $NAME"
-      python -O -c "import $NAME"
-      popd
-      test -f "$PY"c
-      test -f "$PY"o
-    done
-
-    # Fix .la file link dependencies
-    find "$lib/lib" -name \*.la | xargs sed -i \
-      -e 's,-lboost_[a-z]*,-L${boost.out}/lib \0,g' \
-  '' + optionalString (cryptoStr == "cryptopp") ''
-      -e 's,-lcryptopp,-L${optCryptopp}/lib \0,g' \
-  '' + optionalString (cryptoStr == "nss") ''
-      -e 's,-l\(plds4\|plc4\|nspr4\),-L${optNss}/lib \0,g' \
-      -e 's,-l\(ssl3\|smime3\|nss3\|nssutil3\),-L${optNspr}/lib \0,g' \
-  '' + ''
-
-  '';
-
   enableParallelBuilding = true;
   
+  outputs = [ "dev" "lib" "out" "doc" ];
+
   meta = {
     homepage = http://ceph.com/;
     description = "Distributed storage system";
     license = licenses.lgpl21;
-    maintainers = with maintainers; [ ak wkennington ];
+    maintainers = with maintainers; [ adev ak wkennington ];
     platforms = platforms.unix;
-    # Broken because of https://lwn.net/Vulnerabilities/709844/
-    # and our version is quite out of date.
-    broken = true;
   };
 
   passthru.version = version;

pkgs/top-level/all-packages.nix

@@ -1522,9 +1522,8 @@ with pkgs;
   nrg2iso = callPackage ../tools/cd-dvd/nrg2iso { };
 
   libceph = ceph.lib;
-  ceph = callPackage ../tools/filesystems/ceph { boost = boost159; };
+  ceph = callPackage ../tools/filesystems/ceph { boost = boost165; };
   ceph-dev = ceph;
-  #ceph-dev = lowPrio (callPackage ../tools/filesystems/ceph/dev.nix { });
 
   cfdg = callPackage ../tools/graphics/cfdg {
     ffmpeg = ffmpeg_2;