Merge pull request #61009 from thoughtpolice/nixpkgs/fdb-61-fixes

foundationdb: add fdb 6.1 with cmake build

nixos/modules/services/databases/foundationdb.nix

@@ -35,6 +35,7 @@ let
     ${optionalString (cfg.class != null) "class = ${cfg.class}"}
     memory         = ${cfg.memory}
     storage_memory = ${cfg.storageMemory}
+    trace_format   = ${cfg.traceFormat}
 
     ${optionalString (cfg.tls != null) ''
       tls_plugin           = ${pkg}/libexec/plugins/FDBLibTLS.so
@@ -317,6 +318,12 @@ in
       default     = "/run/foundationdb.pid";
       description = "Path to pidfile for fdbmonitor.";
     };
+
+    traceFormat = mkOption {
+      type = types.enum [ "xml" "json" ];
+      default = "xml";
+      description = "Trace logging format.";
+    };
   };
 
   config = mkIf cfg.enable {
@@ -382,7 +389,7 @@ in
           chown -R ${cfg.user}:${cfg.group} ${cfg.pidfile}
 
         for x in "${cfg.logDir}" "${cfg.dataDir}"; do
-          [ ! -d "$x" ] && mkdir -m 0700 -vp "$x";
+          [ ! -d "$x" ] && mkdir -m 0770 -vp "$x";
           chown -R ${cfg.user}:${cfg.group} "$x";
         done
 
@@ -404,7 +411,7 @@ in
 
       postStart = ''
         if [ -e "${cfg.dataDir}/.first_startup" ]; then
-          fdbcli --exec "configure new single memory"
+          fdbcli --exec "configure new single ssd"
           rm -f "${cfg.dataDir}/.first_startup";
         fi
       '';

pkgs/servers/foundationdb/cmake.nix

@@ -0,0 +1,129 @@
+# This builder is for FoundationDB CMake build system.
+
+{ lib, fetchurl, fetchpatch, fetchFromGitHub
+, cmake, ninja, boost, python3, openjdk, mono, libressl
+
+, gccStdenv, llvmPackages
+, useClang ? true
+, ...
+}:
+
+let
+  stdenv = if useClang then llvmPackages.libcxxStdenv else gccStdenv;
+
+  tests = with builtins;
+    builtins.replaceStrings [ "\n" ] [ " " ] (lib.fileContents ./test-list.txt);
+
+  makeFdb =
+    { version
+    , branch
+    , sha256
+    , rev ? "refs/tags/${version}"
+    , officialRelease ? true
+    , patches ? []
+    }: stdenv.mkDerivation rec {
+        name = "foundationdb-${version}";
+        inherit version;
+
+        src = fetchFromGitHub {
+          owner = "apple";
+          repo  = "foundationdb";
+          inherit rev sha256;
+        };
+
+        buildInputs = [ libressl boost ];
+        nativeBuildInputs = [ cmake ninja python3 openjdk mono ]
+          ++ lib.optional useClang [ llvmPackages.lld ];
+
+        separateDebugInfo = true;
+        enableParallelBuilding = true;
+        dontFixCmake = true;
+
+        cmakeFlags =
+          [ "-DCMAKE_BUILD_TYPE=Release"
+            "-DLIBRESSL_USE_STATIC_LIBS=FALSE"
+
+            # CMake can't find these easily for some reason?
+            "-DLIBRESSL_INCLUDE_DIR=${libressl.dev}"
+            "-DLIBRESSL_CRYPTO_LIBRARY=${libressl.out}/lib/libcrypto.so"
+            "-DLIBRESSL_SSL_LIBRARY=${libressl.out}/lib/libssl.so"
+            "-DLIBRESSL_TLS_LIBRARY=${libressl.out}/lib/libtls.so"
+
+            # LTO brings up overall build time, but results in much smaller
+            # binaries for all users and the cache.
+            #
+            # TODO FIXME: bugs :(
+            (lib.optionalString (!useClang) "-DUSE_LTO=ON")
+
+            # Gold helps alleviate the link time, especially when LTO is
+            # enabled. But even then, it still takes a majority of the time.
+            # Same with LLD when Clang is available.
+            (lib.optionalString useClang    "-DUSE_LD=LLD")
+            (lib.optionalString (!useClang) "-DUSE_LD=GOLD")
+          ]
+          ++ lib.optional officialRelease [ "FDB_RELEASE=1" ];
+
+        inherit patches;
+        postPatch = ''
+          for x in bindings/c/CMakeLists.txt fdbserver/CMakeLists.txt fdbmonitor/CMakeLists.txt fdbbackup/CMakeLists.txt fdbcli/CMakeLists.txt; do 
+            substituteInPlace $x --replace 'fdb_install' 'install'
+          done
+        '';
+
+        # the install phase for cmake is pretty wonky right now since it's not designed to
+        # coherently install packages as most linux distros expect -- it's designed to build
+        # packaged artifacts that are shipped in RPMs, etc. we need to add some extra code to
+        # cmake upstream to fix this, and if we do, i think most of this can go away.
+        postInstall = ''
+          mv $out/sbin/fdbserver $out/bin/fdbserver
+          rm -rf \
+            $out/lib/systemd $out/Library $out/usr $out/sbin \
+            $out/var $out/log $out/etc
+
+          mv $out/fdbmonitor/fdbmonitor $out/bin/fdbmonitor && rm -rf $out/fdbmonitor
+
+          rm -rf $out/lib/foundationdb/
+          mkdir $out/libexec && ln -sfv $out/bin/fdbbackup $out/libexec/backup_agent
+
+          mkdir $out/include/foundationdb && \
+            mv $out/include/*.h $out/include/*.options $out/include/foundationdb
+
+          # move results into multi outputs
+          mkdir -p $dev $lib
+          mv $out/include $dev/include
+          mv $out/lib $lib/lib
+
+          # python bindings
+          # NB: use the original setup.py.in, so we can substitute VERSION correctly
+          cp ../LICENSE ./bindings/python
+          substitute ../bindings/python/setup.py.in ./bindings/python/setup.py \
+            --replace 'VERSION' "${version}"
+          rm -f ./bindings/python/setup.py.* ./bindings/python/CMakeLists.txt
+          rm -f ./bindings/python/fdb/*.pth # remove useless files
+          rm -f ./bindings/python/*.rst ./bindings/python/*.mk
+
+          cp -R ./bindings/python/                          tmp-pythonsrc/
+          tar -zcf $pythonsrc --transform s/tmp-pythonsrc/python-foundationdb/ ./tmp-pythonsrc/
+
+          # java bindings
+          mkdir -p $lib/share/java
+          mv lib/fdb-java-*.jar $lib/share/java/fdb-java.jar
+
+          # include the tests
+          mkdir -p $out/share/test
+          (cd ../tests && for x in ${tests}; do
+            cp --parents $x $out/share/test
+          done)
+        '';
+
+        outputs = [ "out" "dev" "lib" "pythonsrc" ];
+
+        meta = with stdenv.lib; {
+          description = "Open source, distributed, transactional key-value store";
+          homepage    = https://www.foundationdb.org;
+          license     = licenses.asl20;
+          platforms   = [ "x86_64-linux" ];
+          maintainers = with maintainers; [ thoughtpolice ];
+       };
+    };
+in makeFdb

pkgs/servers/foundationdb/default.nix

@@ -1,194 +1,84 @@
-{ stdenv49
+{ stdenv, stdenv49, gcc9Stdenv, llvmPackages_8
 , lib, fetchurl, fetchpatch, fetchFromGitHub
 
-, which, findutils, m4, gawk
+, cmake, ninja, which, findutils, m4, gawk
-, python, openjdk, mono, libressl
+, python, python3, openjdk, mono, libressl, boost
-}:
+}@args:
 
 let
-  # hysterical raisins dictate a version of boost this old. however,
+  vsmakeBuild = import ./vsmake.nix args;
-  # we luckily do not need to build anything, we just need the header
+  cmakeBuild = import ./cmake.nix (args // {
-  # files.
+    gccStdenv    = gcc9Stdenv;
-  boost152 = stdenv49.mkDerivation rec {
+    llvmPackages = llvmPackages_8;
-    name = "boost-headers-1.52.0";
+  });
 
-    src = fetchurl {
+  python3-six-patch = fetchpatch {
-      url = "mirror://sourceforge/boost/boost_1_52_0.tar.bz2";
+    name   = "update-python-six.patch";
-      sha256 = "14mc7gsnnahdjaxbbslzk79rc0d12h1i681cd3srdwr3fzynlar2";
+    url    = "https://github.com/apple/foundationdb/commit/4bd9efc4fc74917bc04b07a84eb065070ea7edb2.patch";
-    };
+    sha256 = "030679lmc86f1wzqqyvxnwjyfrhh54pdql20ab3iifqpp9i5mi85";
-
-    configurePhase = ":";
-    buildPhase = ":";
-    installPhase = "mkdir -p $out/include && cp -R boost $out/include/";
   };
 
-  makeFdb =
+  python3-print-patch = fetchpatch {
-    { version
+    name   = "import-for-python-print.patch";
-    , branch
+    url    = "https://github.com/apple/foundationdb/commit/ded17c6cd667f39699cf663c0e87fe01e996c153.patch";
-    , sha256
+    sha256 = "11y434w68cpk7shs2r22hyrpcrqi8vx02cw7v5x79qxvnmdxv2an";
-
+  };
-    # the revision can be inferred from the fdb tagging policy
-    , rev    ? "refs/tags/${version}"
-
-    # in theory newer versions of fdb support newer compilers, but they
-    # don't :( maybe one day
-    , stdenv ? stdenv49
-
-    # in theory newer versions of fdb support newer boost versions, but they
-    # don't :( maybe one day
-    , boost ? boost152
-
-    # if an release is unofficial/a prerelease, then make sure this is set
-    , officialRelease ? true
-    }: stdenv.mkDerivation rec {
-        name = "foundationdb-${version}";
-        inherit version;
-
-        src = fetchFromGitHub {
-          owner = "apple";
-          repo  = "foundationdb";
-          inherit rev sha256;
-        };
-
-        nativeBuildInputs = [ python openjdk gawk which m4 findutils mono ];
-        buildInputs = [ libressl boost ];
-
-        patches =
-          [ # For 5.2+, we need a slightly adjusted patch to fix all the ldflags
-            (if lib.versionAtLeast version "5.2"
-             then (if lib.versionAtLeast version "6.0"
-                   then ./ldflags-6.0.patch
-                   else ./ldflags-5.2.patch)
-             else ./ldflags-5.1.patch)
-          ]
-          # for 6.0+, we do NOT need to apply this version fix, since we can specify
-          # it ourselves. see configurePhase
-          ++ (lib.optional (!lib.versionAtLeast version "6.0") ./fix-scm-version.patch)
-          # Versions less than 6.0 have a busted Python 3 build due to an outdated
-          # use of 'print'. Also apply an update to the six module with many bugfixes,
-          # which is in 6.0+ as well
-          ++ (lib.optional (!lib.versionAtLeast version "6.0") (fetchpatch {
-            name   = "update-python-six.patch";
-            url    = "https://github.com/apple/foundationdb/commit/4bd9efc4fc74917bc04b07a84eb065070ea7edb2.patch";
-            sha256 = "030679lmc86f1wzqqyvxnwjyfrhh54pdql20ab3iifqpp9i5mi85";
-          }))
-          ++ (lib.optional (!lib.versionAtLeast version "6.0") (fetchpatch {
-            name   = "import-for-python-print.patch";
-            url    = "https://github.com/apple/foundationdb/commit/ded17c6cd667f39699cf663c0e87fe01e996c153.patch";
-            sha256 = "11y434w68cpk7shs2r22hyrpcrqi8vx02cw7v5x79qxvnmdxv2an";
-          }))
-          ;
-
-        postPatch = ''
-          # note: this does not do anything for 6.0+
-          substituteInPlace ./build/scver.mk \
-            --subst-var-by NIXOS_FDB_VERSION_ID "${rev}" \
-            --subst-var-by NIXOS_FDB_SCBRANCH   "${branch}"
-
-          substituteInPlace ./Makefile \
-            --replace 'shell which ccache' 'shell true' \
-            --replace -Werror ""
-
-          substituteInPlace ./Makefile \
-            --replace libstdc++_pic libstdc++
-
-          substituteInPlace ./build/link-validate.sh \
-            --replace 'exit 1' '#exit 1'
-
-          patchShebangs .
-        '' + lib.optionalString (lib.versionAtLeast version "6.0") ''
-          substituteInPlace ./Makefile \
-            --replace 'TLS_LIBS +=' '#TLS_LIBS +=' \
-            --replace 'LDFLAGS :=' 'LDFLAGS := -ltls -lssl -lcrypto'
-        '';
-
-        separateDebugInfo = true;
-        enableParallelBuilding = true;
-
-        makeFlags = [ "all" "fdb_java" "fdb_python" ]
-          # Don't compile FDBLibTLS if we don't need it in 6.0 or later;
-          # it gets statically linked in
-          ++ lib.optional (!lib.versionAtLeast version "6.0") [ "fdb_c" ]
-          # Needed environment overrides
-          ++ [ "KVRELEASE=1"
-               "NOSTRIP=1"
-             ] ++ lib.optional officialRelease [ "RELEASE=true" ];
-
-        # on 6.0 and later, we can specify all this information manually
-        configurePhase = lib.optionalString (lib.versionAtLeast version "6.0") ''
-          export SOURCE_CONTROL=GIT
-          export SCBRANCH="${branch}"
-          export VERSION_ID="${rev}"
-        '';
-
-        installPhase = ''
-          mkdir -vp $out/{bin,libexec/plugins} $lib/{lib,share/java} $dev/include/foundationdb
-
-        '' + lib.optionalString (!lib.versionAtLeast version "6.0") ''
-          # we only copy the TLS library on < 6.0, since it's compiled-in otherwise
-          cp -v ./lib/libFDBLibTLS.so $out/libexec/plugins/FDBLibTLS.so
-        '' + ''
-
-          # C API
-          cp -v ./lib/libfdb_c.so                           $lib/lib
-          cp -v ./bindings/c/foundationdb/fdb_c.h           $dev/include/foundationdb
-          cp -v ./bindings/c/foundationdb/fdb_c_options.g.h $dev/include/foundationdb
-          cp -v ./fdbclient/vexillographer/fdb.options      $dev/include/foundationdb
-
-          # java
-          cp -v ./bindings/java/foundationdb-client.jar     $lib/share/java/fdb-java.jar
-
-          # python
-          cp LICENSE ./bindings/python
-          substitute ./bindings/python/setup.py.in ./bindings/python/setup.py \
-            --replace 'VERSION' "${version}"
-          rm -f ./bindings/python/setup.py.in
-          rm -f ./bindings/python/fdb/*.pth # remove useless files
-          rm -f ./bindings/python/*.rst ./bindings/python/*.mk
-
-          cp -R ./bindings/python/                          tmp-pythonsrc/
-          tar -zcf $pythonsrc --transform s/tmp-pythonsrc/python-foundationdb/ ./tmp-pythonsrc/
-
-          # binaries
-          for x in fdbbackup fdbcli fdbserver fdbmonitor; do
-            cp -v "./bin/$x" $out/bin;
-          done
-
-          ln -sfv $out/bin/fdbbackup $out/bin/dr_agent
-          ln -sfv $out/bin/fdbbackup $out/bin/fdbrestore
-          ln -sfv $out/bin/fdbbackup $out/bin/fdbdr
-
-          ln -sfv $out/bin/fdbbackup $out/libexec/backup_agent
-        '';
-
-        outputs = [ "out" "lib" "dev" "pythonsrc" ];
-
-        meta = with stdenv.lib; {
-          description = "Open source, distributed, transactional key-value store";
-          homepage    = https://www.foundationdb.org;
-          license     = licenses.asl20;
-          platforms   = [ "x86_64-linux" ];
-          maintainers = with maintainers; [ thoughtpolice ];
-       };
-    };
 
 in with builtins; {
 
-  foundationdb51 = makeFdb rec {
+  # Older versions use the bespoke 'vsmake' build system
+  # ------------------------------------------------------
+
+  foundationdb51 = vsmakeBuild rec {
     version = "5.1.7";
     branch  = "release-5.1";
     sha256  = "1rc472ih24f9s5g3xmnlp3v62w206ny0pvvw02bzpix2sdrpbp06";
+
+    patches = [
+      ./patches/ldflags-5.1.patch
+      ./patches/fix-scm-version.patch
+      python3-six-patch
+      python3-print-patch
+    ];
   };
 
-  foundationdb52 = makeFdb rec {
+  foundationdb52 = vsmakeBuild rec {
     version = "5.2.8";
     branch  = "release-5.2";
     sha256  = "1kbmmhk2m9486r4kyjlc7bb3wd50204i0p6dxcmvl6pbp1bs0wlb";
+
+    patches = [
+      ./patches/ldflags-5.2.patch
+      ./patches/fix-scm-version.patch
+      python3-six-patch
+      python3-print-patch
+    ];
   };
 
-  foundationdb60 = makeFdb rec {
+  foundationdb60 = vsmakeBuild rec {
     version = "6.0.18";
     branch  = "release-6.0";
     sha256  = "0q1mscailad0z7zf1nypv4g7gx3damfp45nf8nzyq47nsw5gz69p";
+
+    patches = [
+      ./patches/ldflags-6.0.patch
+    ];
   };
+
+  # 6.1 and later versions should always use CMake
+  # ------------------------------------------------------
+
+  foundationdb61 = cmakeBuild rec {
+    version = "6.1.6pre4898_${substring 0 7 rev}";
+    branch  = "release-6.1";
+    rev     = "26fbbbf798971b2b9ecb882a8af766fa36734f53";
+    sha256  = "1q1a1j8h0qlh67khcds0dg416myvjbp6gfm6s4sk8d60zfzny7wb";
+    officialRelease = false;
+
+    patches = [
+      ./patches/clang-libcxx.patch
+      ./patches/suppress-clang-warnings.patch
+    ];
+  };
+
 }

pkgs/servers/foundationdb/patches/clang-libcxx.patch

@@ -0,0 +1,52 @@
+commit 7ed4745a092a203f92fc37ab5894e92117db0c94
+Author: Austin Seipp <aseipp@pobox.com>
+Date:   Sat May 4 15:23:35 2019 -0500
+
+    flow: fix a build failure with Clang/libcxx on Linux
+    
+    11bd7d7da introduced a hack on Linux to work around a missing symbol in
+    libstdc++'s _pic library on Ubuntu. Unfortunately, this causes the build
+    to fail when using Clang, as it doesn't believe this symbol is part of
+    its headers in c++11 mode.
+    
+    Unfortunately there's no good way to distinguish libcxx from libstdc++
+    with the preprocessor, so we merely gate it by only checking for clang,
+    iff we are on Linux.
+    
+    With this change, Clang 8.x can build FoundationDB on Linux using libcxx
+    as the standard C++ library.
+    
+    Signed-off-by: Austin Seipp <aseipp@pobox.com>
+
+diff --git a/flow/Platform.cpp b/flow/Platform.cpp
+index 3d3f1ac0..9f21dfd4 100644
+--- a/flow/Platform.cpp
++++ b/flow/Platform.cpp
+@@ -2841,13 +2841,26 @@ void setupSlowTaskProfiler() {
+ #endif
+ }
+ 
+-#ifdef __linux__
++#if defined(__linux__) && !defined(__clang__)
+ // There's no good place to put this, so it's here.
+ // Ubuntu's packaging of libstdc++_pic offers different symbols than libstdc++.  Go figure.
+ // Notably, it's missing a definition of std::istream::ignore(long), which causes compilation errors
+ // in the bindings.  Thus, we provide weak versions of their definitions, so that if the
+ // linked-against libstdc++ is missing their definitions, we'll be able to use the provided
+ // ignore(long, int) version.
++//
++// Note that this hack is DISABLED when we use Clang. It is only needed when we statically link
++// to the _pic libraries, but only official FDB Linux binaries are built this way using GCC. If we
++// don't use the _pic libraries, then this hack is entirely unneeded -- likely the case when using
++// Clang on Linux.
++//
++// Doing this allows us to use LLVM's libc++ with Clang on Linux -- otherwise, providing
++// a weak symbol definition for an internal (non-public) class member fails (due to that member
++// being non-existant on libc++.) See upstream GitHub issue #1533 for more information.
++//
++// TODO FIXME: Obliterate this when the official build environment is upgraded beyond Ubuntu 14.04.
++// (This problem should be fixed in later LTS releases.)
++
+ #include <istream>
+ namespace std {
+ typedef basic_istream<char, std::char_traits<char>> char_basic_istream;

pkgs/servers/foundationdb/patches/suppress-clang-warnings.patch

@@ -0,0 +1,34 @@
+commit 8076537a52bb026941f13f5542395aac69ef0825
+Author: Austin Seipp <aseipp@pobox.com>
+Date:   Sat May 4 17:34:51 2019 -0500
+
+    cmake: add workarounds for NixOS-specific deficiencies [NixOS]
+    
+    The NixOS debug builder hook adds '-Wa,--compress-debug-sections' to the
+    link flags (it actually adds it to the compiler flags, but the compiler
+    is used for linking, so...). This makes the compiler angry when -Werror
+    is passed, because it's unused at link-time (-Wa applies to the
+    assembler). Suppress this warning with -Wno-unused-command-line-argument
+    
+    NB: we *could* use -Wno-error=unused-command-line-argument, but that
+    still results in warnings anyway, just not fatal ones. We'd like to
+    remove them all for the sake of the build output.
+    
+    Signed-off-by: Austin Seipp <aseipp@pobox.com>
+
+diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake
+index 03af9c10..7d059375 100644
+--- a/cmake/ConfigureCompiler.cmake
++++ b/cmake/ConfigureCompiler.cmake
+@@ -119,6 +119,11 @@ else()
+   else()
+     add_compile_options(-Werror)
+   endif()
++  if (CLANG)
++    # aseipp: NixOS hack
++    add_compile_options(-Wno-unused-command-line-argument)
++    add_link_options(-Wno-unused-command-line-argument)
++  endif()
+   add_compile_options($<$<BOOL:${GCC}>:-Wno-pragmas>)
+   add_compile_options(-Wno-error=format
+     -Wunused-variable

pkgs/servers/foundationdb/test-list.txt

@@ -0,0 +1,80 @@
+fast/AtomicBackupCorrectness.txt
+fast/AtomicBackupToDBCorrectness.txt
+fast/AtomicOps.txt
+fast/AtomicOpsApiCorrectness.txt
+fast/BackupCorrectness.txt
+fast/BackupCorrectnessClean.txt
+fast/BackupToDBCorrectness.txt
+fast/BackupToDBCorrectnessClean.txt
+fast/CloggedSideband.txt
+fast/ConstrainedRandomSelector.txt
+fast/CycleAndLock.txt
+fast/CycleTest.txt
+fast/FuzzApiCorrectness.txt
+fast/FuzzApiCorrectnessClean.txt
+fast/IncrementTest.txt
+fast/InventoryTestAlmostReadOnly.txt
+fast/InventoryTestSomeWrites.txt
+fast/KillRegionCycle.txt
+fast/LongStackWriteDuringRead.txt
+fast/LowLatency.txt
+fast/MemoryLifetime.txt
+fast/MoveKeysCycle.txt
+fast/RandomSelector.txt
+fast/RandomUnitTests.txt
+fast/SelectorCorrectness.txt
+fast/Sideband.txt
+fast/SidebandWithStatus.txt
+fast/SwizzledRollbackSideband.txt
+fast/SystemRebootTestCycle.txt
+fast/TaskBucketCorrectness.txt
+fast/TimeKeeperCorrectness.txt
+fast/Unreadable.txt
+fast/VersionStamp.txt
+fast/Watches.txt
+fast/WriteDuringRead.txt
+fast/WriteDuringReadClean.txt
+rare/CheckRelocation.txt
+rare/ClogUnclog.txt
+rare/CloggedCycleWithKills.txt
+rare/ConflictRangeCheck.txt
+rare/ConflictRangeRYOWCheck.txt
+rare/CycleRollbackClogged.txt
+rare/CycleWithKills.txt
+rare/FuzzTest.txt
+rare/InventoryTestHeavyWrites.txt
+rare/LargeApiCorrectness.txt
+rare/LargeApiCorrectnessStatus.txt
+rare/RYWDisable.txt
+rare/RandomReadWriteTest.txt
+rare/SwizzledLargeApiCorrectness.txt
+slow/ApiCorrectness.txt
+slow/ApiCorrectnessAtomicRestore.txt
+slow/ApiCorrectnessSwitchover.txt
+slow/ClogWithRollbacks.txt
+slow/CloggedCycleTest.txt
+slow/CloggedStorefront.txt
+slow/CommitBug.txt
+slow/ConfigureTest.txt
+slow/CycleRollbackPlain.txt
+slow/DDBalanceAndRemove.txt
+slow/DDBalanceAndRemoveStatus.txt
+slow/FastTriggeredWatches.txt
+slow/LowLatencyWithFailures.txt
+slow/MoveKeysClean.txt
+slow/MoveKeysSideband.txt
+slow/RyowCorrectness.txt
+slow/Serializability.txt
+slow/SharedBackupCorrectness.txt
+slow/SharedBackupToDBCorrectness.txt
+slow/StorefrontTest.txt
+slow/SwizzledApiCorrectness.txt
+slow/SwizzledCycleTest.txt
+slow/SwizzledDdBalance.txt
+slow/SwizzledRollbackTimeLapse.txt
+slow/SwizzledRollbackTimeLapseIncrement.txt
+slow/VersionStampBackupToDB.txt
+slow/VersionStampSwitchover.txt
+slow/WriteDuringReadAtomicRestore.txt
+slow/WriteDuringReadSwitchover.txt
+slow/ddbalance.txt

pkgs/servers/foundationdb/vsmake.nix

@@ -0,0 +1,154 @@
+# This builder is for FoundationDB's original, somewhat strange visual studio +
+# make build system. In FoundationDB 6.1 and later, there's a new CMake system
+# (which will eventually become the default version.)
+{ stdenv49, lib, fetchurl, fetchFromGitHub
+
+, which, findutils, m4, gawk
+, python, openjdk, mono, libressl
+, ...
+}:
+
+let
+  # hysterical raisins dictate a version of boost this old. however,
+  # we luckily do not need to build anything, we just need the header
+  # files.
+  boost152 = stdenv49.mkDerivation rec {
+    name = "boost-headers-1.52.0";
+
+    src = fetchurl {
+      url = "mirror://sourceforge/boost/boost_1_52_0.tar.bz2";
+      sha256 = "14mc7gsnnahdjaxbbslzk79rc0d12h1i681cd3srdwr3fzynlar2";
+    };
+
+    configurePhase = ":";
+    buildPhase = ":";
+    installPhase = "mkdir -p $out/include && cp -R boost $out/include/";
+  };
+
+  makeFdb =
+    { version
+    , branch
+    , sha256
+
+    # the revision can be inferred from the fdb tagging policy
+    , rev    ? "refs/tags/${version}"
+
+    # in theory newer versions of fdb support newer compilers, but they
+    # don't :( maybe one day
+    , stdenv ? stdenv49
+
+    # in theory newer versions of fdb support newer boost versions, but they
+    # don't :( maybe one day
+    , boost ? boost152
+
+    # if an release is unofficial/a prerelease, then make sure this is set
+    , officialRelease ? true
+
+    , patches ? []
+    }: stdenv.mkDerivation rec {
+        name = "foundationdb-${version}";
+        inherit version;
+
+        src = fetchFromGitHub {
+          owner = "apple";
+          repo  = "foundationdb";
+          inherit rev sha256;
+        };
+
+        nativeBuildInputs = [ python openjdk gawk which m4 findutils mono ];
+        buildInputs = [ libressl boost ];
+
+        inherit patches;
+        postPatch = ''
+          # note: this does not do anything for 6.0+
+          substituteInPlace ./build/scver.mk \
+            --subst-var-by NIXOS_FDB_VERSION_ID "${rev}" \
+            --subst-var-by NIXOS_FDB_SCBRANCH   "${branch}"
+
+          substituteInPlace ./Makefile \
+            --replace 'shell which ccache' 'shell true' \
+            --replace -Werror ""
+
+          substituteInPlace ./Makefile \
+            --replace libstdc++_pic libstdc++
+
+          substituteInPlace ./build/link-validate.sh \
+            --replace 'exit 1' '#exit 1'
+
+          patchShebangs .
+        '' + lib.optionalString (lib.versionAtLeast version "6.0") ''
+          substituteInPlace ./Makefile \
+            --replace 'TLS_LIBS +=' '#TLS_LIBS +=' \
+            --replace 'LDFLAGS :=' 'LDFLAGS := -ltls -lssl -lcrypto'
+        '';
+
+        separateDebugInfo = true;
+        enableParallelBuilding = true;
+
+        makeFlags = [ "all" "fdb_java" "fdb_python" ]
+          # Don't compile FDBLibTLS if we don't need it in 6.0 or later;
+          # it gets statically linked in
+          ++ lib.optional (!lib.versionAtLeast version "6.0") [ "fdb_c" ]
+          # Needed environment overrides
+          ++ [ "KVRELEASE=1"
+               "NOSTRIP=1"
+             ] ++ lib.optional officialRelease [ "RELEASE=true" ];
+
+        # on 6.0 and later, we can specify all this information manually
+        configurePhase = lib.optionalString (lib.versionAtLeast version "6.0") ''
+          export SOURCE_CONTROL=GIT
+          export SCBRANCH="${branch}"
+          export VERSION_ID="${rev}"
+        '';
+
+        installPhase = ''
+          mkdir -vp $out/{bin,libexec/plugins} $lib/{lib,share/java} $dev/include/foundationdb
+
+        '' + lib.optionalString (!lib.versionAtLeast version "6.0") ''
+          # we only copy the TLS library on < 6.0, since it's compiled-in otherwise
+          cp -v ./lib/libFDBLibTLS.so $out/libexec/plugins/FDBLibTLS.so
+        '' + ''
+
+          # C API
+          cp -v ./lib/libfdb_c.so                           $lib/lib
+          cp -v ./bindings/c/foundationdb/fdb_c.h           $dev/include/foundationdb
+          cp -v ./bindings/c/foundationdb/fdb_c_options.g.h $dev/include/foundationdb
+          cp -v ./fdbclient/vexillographer/fdb.options      $dev/include/foundationdb
+
+          # java
+          cp -v ./bindings/java/foundationdb-client.jar     $lib/share/java/fdb-java.jar
+
+          # python
+          cp LICENSE ./bindings/python
+          substitute ./bindings/python/setup.py.in ./bindings/python/setup.py \
+            --replace 'VERSION' "${version}"
+          rm -f ./bindings/python/setup.py.in
+          rm -f ./bindings/python/fdb/*.pth # remove useless files
+          rm -f ./bindings/python/*.rst ./bindings/python/*.mk
+
+          cp -R ./bindings/python/                          tmp-pythonsrc/
+          tar -zcf $pythonsrc --transform s/tmp-pythonsrc/python-foundationdb/ ./tmp-pythonsrc/
+
+          # binaries
+          for x in fdbbackup fdbcli fdbserver fdbmonitor; do
+            cp -v "./bin/$x" $out/bin;
+          done
+
+          ln -sfv $out/bin/fdbbackup $out/bin/dr_agent
+          ln -sfv $out/bin/fdbbackup $out/bin/fdbrestore
+          ln -sfv $out/bin/fdbbackup $out/bin/fdbdr
+
+          ln -sfv $out/bin/fdbbackup $out/libexec/backup_agent
+        '';
+
+        outputs = [ "out" "lib" "dev" "pythonsrc" ];
+
+        meta = with stdenv.lib; {
+          description = "Open source, distributed, transactional key-value store";
+          homepage    = https://www.foundationdb.org;
+          license     = licenses.asl20;
+          platforms   = [ "x86_64-linux" ];
+          maintainers = with maintainers; [ thoughtpolice ];
+       };
+    };
+in makeFdb

pkgs/top-level/all-packages.nix

@@ -3026,7 +3026,9 @@ in
   inherit (fdbPackages)
     foundationdb51
     foundationdb52
-    foundationdb60;
+    foundationdb60
+    foundationdb61
+  ;
 
   foundationdb = foundationdb60;
 

pkgs/top-level/python-packages.nix

@@ -5528,6 +5528,7 @@ in {
   foundationdb51 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb51; };
   foundationdb52 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb52; };
   foundationdb60 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb60; };
+  foundationdb61 = callPackage ../servers/foundationdb/python.nix { foundationdb = pkgs.foundationdb61; };
 
   libtorrentRasterbar = (toPythonModule (pkgs.libtorrentRasterbar.override {
     inherit python;