Merge pull request #112500 from danieldk/cargo-setup-hook

doc/languages-frameworks/rust.section.md

@@ -237,6 +237,104 @@ rustPlatform.buildRustPackage rec {
 }
 ```
 
+## Compiling non-Rust packages that include Rust code
+
+Several non-Rust packages incorporate Rust code for performance- or
+security-sensitive parts. `rustPlatform` exposes several functions and
+hooks that can be used to integrate Cargo in non-Rust packages.
+
+### Vendoring of dependencies
+
+Since network access is not allowed in sandboxed builds, Rust crate
+dependencies need to be retrieved using a fetcher. `rustPlatform`
+provides the `fetchCargoTarball` fetcher, which vendors all
+dependencies of a crate. This fetcher can be used jointly with
+`cargoSetupHook` to vendor dependencies in derivations that do not use
+`buildRustPackage`.
+
+In the following partial example, `fetchCargoTarball` and
+`cargoSetupHook` are used to vendor dependencies in the Python
+`tokenizers` derivation. The `tokenizers` Python package is in the
+`source/bindings/python` directory of the project's source archive. We
+use `fetchCargoTarball` to retrieve the dependencies specified in
+`source/bidings/Cargo.{lock,toml}`. The resulting path is assigned to
+the `cargoDeps` attribute, which is used by `cargoSetupHook` to
+configure Cargo.
+
+```nix
+{ fetchFromGitHub
+, buildPythonPackage
+, rustPlatform
+, setuptools-rust
+}:
+
+buildPythonPackage rec {
+  pname = "tokenizers";
+  version = "0.10.0";
+
+  src = fetchFromGitHub {
+    owner = "huggingface";
+    repo = pname;
+    rev = "python-v${version}";
+    hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
+  };
+
+  cargoDeps = rustPlatform.fetchCargoTarball {
+    inherit src sourceRoot;
+    name = "${pname}-${version}";
+    hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
+  };
+
+  sourceRoot = "source/bindings/python";
+
+  nativeBuildInputs = [ setuptools-rust ] ++ (with rustPlatform; [
+    cargoSetupHook
+    rust.cargo
+    rust.rustc
+  ]);
+
+  # ...
+}
+```
+
+In some projects, the Rust crate is not in the main source directory
+of the projects. In such cases, the `cargoRoot` attribute can be used
+to specify the crate's directory relative to `sourceRoot`. In the
+following example, the crate is in `src/rust`, as specified in the
+`cargoRoot` attribute. Note that we also need to specify the correct
+path for `fetchCargoTarball`.
+
+```nix
+
+{ buildPythonPackage
+, fetchPypi
+, rustPlatform
+, setuptools-rust
+, openssl
+}:
+
+buildPythonPackage rec {
+  pname = "cryptography";
+  version = "3.4.2"; # Also update the hash in vectors.nix
+
+  src = fetchPypi {
+    inherit pname version;
+    sha256 = "1i1mx5y9hkyfi9jrrkcw804hmkcglxi6rmf7vin7jfnbr2bf4q64";
+  };
+
+  cargoDeps = rustPlatform.fetchCargoTarball {
+    inherit src;
+    sourceRoot = "${pname}-${version}/${cargoRoot}";
+    name = "${pname}-${version}";
+    hash = "sha256-PS562W4L1NimqDV2H0jl5vYhL08H9est/pbIxSdYVfo=";
+  };
+
+  cargoRoot = "src/rust";
+
+  # ...
+}
+```
+
 ## Compiling Rust crates using Nix instead of Cargo
 
 ### Simple operation

pkgs/build-support/rust/default.nix

@@ -3,7 +3,7 @@
 , buildPackages
 , cacert
 , cargo
-, diffutils
+, cargoSetupHook
 , fetchCargoTarball
 , runCommandNoCC
 , rustPlatform
@@ -71,19 +71,6 @@ let
   # against the src fixed-output derivation to check consistency.
   validateCargoDeps = !(cargoHash == "" && cargoSha256 == "");
 
-  # Some cargo builds include build hooks that modify their own vendor
-  # dependencies. This copies the vendor directory into the build tree and makes
-  # it writable. If we're using a tarball, the unpackFile hook already handles
-  # this for us automatically.
-  setupVendorDir = if cargoVendorDir == null
-    then (''
-      unpackFile "$cargoDeps"
-      cargoDepsCopy=$(stripHash $cargoDeps)
-    '')
-    else ''
-      cargoDepsCopy="$sourceRoot/${cargoVendorDir}"
-    '';
-
   targetIsJSON = lib.hasSuffix ".json" target;
   useSysroot = targetIsJSON && !__internal_dontAddSysroot;
 
@@ -106,11 +93,6 @@ let
   releaseDir = "target/${shortTarget}/${buildType}";
   tmpDir = "${releaseDir}-tmp";
 
-  # Specify the stdenv's `diff` by abspath to ensure that the user's build
-  # inputs do not cause us to find the wrong `diff`.
-  # The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available.
-  diff = "${diffutils.nativeDrv or diffutils}/bin/diff";
-
 in
 
 # Tests don't currently work for `no_std`, and all custom sysroots are currently built without `std`.
@@ -124,7 +106,7 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u
 
   patchRegistryDeps = ./patch-registry-deps;
 
-  nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo rustc ];
+  nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo cargoSetupHook rustc ];
   buildInputs = buildInputs ++ lib.optional stdenv.hostPlatform.isMinGW windows.pthreads;
 
   patches = cargoPatches ++ patches;
@@ -135,72 +117,9 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u
   postUnpack = ''
     eval "$cargoDepsHook"
 
-    ${setupVendorDir}
-
-    mkdir .cargo
-    config="$(pwd)/$cargoDepsCopy/.cargo/config";
-    if [[ ! -e $config ]]; then
-      config=${./fetchcargo-default-config.toml};
-    fi;
-    substitute $config .cargo/config \
-      --subst-var-by vendor "$(pwd)/$cargoDepsCopy"
-
-    cat >> .cargo/config <<'EOF'
-    [target."${rust.toRustTarget stdenv.buildPlatform}"]
-    "linker" = "${ccForBuild}"
-    ${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) ''
-    [target."${shortTarget}"]
-    "linker" = "${ccForHost}"
-    ${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633
-      lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) ''
-    "rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ]
-    ''}
-    ''}
-    EOF
-
     export RUST_LOG=${logLevel}
   '' + (args.postUnpack or "");
 
-  # After unpacking and applying patches, check that the Cargo.lock matches our
-  # src package. Note that we do this after the patchPhase, because the
-  # patchPhase may create the Cargo.lock if upstream has not shipped one.
-  postPatch = (args.postPatch or "") + lib.optionalString validateCargoDeps ''
-    cargoDepsLockfile=$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock
-    srcLockfile=$NIX_BUILD_TOP/$sourceRoot/Cargo.lock
-
-    echo "Validating consistency between $srcLockfile and $cargoDepsLockfile"
-    if ! ${diff} $srcLockfile $cargoDepsLockfile; then
-
-      # If the diff failed, first double-check that the file exists, so we can
-      # give a friendlier error msg.
-      if ! [ -e $srcLockfile ]; then
-        echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile"
-        echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build."
-        exit 1
-      fi
-
-      if ! [ -e $cargoDepsLockfile ]; then
-        echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile"
-        exit 1
-      fi
-
-      echo
-      echo "ERROR: cargoSha256 is out of date"
-      echo
-      echo "Cargo.lock is not the same in $cargoDepsCopy"
-      echo
-      echo "To fix the issue:"
-      echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value'
-      echo "2. Build the derivation and wait for it to fail with a hash mismatch"
-      echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field"
-      echo
-
-      exit 1
-    fi
-  '' + ''
-    unset cargoDepsCopy
-  '';
-
   configurePhase = args.configurePhase or ''
     runHook preConfigure
     runHook postConfigure

pkgs/build-support/rust/hooks/cargo-setup-hook.sh

@@ -0,0 +1,84 @@
+cargoSetupPostUnpackHook() {
+    echo "Executing cargoSetupPostUnpackHook"
+
+    # Some cargo builds include build hooks that modify their own vendor
+    # dependencies. This copies the vendor directory into the build tree and makes
+    # it writable. If we're using a tarball, the unpackFile hook already handles
+    # this for us automatically.
+    if [ -z $cargoVendorDir ]; then
+        unpackFile "$cargoDeps"
+        export cargoDepsCopy=$(stripHash $cargoDeps)
+    else
+      cargoDepsCopy="$sourceRoot/${cargoRoot:+$cargoRoot/}${cargoVendorDir}"
+    fi
+
+    if [ ! -d .cargo ]; then
+        mkdir .cargo
+    fi
+
+    config="$(pwd)/$cargoDepsCopy/.cargo/config";
+    if [[ ! -e $config ]]; then
+      config=@defaultConfig@
+    fi;
+
+    tmp_config=$(mktemp)
+    substitute $config $tmp_config \
+      --subst-var-by vendor "$(pwd)/$cargoDepsCopy"
+    cat ${tmp_config} >> .cargo/config
+
+    cat >> .cargo/config <<'EOF'
+    @rustTarget@
+EOF
+
+    echo "Finished cargoSetupPostUnpackHook"
+}
+
+# After unpacking and applying patches, check that the Cargo.lock matches our
+# src package. Note that we do this after the patchPhase, because the
+# patchPhase may create the Cargo.lock if upstream has not shipped one.
+cargoSetupPostPatchHook() {
+    echo "Executing cargoSetupPostPatchHook"
+
+    cargoDepsLockfile="$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock"
+    srcLockfile="$NIX_BUILD_TOP/$sourceRoot/${cargoRoot:+$cargoRoot/}/Cargo.lock"
+
+    echo "Validating consistency between $srcLockfile and $cargoDepsLockfile"
+    if ! @diff@ $srcLockfile $cargoDepsLockfile; then
+
+      # If the diff failed, first double-check that the file exists, so we can
+      # give a friendlier error msg.
+      if ! [ -e $srcLockfile ]; then
+        echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile"
+        echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build."
+        exit 1
+      fi
+
+      if ! [ -e $cargoDepsLockfile ]; then
+        echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile"
+        exit 1
+      fi
+
+      echo
+      echo "ERROR: cargoSha256 is out of date"
+      echo
+      echo "Cargo.lock is not the same in $cargoDepsCopy"
+      echo
+      echo "To fix the issue:"
+      echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value'
+      echo "2. Build the derivation and wait for it to fail with a hash mismatch"
+      echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field"
+      echo
+
+      exit 1
+    fi
+
+    unset cargoDepsCopy
+
+    echo "Finished cargoSetupPostPatchHook"
+}
+
+postUnpackHooks+=(cargoSetupPostUnpackHook)
+
+if [ -z ${cargoVendorDir-} ]; then
+  postPatchHooks+=(cargoSetupPostPatchHook)
+fi

pkgs/build-support/rust/hooks/default.nix

@@ -0,0 +1,49 @@
+{ buildPackages
+, callPackage
+, diffutils
+, lib
+, makeSetupHook
+, rust
+, stdenv
+, target ? rust.toRustTargetSpec stdenv.hostPlatform
+}:
+
+let
+  targetIsJSON = lib.hasSuffix ".json" target;
+
+  # see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168
+  # the "${}" is needed to transform the path into a /nix/store path before baseNameOf
+  shortTarget = if targetIsJSON then
+      (lib.removeSuffix ".json" (builtins.baseNameOf "${target}"))
+    else target;
+  ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
+  ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
+in {
+  cargoSetupHook = callPackage ({ }:
+    makeSetupHook {
+      name = "cargo-setup-hook.sh";
+      deps = [ ];
+      substitutions = {
+        defaultConfig = ../fetchcargo-default-config.toml;
+
+        # Specify the stdenv's `diff` by abspath to ensure that the user's build
+        # inputs do not cause us to find the wrong `diff`.
+        # The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available.
+        diff = "${diffutils.nativeDrv or diffutils}/bin/diff";
+
+        # Target platform
+        rustTarget = ''
+          [target."${rust.toRustTarget stdenv.buildPlatform}"]
+          "linker" = "${ccForBuild}"
+          ${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) ''
+            [target."${shortTarget}"]
+            "linker" = "${ccForHost}"
+            ${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633
+            lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) ''
+              "rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ]
+            ''}
+          ''}
+        '';
+      };
+    } ./cargo-setup-hook.sh) {};
+}

pkgs/development/compilers/rust/make-rust-platform.nix

@@ -12,7 +12,7 @@ rec {
   };
 
   buildRustPackage = callPackage ../../../build-support/rust {
-    inherit rustc cargo fetchCargoTarball;
+    inherit rustc cargo cargoSetupHook fetchCargoTarball;
   };
 
   rustcSrc = callPackage ./rust-src.nix {
@@ -22,4 +22,7 @@ rec {
   rustLibSrc = callPackage ./rust-lib-src.nix {
     inherit rustc;
   };
+
+  # Hooks
+  inherit (callPackage ../../../build-support/rust/hooks { }) cargoSetupHook;
 }

pkgs/development/python-modules/cryptography/default.nix

@@ -2,8 +2,8 @@
 , buildPythonPackage
 , fetchPypi
 , fetchpatch
-, isPy27
+, rustPlatform
-, ipaddress
+, setuptools-rust
 , openssl
 , cryptography_vectors
 , darwin
@@ -13,27 +13,38 @@
 , isPyPy
 , cffi
 , pytest
+, pytest-subtests
 , pretend
 , iso8601
 , pytz
 , hypothesis
-, enum34
 }:
 
 buildPythonPackage rec {
   pname = "cryptography";
-  version = "3.3.2"; # Also update the hash in vectors.nix
+  version = "3.4.2"; # Also update the hash in vectors.nix
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "1vcvw4lkw1spiq322pm1256kail8nck6bbgpdxx3pqa905wd6q2s";
+    sha256 = "1i1mx5y9hkyfi9jrrkcw804hmkcglxi6rmf7vin7jfnbr2bf4q64";
   };
 
+  cargoDeps = rustPlatform.fetchCargoTarball {
+    inherit src;
+    sourceRoot = "${pname}-${version}/${cargoRoot}";
+    name = "${pname}-${version}";
+    hash = "sha256-PS562W4L1NimqDV2H0jl5vYhL08H9est/pbIxSdYVfo=";
+  };
+
+  cargoRoot = "src/rust";
+
   outputs = [ "out" "dev" ];
 
   nativeBuildInputs = lib.optionals (!isPyPy) [
     cffi
-  ];
+    rustPlatform.cargoSetupHook
+    setuptools-rust
+  ] ++ (with rustPlatform; [ rust.cargo rust.rustc ]);
 
   buildInputs = [ openssl ]
              ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
@@ -42,8 +53,6 @@ buildPythonPackage rec {
     six
   ] ++ lib.optionals (!isPyPy) [
     cffi
-  ] ++ lib.optionals isPy27 [
-    ipaddress enum34
   ];
 
   checkInputs = [
@@ -52,6 +61,7 @@ buildPythonPackage rec {
     iso8601
     pretend
     pytest
+    pytest-subtests
     pytz
   ];
 

pkgs/development/python-modules/cryptography/vectors.nix

@@ -7,7 +7,7 @@ buildPythonPackage rec {
 
   src = fetchPypi {
     inherit pname version;
-    sha256 = "1yhaps0f3h2yjb6lmz953z1l1d84y9swk4k3gj9nqyk4vbx5m7cc";
+    sha256 = "0i888rrfn7116lj7f2nr4amd2z45sk6866zizjfpsn5wh2713cls";
   };
 
   # No tests included

pkgs/development/python-modules/tokenizers/default.nix

@@ -1,12 +1,10 @@
 { lib
-, rustPlatform
 , fetchFromGitHub
 , fetchurl
-, pipInstallHook
+, buildPythonPackage
+, rustPlatform
 , setuptools-rust
-, wheel
 , numpy
-, python
 , datasets
 , pytestCheckHook
 , requests
@@ -49,7 +47,7 @@ let
     url = "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt";
     sha256 = "09a754pm4djjglv3x5pkgwd6f79i2rq8ydg0f7c3q1wmwqdbba8f";
   };
-in rustPlatform.buildRustPackage rec {
+in buildPythonPackage rec {
   pname = "tokenizers";
   version = "0.10.0";
 
@@ -60,19 +58,22 @@ in rustPlatform.buildRustPackage rec {
     hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
   };
 
-  cargoSha256 = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
+  cargoDeps = rustPlatform.fetchCargoTarball {
+    inherit src sourceRoot;
+    name = "${pname}-${version}";
+    hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
+  };
 
   sourceRoot = "source/bindings/python";
 
-  nativeBuildInputs = [
+  nativeBuildInputs = [ setuptools-rust ] ++ (with rustPlatform; [
-    pipInstallHook
+    cargoSetupHook
-    setuptools-rust
+    rust.cargo
-    wheel
+    rust.rustc
-  ];
+  ]);
 
   propagatedBuildInputs = [
     numpy
-    python
   ];
 
   installCheckInputs = [
@@ -99,14 +100,6 @@ in rustPlatform.buildRustPackage rec {
       ln -s ${openaiMerges} openai-gpt-merges.txt )
   '';
 
-  buildPhase = ''
-    ${python.interpreter} setup.py bdist_wheel
-  '';
-
-  installPhase = ''
-    pipInstallPhase
-  '';
-
   preCheck = ''
     HOME=$TMPDIR
   '';