diff --git a/doc/languages-frameworks/rust.section.md b/doc/languages-frameworks/rust.section.md index 8f6db28ab4d..e9850111376 100644 --- a/doc/languages-frameworks/rust.section.md +++ b/doc/languages-frameworks/rust.section.md @@ -237,6 +237,104 @@ rustPlatform.buildRustPackage rec { } ``` +## Compiling non-Rust packages that include Rust code + +Several non-Rust packages incorporate Rust code for performance- or +security-sensitive parts. `rustPlatform` exposes several functions and +hooks that can be used to integrate Cargo in non-Rust packages. + +### Vendoring of dependencies + +Since network access is not allowed in sandboxed builds, Rust crate +dependencies need to be retrieved using a fetcher. `rustPlatform` +provides the `fetchCargoTarball` fetcher, which vendors all +dependencies of a crate. This fetcher can be used jointly with +`cargoSetupHook` to vendor dependencies in derivations that do not use +`buildRustPackage`. + +In the following partial example, `fetchCargoTarball` and +`cargoSetupHook` are used to vendor dependencies in the Python +`tokenizers` derivation. The `tokenizers` Python package is in the +`source/bindings/python` directory of the project's source archive. We +use `fetchCargoTarball` to retrieve the dependencies specified in +`source/bidings/Cargo.{lock,toml}`. The resulting path is assigned to +the `cargoDeps` attribute, which is used by `cargoSetupHook` to +configure Cargo. + +```nix +{ fetchFromGitHub +, buildPythonPackage +, rustPlatform +, setuptools-rust +}: + +buildPythonPackage rec { + pname = "tokenizers"; + version = "0.10.0"; + + src = fetchFromGitHub { + owner = "huggingface"; + repo = pname; + rev = "python-v${version}"; + hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw="; + }; + + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src sourceRoot; + name = "${pname}-${version}"; + hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0="; + }; + + sourceRoot = "source/bindings/python"; + + nativeBuildInputs = [ setuptools-rust ] ++ (with rustPlatform; [ + cargoSetupHook + rust.cargo + rust.rustc + ]); + + # ... +} +``` + +In some projects, the Rust crate is not in the main source directory +of the projects. In such cases, the `cargoRoot` attribute can be used +to specify the crate's directory relative to `sourceRoot`. In the +following example, the crate is in `src/rust`, as specified in the +`cargoRoot` attribute. Note that we also need to specify the correct +path for `fetchCargoTarball`. + +```nix + +{ buildPythonPackage +, fetchPypi +, rustPlatform +, setuptools-rust +, openssl +}: + +buildPythonPackage rec { + pname = "cryptography"; + version = "3.4.2"; # Also update the hash in vectors.nix + + src = fetchPypi { + inherit pname version; + sha256 = "1i1mx5y9hkyfi9jrrkcw804hmkcglxi6rmf7vin7jfnbr2bf4q64"; + }; + + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src; + sourceRoot = "${pname}-${version}/${cargoRoot}"; + name = "${pname}-${version}"; + hash = "sha256-PS562W4L1NimqDV2H0jl5vYhL08H9est/pbIxSdYVfo="; + }; + + cargoRoot = "src/rust"; + + # ... +} +``` + ## Compiling Rust crates using Nix instead of Cargo ### Simple operation diff --git a/pkgs/build-support/rust/default.nix b/pkgs/build-support/rust/default.nix index dc86a7dc581..6741b329a30 100644 --- a/pkgs/build-support/rust/default.nix +++ b/pkgs/build-support/rust/default.nix @@ -3,7 +3,7 @@ , buildPackages , cacert , cargo -, diffutils +, cargoSetupHook , fetchCargoTarball , runCommandNoCC , rustPlatform @@ -71,19 +71,6 @@ let # against the src fixed-output derivation to check consistency. validateCargoDeps = !(cargoHash == "" && cargoSha256 == ""); - # Some cargo builds include build hooks that modify their own vendor - # dependencies. This copies the vendor directory into the build tree and makes - # it writable. If we're using a tarball, the unpackFile hook already handles - # this for us automatically. - setupVendorDir = if cargoVendorDir == null - then ('' - unpackFile "$cargoDeps" - cargoDepsCopy=$(stripHash $cargoDeps) - '') - else '' - cargoDepsCopy="$sourceRoot/${cargoVendorDir}" - ''; - targetIsJSON = lib.hasSuffix ".json" target; useSysroot = targetIsJSON && !__internal_dontAddSysroot; @@ -106,11 +93,6 @@ let releaseDir = "target/${shortTarget}/${buildType}"; tmpDir = "${releaseDir}-tmp"; - # Specify the stdenv's `diff` by abspath to ensure that the user's build - # inputs do not cause us to find the wrong `diff`. - # The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available. - diff = "${diffutils.nativeDrv or diffutils}/bin/diff"; - in # Tests don't currently work for `no_std`, and all custom sysroots are currently built without `std`. @@ -124,7 +106,7 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u patchRegistryDeps = ./patch-registry-deps; - nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo rustc ]; + nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo cargoSetupHook rustc ]; buildInputs = buildInputs ++ lib.optional stdenv.hostPlatform.isMinGW windows.pthreads; patches = cargoPatches ++ patches; @@ -135,72 +117,9 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u postUnpack = '' eval "$cargoDepsHook" - ${setupVendorDir} - - mkdir .cargo - config="$(pwd)/$cargoDepsCopy/.cargo/config"; - if [[ ! -e $config ]]; then - config=${./fetchcargo-default-config.toml}; - fi; - substitute $config .cargo/config \ - --subst-var-by vendor "$(pwd)/$cargoDepsCopy" - - cat >> .cargo/config <<'EOF' - [target."${rust.toRustTarget stdenv.buildPlatform}"] - "linker" = "${ccForBuild}" - ${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) '' - [target."${shortTarget}"] - "linker" = "${ccForHost}" - ${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633 - lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) '' - "rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ] - ''} - ''} - EOF - export RUST_LOG=${logLevel} '' + (args.postUnpack or ""); - # After unpacking and applying patches, check that the Cargo.lock matches our - # src package. Note that we do this after the patchPhase, because the - # patchPhase may create the Cargo.lock if upstream has not shipped one. - postPatch = (args.postPatch or "") + lib.optionalString validateCargoDeps '' - cargoDepsLockfile=$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock - srcLockfile=$NIX_BUILD_TOP/$sourceRoot/Cargo.lock - - echo "Validating consistency between $srcLockfile and $cargoDepsLockfile" - if ! ${diff} $srcLockfile $cargoDepsLockfile; then - - # If the diff failed, first double-check that the file exists, so we can - # give a friendlier error msg. - if ! [ -e $srcLockfile ]; then - echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile" - echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build." - exit 1 - fi - - if ! [ -e $cargoDepsLockfile ]; then - echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile" - exit 1 - fi - - echo - echo "ERROR: cargoSha256 is out of date" - echo - echo "Cargo.lock is not the same in $cargoDepsCopy" - echo - echo "To fix the issue:" - echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value' - echo "2. Build the derivation and wait for it to fail with a hash mismatch" - echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field" - echo - - exit 1 - fi - '' + '' - unset cargoDepsCopy - ''; - configurePhase = args.configurePhase or '' runHook preConfigure runHook postConfigure diff --git a/pkgs/build-support/rust/hooks/cargo-setup-hook.sh b/pkgs/build-support/rust/hooks/cargo-setup-hook.sh new file mode 100644 index 00000000000..0fddd30582a --- /dev/null +++ b/pkgs/build-support/rust/hooks/cargo-setup-hook.sh @@ -0,0 +1,84 @@ +cargoSetupPostUnpackHook() { + echo "Executing cargoSetupPostUnpackHook" + + # Some cargo builds include build hooks that modify their own vendor + # dependencies. This copies the vendor directory into the build tree and makes + # it writable. If we're using a tarball, the unpackFile hook already handles + # this for us automatically. + if [ -z $cargoVendorDir ]; then + unpackFile "$cargoDeps" + export cargoDepsCopy=$(stripHash $cargoDeps) + else + cargoDepsCopy="$sourceRoot/${cargoRoot:+$cargoRoot/}${cargoVendorDir}" + fi + + if [ ! -d .cargo ]; then + mkdir .cargo + fi + + config="$(pwd)/$cargoDepsCopy/.cargo/config"; + if [[ ! -e $config ]]; then + config=@defaultConfig@ + fi; + + tmp_config=$(mktemp) + substitute $config $tmp_config \ + --subst-var-by vendor "$(pwd)/$cargoDepsCopy" + cat ${tmp_config} >> .cargo/config + + cat >> .cargo/config <<'EOF' + @rustTarget@ +EOF + + echo "Finished cargoSetupPostUnpackHook" +} + +# After unpacking and applying patches, check that the Cargo.lock matches our +# src package. Note that we do this after the patchPhase, because the +# patchPhase may create the Cargo.lock if upstream has not shipped one. +cargoSetupPostPatchHook() { + echo "Executing cargoSetupPostPatchHook" + + cargoDepsLockfile="$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock" + srcLockfile="$NIX_BUILD_TOP/$sourceRoot/${cargoRoot:+$cargoRoot/}/Cargo.lock" + + echo "Validating consistency between $srcLockfile and $cargoDepsLockfile" + if ! @diff@ $srcLockfile $cargoDepsLockfile; then + + # If the diff failed, first double-check that the file exists, so we can + # give a friendlier error msg. + if ! [ -e $srcLockfile ]; then + echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile" + echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build." + exit 1 + fi + + if ! [ -e $cargoDepsLockfile ]; then + echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile" + exit 1 + fi + + echo + echo "ERROR: cargoSha256 is out of date" + echo + echo "Cargo.lock is not the same in $cargoDepsCopy" + echo + echo "To fix the issue:" + echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value' + echo "2. Build the derivation and wait for it to fail with a hash mismatch" + echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field" + echo + + exit 1 + fi + + unset cargoDepsCopy + + echo "Finished cargoSetupPostPatchHook" +} + +postUnpackHooks+=(cargoSetupPostUnpackHook) + +if [ -z ${cargoVendorDir-} ]; then + postPatchHooks+=(cargoSetupPostPatchHook) +fi diff --git a/pkgs/build-support/rust/hooks/default.nix b/pkgs/build-support/rust/hooks/default.nix new file mode 100644 index 00000000000..dea749f9393 --- /dev/null +++ b/pkgs/build-support/rust/hooks/default.nix @@ -0,0 +1,49 @@ +{ buildPackages +, callPackage +, diffutils +, lib +, makeSetupHook +, rust +, stdenv +, target ? rust.toRustTargetSpec stdenv.hostPlatform +}: + +let + targetIsJSON = lib.hasSuffix ".json" target; + + # see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168 + # the "${}" is needed to transform the path into a /nix/store path before baseNameOf + shortTarget = if targetIsJSON then + (lib.removeSuffix ".json" (builtins.baseNameOf "${target}")) + else target; + ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc"; + ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; +in { + cargoSetupHook = callPackage ({ }: + makeSetupHook { + name = "cargo-setup-hook.sh"; + deps = [ ]; + substitutions = { + defaultConfig = ../fetchcargo-default-config.toml; + + # Specify the stdenv's `diff` by abspath to ensure that the user's build + # inputs do not cause us to find the wrong `diff`. + # The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available. + diff = "${diffutils.nativeDrv or diffutils}/bin/diff"; + + # Target platform + rustTarget = '' + [target."${rust.toRustTarget stdenv.buildPlatform}"] + "linker" = "${ccForBuild}" + ${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) '' + [target."${shortTarget}"] + "linker" = "${ccForHost}" + ${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633 + lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) '' + "rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ] + ''} + ''} + ''; + }; + } ./cargo-setup-hook.sh) {}; +} diff --git a/pkgs/development/compilers/rust/make-rust-platform.nix b/pkgs/development/compilers/rust/make-rust-platform.nix index 4b1f572bebb..c7c7a8cdb45 100644 --- a/pkgs/development/compilers/rust/make-rust-platform.nix +++ b/pkgs/development/compilers/rust/make-rust-platform.nix @@ -12,7 +12,7 @@ rec { }; buildRustPackage = callPackage ../../../build-support/rust { - inherit rustc cargo fetchCargoTarball; + inherit rustc cargo cargoSetupHook fetchCargoTarball; }; rustcSrc = callPackage ./rust-src.nix { @@ -22,4 +22,7 @@ rec { rustLibSrc = callPackage ./rust-lib-src.nix { inherit rustc; }; + + # Hooks + inherit (callPackage ../../../build-support/rust/hooks { }) cargoSetupHook; } diff --git a/pkgs/development/python-modules/cryptography/default.nix b/pkgs/development/python-modules/cryptography/default.nix index eb4eba0f587..d64f4a9792b 100644 --- a/pkgs/development/python-modules/cryptography/default.nix +++ b/pkgs/development/python-modules/cryptography/default.nix @@ -2,8 +2,8 @@ , buildPythonPackage , fetchPypi , fetchpatch -, isPy27 -, ipaddress +, rustPlatform +, setuptools-rust , openssl , cryptography_vectors , darwin @@ -13,27 +13,38 @@ , isPyPy , cffi , pytest +, pytest-subtests , pretend , iso8601 , pytz , hypothesis -, enum34 }: buildPythonPackage rec { pname = "cryptography"; - version = "3.3.2"; # Also update the hash in vectors.nix + version = "3.4.2"; # Also update the hash in vectors.nix src = fetchPypi { inherit pname version; - sha256 = "1vcvw4lkw1spiq322pm1256kail8nck6bbgpdxx3pqa905wd6q2s"; + sha256 = "1i1mx5y9hkyfi9jrrkcw804hmkcglxi6rmf7vin7jfnbr2bf4q64"; }; + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src; + sourceRoot = "${pname}-${version}/${cargoRoot}"; + name = "${pname}-${version}"; + hash = "sha256-PS562W4L1NimqDV2H0jl5vYhL08H9est/pbIxSdYVfo="; + }; + + cargoRoot = "src/rust"; + outputs = [ "out" "dev" ]; nativeBuildInputs = lib.optionals (!isPyPy) [ cffi - ]; + rustPlatform.cargoSetupHook + setuptools-rust + ] ++ (with rustPlatform; [ rust.cargo rust.rustc ]); buildInputs = [ openssl ] ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; @@ -42,8 +53,6 @@ buildPythonPackage rec { six ] ++ lib.optionals (!isPyPy) [ cffi - ] ++ lib.optionals isPy27 [ - ipaddress enum34 ]; checkInputs = [ @@ -52,6 +61,7 @@ buildPythonPackage rec { iso8601 pretend pytest + pytest-subtests pytz ]; diff --git a/pkgs/development/python-modules/cryptography/vectors.nix b/pkgs/development/python-modules/cryptography/vectors.nix index f9b7c525237..16151a84124 100644 --- a/pkgs/development/python-modules/cryptography/vectors.nix +++ b/pkgs/development/python-modules/cryptography/vectors.nix @@ -7,7 +7,7 @@ buildPythonPackage rec { src = fetchPypi { inherit pname version; - sha256 = "1yhaps0f3h2yjb6lmz953z1l1d84y9swk4k3gj9nqyk4vbx5m7cc"; + sha256 = "0i888rrfn7116lj7f2nr4amd2z45sk6866zizjfpsn5wh2713cls"; }; # No tests included diff --git a/pkgs/development/python-modules/tokenizers/default.nix b/pkgs/development/python-modules/tokenizers/default.nix index cf122613f63..ef265b87c05 100644 --- a/pkgs/development/python-modules/tokenizers/default.nix +++ b/pkgs/development/python-modules/tokenizers/default.nix @@ -1,12 +1,10 @@ { lib -, rustPlatform , fetchFromGitHub , fetchurl -, pipInstallHook +, buildPythonPackage +, rustPlatform , setuptools-rust -, wheel , numpy -, python , datasets , pytestCheckHook , requests @@ -49,7 +47,7 @@ let url = "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt"; sha256 = "09a754pm4djjglv3x5pkgwd6f79i2rq8ydg0f7c3q1wmwqdbba8f"; }; -in rustPlatform.buildRustPackage rec { +in buildPythonPackage rec { pname = "tokenizers"; version = "0.10.0"; @@ -60,19 +58,22 @@ in rustPlatform.buildRustPackage rec { hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw="; }; - cargoSha256 = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0="; + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src sourceRoot; + name = "${pname}-${version}"; + hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0="; + }; sourceRoot = "source/bindings/python"; - nativeBuildInputs = [ - pipInstallHook - setuptools-rust - wheel - ]; + nativeBuildInputs = [ setuptools-rust ] ++ (with rustPlatform; [ + cargoSetupHook + rust.cargo + rust.rustc + ]); propagatedBuildInputs = [ numpy - python ]; installCheckInputs = [ @@ -99,14 +100,6 @@ in rustPlatform.buildRustPackage rec { ln -s ${openaiMerges} openai-gpt-merges.txt ) ''; - buildPhase = '' - ${python.interpreter} setup.py bdist_wheel - ''; - - installPhase = '' - pipInstallPhase - ''; - preCheck = '' HOME=$TMPDIR '';