From d083f412fa516e18bb3e48602fded860a2693d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Tue, 9 Feb 2021 11:38:25 +0100 Subject: [PATCH 1/4] buildRustPackage: factor out setting up .cargo/config to cargoSetupHook This makes it possible to reuse this functionality as a hook in derivations that do not use buildRustPackage. --- pkgs/build-support/rust/default.nix | 85 +------------------ .../rust/hooks/cargo-setup-hook.sh | 84 ++++++++++++++++++ pkgs/build-support/rust/hooks/default.nix | 49 +++++++++++ .../compilers/rust/make-rust-platform.nix | 5 +- 4 files changed, 139 insertions(+), 84 deletions(-) create mode 100644 pkgs/build-support/rust/hooks/cargo-setup-hook.sh create mode 100644 pkgs/build-support/rust/hooks/default.nix diff --git a/pkgs/build-support/rust/default.nix b/pkgs/build-support/rust/default.nix index dc86a7dc581..6741b329a30 100644 --- a/pkgs/build-support/rust/default.nix +++ b/pkgs/build-support/rust/default.nix @@ -3,7 +3,7 @@ , buildPackages , cacert , cargo -, diffutils +, cargoSetupHook , fetchCargoTarball , runCommandNoCC , rustPlatform @@ -71,19 +71,6 @@ let # against the src fixed-output derivation to check consistency. validateCargoDeps = !(cargoHash == "" && cargoSha256 == ""); - # Some cargo builds include build hooks that modify their own vendor - # dependencies. This copies the vendor directory into the build tree and makes - # it writable. If we're using a tarball, the unpackFile hook already handles - # this for us automatically. - setupVendorDir = if cargoVendorDir == null - then ('' - unpackFile "$cargoDeps" - cargoDepsCopy=$(stripHash $cargoDeps) - '') - else '' - cargoDepsCopy="$sourceRoot/${cargoVendorDir}" - ''; - targetIsJSON = lib.hasSuffix ".json" target; useSysroot = targetIsJSON && !__internal_dontAddSysroot; @@ -106,11 +93,6 @@ let releaseDir = "target/${shortTarget}/${buildType}"; tmpDir = "${releaseDir}-tmp"; - # Specify the stdenv's `diff` by abspath to ensure that the user's build - # inputs do not cause us to find the wrong `diff`. - # The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available. - diff = "${diffutils.nativeDrv or diffutils}/bin/diff"; - in # Tests don't currently work for `no_std`, and all custom sysroots are currently built without `std`. @@ -124,7 +106,7 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u patchRegistryDeps = ./patch-registry-deps; - nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo rustc ]; + nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo cargoSetupHook rustc ]; buildInputs = buildInputs ++ lib.optional stdenv.hostPlatform.isMinGW windows.pthreads; patches = cargoPatches ++ patches; @@ -135,72 +117,9 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u postUnpack = '' eval "$cargoDepsHook" - ${setupVendorDir} - - mkdir .cargo - config="$(pwd)/$cargoDepsCopy/.cargo/config"; - if [[ ! -e $config ]]; then - config=${./fetchcargo-default-config.toml}; - fi; - substitute $config .cargo/config \ - --subst-var-by vendor "$(pwd)/$cargoDepsCopy" - - cat >> .cargo/config <<'EOF' - [target."${rust.toRustTarget stdenv.buildPlatform}"] - "linker" = "${ccForBuild}" - ${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) '' - [target."${shortTarget}"] - "linker" = "${ccForHost}" - ${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633 - lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) '' - "rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ] - ''} - ''} - EOF - export RUST_LOG=${logLevel} '' + (args.postUnpack or ""); - # After unpacking and applying patches, check that the Cargo.lock matches our - # src package. Note that we do this after the patchPhase, because the - # patchPhase may create the Cargo.lock if upstream has not shipped one. - postPatch = (args.postPatch or "") + lib.optionalString validateCargoDeps '' - cargoDepsLockfile=$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock - srcLockfile=$NIX_BUILD_TOP/$sourceRoot/Cargo.lock - - echo "Validating consistency between $srcLockfile and $cargoDepsLockfile" - if ! ${diff} $srcLockfile $cargoDepsLockfile; then - - # If the diff failed, first double-check that the file exists, so we can - # give a friendlier error msg. - if ! [ -e $srcLockfile ]; then - echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile" - echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build." - exit 1 - fi - - if ! [ -e $cargoDepsLockfile ]; then - echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile" - exit 1 - fi - - echo - echo "ERROR: cargoSha256 is out of date" - echo - echo "Cargo.lock is not the same in $cargoDepsCopy" - echo - echo "To fix the issue:" - echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value' - echo "2. Build the derivation and wait for it to fail with a hash mismatch" - echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field" - echo - - exit 1 - fi - '' + '' - unset cargoDepsCopy - ''; - configurePhase = args.configurePhase or '' runHook preConfigure runHook postConfigure diff --git a/pkgs/build-support/rust/hooks/cargo-setup-hook.sh b/pkgs/build-support/rust/hooks/cargo-setup-hook.sh new file mode 100644 index 00000000000..0fddd30582a --- /dev/null +++ b/pkgs/build-support/rust/hooks/cargo-setup-hook.sh @@ -0,0 +1,84 @@ +cargoSetupPostUnpackHook() { + echo "Executing cargoSetupPostUnpackHook" + + # Some cargo builds include build hooks that modify their own vendor + # dependencies. This copies the vendor directory into the build tree and makes + # it writable. If we're using a tarball, the unpackFile hook already handles + # this for us automatically. + if [ -z $cargoVendorDir ]; then + unpackFile "$cargoDeps" + export cargoDepsCopy=$(stripHash $cargoDeps) + else + cargoDepsCopy="$sourceRoot/${cargoRoot:+$cargoRoot/}${cargoVendorDir}" + fi + + if [ ! -d .cargo ]; then + mkdir .cargo + fi + + config="$(pwd)/$cargoDepsCopy/.cargo/config"; + if [[ ! -e $config ]]; then + config=@defaultConfig@ + fi; + + tmp_config=$(mktemp) + substitute $config $tmp_config \ + --subst-var-by vendor "$(pwd)/$cargoDepsCopy" + cat ${tmp_config} >> .cargo/config + + cat >> .cargo/config <<'EOF' + @rustTarget@ +EOF + + echo "Finished cargoSetupPostUnpackHook" +} + +# After unpacking and applying patches, check that the Cargo.lock matches our +# src package. Note that we do this after the patchPhase, because the +# patchPhase may create the Cargo.lock if upstream has not shipped one. +cargoSetupPostPatchHook() { + echo "Executing cargoSetupPostPatchHook" + + cargoDepsLockfile="$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock" + srcLockfile="$NIX_BUILD_TOP/$sourceRoot/${cargoRoot:+$cargoRoot/}/Cargo.lock" + + echo "Validating consistency between $srcLockfile and $cargoDepsLockfile" + if ! @diff@ $srcLockfile $cargoDepsLockfile; then + + # If the diff failed, first double-check that the file exists, so we can + # give a friendlier error msg. + if ! [ -e $srcLockfile ]; then + echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile" + echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build." + exit 1 + fi + + if ! [ -e $cargoDepsLockfile ]; then + echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile" + exit 1 + fi + + echo + echo "ERROR: cargoSha256 is out of date" + echo + echo "Cargo.lock is not the same in $cargoDepsCopy" + echo + echo "To fix the issue:" + echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value' + echo "2. Build the derivation and wait for it to fail with a hash mismatch" + echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field" + echo + + exit 1 + fi + + unset cargoDepsCopy + + echo "Finished cargoSetupPostPatchHook" +} + +postUnpackHooks+=(cargoSetupPostUnpackHook) + +if [ -z ${cargoVendorDir-} ]; then + postPatchHooks+=(cargoSetupPostPatchHook) +fi diff --git a/pkgs/build-support/rust/hooks/default.nix b/pkgs/build-support/rust/hooks/default.nix new file mode 100644 index 00000000000..dea749f9393 --- /dev/null +++ b/pkgs/build-support/rust/hooks/default.nix @@ -0,0 +1,49 @@ +{ buildPackages +, callPackage +, diffutils +, lib +, makeSetupHook +, rust +, stdenv +, target ? rust.toRustTargetSpec stdenv.hostPlatform +}: + +let + targetIsJSON = lib.hasSuffix ".json" target; + + # see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168 + # the "${}" is needed to transform the path into a /nix/store path before baseNameOf + shortTarget = if targetIsJSON then + (lib.removeSuffix ".json" (builtins.baseNameOf "${target}")) + else target; + ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc"; + ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; +in { + cargoSetupHook = callPackage ({ }: + makeSetupHook { + name = "cargo-setup-hook.sh"; + deps = [ ]; + substitutions = { + defaultConfig = ../fetchcargo-default-config.toml; + + # Specify the stdenv's `diff` by abspath to ensure that the user's build + # inputs do not cause us to find the wrong `diff`. + # The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available. + diff = "${diffutils.nativeDrv or diffutils}/bin/diff"; + + # Target platform + rustTarget = '' + [target."${rust.toRustTarget stdenv.buildPlatform}"] + "linker" = "${ccForBuild}" + ${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) '' + [target."${shortTarget}"] + "linker" = "${ccForHost}" + ${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633 + lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) '' + "rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ] + ''} + ''} + ''; + }; + } ./cargo-setup-hook.sh) {}; +} diff --git a/pkgs/development/compilers/rust/make-rust-platform.nix b/pkgs/development/compilers/rust/make-rust-platform.nix index 4b1f572bebb..c7c7a8cdb45 100644 --- a/pkgs/development/compilers/rust/make-rust-platform.nix +++ b/pkgs/development/compilers/rust/make-rust-platform.nix @@ -12,7 +12,7 @@ rec { }; buildRustPackage = callPackage ../../../build-support/rust { - inherit rustc cargo fetchCargoTarball; + inherit rustc cargo cargoSetupHook fetchCargoTarball; }; rustcSrc = callPackage ./rust-src.nix { @@ -22,4 +22,7 @@ rec { rustLibSrc = callPackage ./rust-lib-src.nix { inherit rustc; }; + + # Hooks + inherit (callPackage ../../../build-support/rust/hooks { }) cargoSetupHook; } From 4a4da22674b0808a85604d72db41adb9bc2de1a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Tue, 9 Feb 2021 11:41:01 +0100 Subject: [PATCH 2/4] python3Packages.tokenizers: switch to buildPythonPackage Use the new cargoSetupHook to set up Cargo vendoring, so that we do not need buildRustPackage anymore. --- .../python-modules/tokenizers/default.nix | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/pkgs/development/python-modules/tokenizers/default.nix b/pkgs/development/python-modules/tokenizers/default.nix index cf122613f63..ef265b87c05 100644 --- a/pkgs/development/python-modules/tokenizers/default.nix +++ b/pkgs/development/python-modules/tokenizers/default.nix @@ -1,12 +1,10 @@ { lib -, rustPlatform , fetchFromGitHub , fetchurl -, pipInstallHook +, buildPythonPackage +, rustPlatform , setuptools-rust -, wheel , numpy -, python , datasets , pytestCheckHook , requests @@ -49,7 +47,7 @@ let url = "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt"; sha256 = "09a754pm4djjglv3x5pkgwd6f79i2rq8ydg0f7c3q1wmwqdbba8f"; }; -in rustPlatform.buildRustPackage rec { +in buildPythonPackage rec { pname = "tokenizers"; version = "0.10.0"; @@ -60,19 +58,22 @@ in rustPlatform.buildRustPackage rec { hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw="; }; - cargoSha256 = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0="; + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src sourceRoot; + name = "${pname}-${version}"; + hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0="; + }; sourceRoot = "source/bindings/python"; - nativeBuildInputs = [ - pipInstallHook - setuptools-rust - wheel - ]; + nativeBuildInputs = [ setuptools-rust ] ++ (with rustPlatform; [ + cargoSetupHook + rust.cargo + rust.rustc + ]); propagatedBuildInputs = [ numpy - python ]; installCheckInputs = [ @@ -99,14 +100,6 @@ in rustPlatform.buildRustPackage rec { ln -s ${openaiMerges} openai-gpt-merges.txt ) ''; - buildPhase = '' - ${python.interpreter} setup.py bdist_wheel - ''; - - installPhase = '' - pipInstallPhase - ''; - preCheck = '' HOME=$TMPDIR ''; From da73f94622f309b13230c50c0ec93c22c705c4ba Mon Sep 17 00:00:00 2001 From: Michael Weiss Date: Mon, 8 Feb 2021 14:07:11 +0100 Subject: [PATCH 3/4] python3Packages.cryptography: 3.3.2 -> 3.4.2 Backwards incompatible changes: Support for Python 2 has been removed. Note: This isn't a problem for Nixpkgs because pythonPackages.cryptography is frozen at version 3.3.2. Other important packaging changes: "Cryptography now incorporates Rust code. Users building cryptography themselves will need to have the Rust toolchain installed. Users who use an officially produced wheel will not need to make any changes. The minimum supported Rust version is 1.45.0." --- .../python-modules/cryptography/default.nix | 26 +++++++++++++------ .../python-modules/cryptography/vectors.nix | 2 +- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/pkgs/development/python-modules/cryptography/default.nix b/pkgs/development/python-modules/cryptography/default.nix index eb4eba0f587..d64f4a9792b 100644 --- a/pkgs/development/python-modules/cryptography/default.nix +++ b/pkgs/development/python-modules/cryptography/default.nix @@ -2,8 +2,8 @@ , buildPythonPackage , fetchPypi , fetchpatch -, isPy27 -, ipaddress +, rustPlatform +, setuptools-rust , openssl , cryptography_vectors , darwin @@ -13,27 +13,38 @@ , isPyPy , cffi , pytest +, pytest-subtests , pretend , iso8601 , pytz , hypothesis -, enum34 }: buildPythonPackage rec { pname = "cryptography"; - version = "3.3.2"; # Also update the hash in vectors.nix + version = "3.4.2"; # Also update the hash in vectors.nix src = fetchPypi { inherit pname version; - sha256 = "1vcvw4lkw1spiq322pm1256kail8nck6bbgpdxx3pqa905wd6q2s"; + sha256 = "1i1mx5y9hkyfi9jrrkcw804hmkcglxi6rmf7vin7jfnbr2bf4q64"; }; + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src; + sourceRoot = "${pname}-${version}/${cargoRoot}"; + name = "${pname}-${version}"; + hash = "sha256-PS562W4L1NimqDV2H0jl5vYhL08H9est/pbIxSdYVfo="; + }; + + cargoRoot = "src/rust"; + outputs = [ "out" "dev" ]; nativeBuildInputs = lib.optionals (!isPyPy) [ cffi - ]; + rustPlatform.cargoSetupHook + setuptools-rust + ] ++ (with rustPlatform; [ rust.cargo rust.rustc ]); buildInputs = [ openssl ] ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; @@ -42,8 +53,6 @@ buildPythonPackage rec { six ] ++ lib.optionals (!isPyPy) [ cffi - ] ++ lib.optionals isPy27 [ - ipaddress enum34 ]; checkInputs = [ @@ -52,6 +61,7 @@ buildPythonPackage rec { iso8601 pretend pytest + pytest-subtests pytz ]; diff --git a/pkgs/development/python-modules/cryptography/vectors.nix b/pkgs/development/python-modules/cryptography/vectors.nix index f9b7c525237..16151a84124 100644 --- a/pkgs/development/python-modules/cryptography/vectors.nix +++ b/pkgs/development/python-modules/cryptography/vectors.nix @@ -7,7 +7,7 @@ buildPythonPackage rec { src = fetchPypi { inherit pname version; - sha256 = "1yhaps0f3h2yjb6lmz953z1l1d84y9swk4k3gj9nqyk4vbx5m7cc"; + sha256 = "0i888rrfn7116lj7f2nr4amd2z45sk6866zizjfpsn5wh2713cls"; }; # No tests included From 198dd776352220e1f3e50c0714874425adaeaa77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Tue, 9 Feb 2021 13:46:32 +0100 Subject: [PATCH 4/4] doc: describe cargoSetupHook in the Rust section --- doc/languages-frameworks/rust.section.md | 98 ++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/doc/languages-frameworks/rust.section.md b/doc/languages-frameworks/rust.section.md index 8f6db28ab4d..e9850111376 100644 --- a/doc/languages-frameworks/rust.section.md +++ b/doc/languages-frameworks/rust.section.md @@ -237,6 +237,104 @@ rustPlatform.buildRustPackage rec { } ``` +## Compiling non-Rust packages that include Rust code + +Several non-Rust packages incorporate Rust code for performance- or +security-sensitive parts. `rustPlatform` exposes several functions and +hooks that can be used to integrate Cargo in non-Rust packages. + +### Vendoring of dependencies + +Since network access is not allowed in sandboxed builds, Rust crate +dependencies need to be retrieved using a fetcher. `rustPlatform` +provides the `fetchCargoTarball` fetcher, which vendors all +dependencies of a crate. This fetcher can be used jointly with +`cargoSetupHook` to vendor dependencies in derivations that do not use +`buildRustPackage`. + +In the following partial example, `fetchCargoTarball` and +`cargoSetupHook` are used to vendor dependencies in the Python +`tokenizers` derivation. The `tokenizers` Python package is in the +`source/bindings/python` directory of the project's source archive. We +use `fetchCargoTarball` to retrieve the dependencies specified in +`source/bidings/Cargo.{lock,toml}`. The resulting path is assigned to +the `cargoDeps` attribute, which is used by `cargoSetupHook` to +configure Cargo. + +```nix +{ fetchFromGitHub +, buildPythonPackage +, rustPlatform +, setuptools-rust +}: + +buildPythonPackage rec { + pname = "tokenizers"; + version = "0.10.0"; + + src = fetchFromGitHub { + owner = "huggingface"; + repo = pname; + rev = "python-v${version}"; + hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw="; + }; + + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src sourceRoot; + name = "${pname}-${version}"; + hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0="; + }; + + sourceRoot = "source/bindings/python"; + + nativeBuildInputs = [ setuptools-rust ] ++ (with rustPlatform; [ + cargoSetupHook + rust.cargo + rust.rustc + ]); + + # ... +} +``` + +In some projects, the Rust crate is not in the main source directory +of the projects. In such cases, the `cargoRoot` attribute can be used +to specify the crate's directory relative to `sourceRoot`. In the +following example, the crate is in `src/rust`, as specified in the +`cargoRoot` attribute. Note that we also need to specify the correct +path for `fetchCargoTarball`. + +```nix + +{ buildPythonPackage +, fetchPypi +, rustPlatform +, setuptools-rust +, openssl +}: + +buildPythonPackage rec { + pname = "cryptography"; + version = "3.4.2"; # Also update the hash in vectors.nix + + src = fetchPypi { + inherit pname version; + sha256 = "1i1mx5y9hkyfi9jrrkcw804hmkcglxi6rmf7vin7jfnbr2bf4q64"; + }; + + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src; + sourceRoot = "${pname}-${version}/${cargoRoot}"; + name = "${pname}-${version}"; + hash = "sha256-PS562W4L1NimqDV2H0jl5vYhL08H9est/pbIxSdYVfo="; + }; + + cargoRoot = "src/rust"; + + # ... +} +``` + ## Compiling Rust crates using Nix instead of Cargo ### Simple operation