Merge pull request #112500 from danieldk/cargo-setup-hook

This commit is contained in:
Jörg Thalheim 2021-02-11 09:41:40 +00:00 committed by GitHub
commit 674bfaf5e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 269 additions and 113 deletions

View File

@ -237,6 +237,104 @@ rustPlatform.buildRustPackage rec {
}
```
## Compiling non-Rust packages that include Rust code
Several non-Rust packages incorporate Rust code for performance- or
security-sensitive parts. `rustPlatform` exposes several functions and
hooks that can be used to integrate Cargo in non-Rust packages.
### Vendoring of dependencies
Since network access is not allowed in sandboxed builds, Rust crate
dependencies need to be retrieved using a fetcher. `rustPlatform`
provides the `fetchCargoTarball` fetcher, which vendors all
dependencies of a crate. This fetcher can be used jointly with
`cargoSetupHook` to vendor dependencies in derivations that do not use
`buildRustPackage`.
In the following partial example, `fetchCargoTarball` and
`cargoSetupHook` are used to vendor dependencies in the Python
`tokenizers` derivation. The `tokenizers` Python package is in the
`source/bindings/python` directory of the project's source archive. We
use `fetchCargoTarball` to retrieve the dependencies specified in
`source/bidings/Cargo.{lock,toml}`. The resulting path is assigned to
the `cargoDeps` attribute, which is used by `cargoSetupHook` to
configure Cargo.
```nix
{ fetchFromGitHub
, buildPythonPackage
, rustPlatform
, setuptools-rust
}:
buildPythonPackage rec {
pname = "tokenizers";
version = "0.10.0";
src = fetchFromGitHub {
owner = "huggingface";
repo = pname;
rev = "python-v${version}";
hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
};
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src sourceRoot;
name = "${pname}-${version}";
hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
};
sourceRoot = "source/bindings/python";
nativeBuildInputs = [ setuptools-rust ] ++ (with rustPlatform; [
cargoSetupHook
rust.cargo
rust.rustc
]);
# ...
}
```
In some projects, the Rust crate is not in the main source directory
of the projects. In such cases, the `cargoRoot` attribute can be used
to specify the crate's directory relative to `sourceRoot`. In the
following example, the crate is in `src/rust`, as specified in the
`cargoRoot` attribute. Note that we also need to specify the correct
path for `fetchCargoTarball`.
```nix
{ buildPythonPackage
, fetchPypi
, rustPlatform
, setuptools-rust
, openssl
}:
buildPythonPackage rec {
pname = "cryptography";
version = "3.4.2"; # Also update the hash in vectors.nix
src = fetchPypi {
inherit pname version;
sha256 = "1i1mx5y9hkyfi9jrrkcw804hmkcglxi6rmf7vin7jfnbr2bf4q64";
};
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
sourceRoot = "${pname}-${version}/${cargoRoot}";
name = "${pname}-${version}";
hash = "sha256-PS562W4L1NimqDV2H0jl5vYhL08H9est/pbIxSdYVfo=";
};
cargoRoot = "src/rust";
# ...
}
```
## Compiling Rust crates using Nix instead of Cargo
### Simple operation

View File

@ -3,7 +3,7 @@
, buildPackages
, cacert
, cargo
, diffutils
, cargoSetupHook
, fetchCargoTarball
, runCommandNoCC
, rustPlatform
@ -71,19 +71,6 @@ let
# against the src fixed-output derivation to check consistency.
validateCargoDeps = !(cargoHash == "" && cargoSha256 == "");
# Some cargo builds include build hooks that modify their own vendor
# dependencies. This copies the vendor directory into the build tree and makes
# it writable. If we're using a tarball, the unpackFile hook already handles
# this for us automatically.
setupVendorDir = if cargoVendorDir == null
then (''
unpackFile "$cargoDeps"
cargoDepsCopy=$(stripHash $cargoDeps)
'')
else ''
cargoDepsCopy="$sourceRoot/${cargoVendorDir}"
'';
targetIsJSON = lib.hasSuffix ".json" target;
useSysroot = targetIsJSON && !__internal_dontAddSysroot;
@ -106,11 +93,6 @@ let
releaseDir = "target/${shortTarget}/${buildType}";
tmpDir = "${releaseDir}-tmp";
# Specify the stdenv's `diff` by abspath to ensure that the user's build
# inputs do not cause us to find the wrong `diff`.
# The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available.
diff = "${diffutils.nativeDrv or diffutils}/bin/diff";
in
# Tests don't currently work for `no_std`, and all custom sysroots are currently built without `std`.
@ -124,7 +106,7 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u
patchRegistryDeps = ./patch-registry-deps;
nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo rustc ];
nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo cargoSetupHook rustc ];
buildInputs = buildInputs ++ lib.optional stdenv.hostPlatform.isMinGW windows.pthreads;
patches = cargoPatches ++ patches;
@ -135,72 +117,9 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u
postUnpack = ''
eval "$cargoDepsHook"
${setupVendorDir}
mkdir .cargo
config="$(pwd)/$cargoDepsCopy/.cargo/config";
if [[ ! -e $config ]]; then
config=${./fetchcargo-default-config.toml};
fi;
substitute $config .cargo/config \
--subst-var-by vendor "$(pwd)/$cargoDepsCopy"
cat >> .cargo/config <<'EOF'
[target."${rust.toRustTarget stdenv.buildPlatform}"]
"linker" = "${ccForBuild}"
${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) ''
[target."${shortTarget}"]
"linker" = "${ccForHost}"
${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633
lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) ''
"rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ]
''}
''}
EOF
export RUST_LOG=${logLevel}
'' + (args.postUnpack or "");
# After unpacking and applying patches, check that the Cargo.lock matches our
# src package. Note that we do this after the patchPhase, because the
# patchPhase may create the Cargo.lock if upstream has not shipped one.
postPatch = (args.postPatch or "") + lib.optionalString validateCargoDeps ''
cargoDepsLockfile=$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock
srcLockfile=$NIX_BUILD_TOP/$sourceRoot/Cargo.lock
echo "Validating consistency between $srcLockfile and $cargoDepsLockfile"
if ! ${diff} $srcLockfile $cargoDepsLockfile; then
# If the diff failed, first double-check that the file exists, so we can
# give a friendlier error msg.
if ! [ -e $srcLockfile ]; then
echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile"
echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build."
exit 1
fi
if ! [ -e $cargoDepsLockfile ]; then
echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile"
exit 1
fi
echo
echo "ERROR: cargoSha256 is out of date"
echo
echo "Cargo.lock is not the same in $cargoDepsCopy"
echo
echo "To fix the issue:"
echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value'
echo "2. Build the derivation and wait for it to fail with a hash mismatch"
echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field"
echo
exit 1
fi
'' + ''
unset cargoDepsCopy
'';
configurePhase = args.configurePhase or ''
runHook preConfigure
runHook postConfigure

View File

@ -0,0 +1,84 @@
cargoSetupPostUnpackHook() {
echo "Executing cargoSetupPostUnpackHook"
# Some cargo builds include build hooks that modify their own vendor
# dependencies. This copies the vendor directory into the build tree and makes
# it writable. If we're using a tarball, the unpackFile hook already handles
# this for us automatically.
if [ -z $cargoVendorDir ]; then
unpackFile "$cargoDeps"
export cargoDepsCopy=$(stripHash $cargoDeps)
else
cargoDepsCopy="$sourceRoot/${cargoRoot:+$cargoRoot/}${cargoVendorDir}"
fi
if [ ! -d .cargo ]; then
mkdir .cargo
fi
config="$(pwd)/$cargoDepsCopy/.cargo/config";
if [[ ! -e $config ]]; then
config=@defaultConfig@
fi;
tmp_config=$(mktemp)
substitute $config $tmp_config \
--subst-var-by vendor "$(pwd)/$cargoDepsCopy"
cat ${tmp_config} >> .cargo/config
cat >> .cargo/config <<'EOF'
@rustTarget@
EOF
echo "Finished cargoSetupPostUnpackHook"
}
# After unpacking and applying patches, check that the Cargo.lock matches our
# src package. Note that we do this after the patchPhase, because the
# patchPhase may create the Cargo.lock if upstream has not shipped one.
cargoSetupPostPatchHook() {
echo "Executing cargoSetupPostPatchHook"
cargoDepsLockfile="$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock"
srcLockfile="$NIX_BUILD_TOP/$sourceRoot/${cargoRoot:+$cargoRoot/}/Cargo.lock"
echo "Validating consistency between $srcLockfile and $cargoDepsLockfile"
if ! @diff@ $srcLockfile $cargoDepsLockfile; then
# If the diff failed, first double-check that the file exists, so we can
# give a friendlier error msg.
if ! [ -e $srcLockfile ]; then
echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile"
echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build."
exit 1
fi
if ! [ -e $cargoDepsLockfile ]; then
echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile"
exit 1
fi
echo
echo "ERROR: cargoSha256 is out of date"
echo
echo "Cargo.lock is not the same in $cargoDepsCopy"
echo
echo "To fix the issue:"
echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value'
echo "2. Build the derivation and wait for it to fail with a hash mismatch"
echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field"
echo
exit 1
fi
unset cargoDepsCopy
echo "Finished cargoSetupPostPatchHook"
}
postUnpackHooks+=(cargoSetupPostUnpackHook)
if [ -z ${cargoVendorDir-} ]; then
postPatchHooks+=(cargoSetupPostPatchHook)
fi

View File

@ -0,0 +1,49 @@
{ buildPackages
, callPackage
, diffutils
, lib
, makeSetupHook
, rust
, stdenv
, target ? rust.toRustTargetSpec stdenv.hostPlatform
}:
let
targetIsJSON = lib.hasSuffix ".json" target;
# see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168
# the "${}" is needed to transform the path into a /nix/store path before baseNameOf
shortTarget = if targetIsJSON then
(lib.removeSuffix ".json" (builtins.baseNameOf "${target}"))
else target;
ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
in {
cargoSetupHook = callPackage ({ }:
makeSetupHook {
name = "cargo-setup-hook.sh";
deps = [ ];
substitutions = {
defaultConfig = ../fetchcargo-default-config.toml;
# Specify the stdenv's `diff` by abspath to ensure that the user's build
# inputs do not cause us to find the wrong `diff`.
# The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available.
diff = "${diffutils.nativeDrv or diffutils}/bin/diff";
# Target platform
rustTarget = ''
[target."${rust.toRustTarget stdenv.buildPlatform}"]
"linker" = "${ccForBuild}"
${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) ''
[target."${shortTarget}"]
"linker" = "${ccForHost}"
${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633
lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) ''
"rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ]
''}
''}
'';
};
} ./cargo-setup-hook.sh) {};
}

View File

@ -12,7 +12,7 @@ rec {
};
buildRustPackage = callPackage ../../../build-support/rust {
inherit rustc cargo fetchCargoTarball;
inherit rustc cargo cargoSetupHook fetchCargoTarball;
};
rustcSrc = callPackage ./rust-src.nix {
@ -22,4 +22,7 @@ rec {
rustLibSrc = callPackage ./rust-lib-src.nix {
inherit rustc;
};
# Hooks
inherit (callPackage ../../../build-support/rust/hooks { }) cargoSetupHook;
}

View File

@ -2,8 +2,8 @@
, buildPythonPackage
, fetchPypi
, fetchpatch
, isPy27
, ipaddress
, rustPlatform
, setuptools-rust
, openssl
, cryptography_vectors
, darwin
@ -13,27 +13,38 @@
, isPyPy
, cffi
, pytest
, pytest-subtests
, pretend
, iso8601
, pytz
, hypothesis
, enum34
}:
buildPythonPackage rec {
pname = "cryptography";
version = "3.3.2"; # Also update the hash in vectors.nix
version = "3.4.2"; # Also update the hash in vectors.nix
src = fetchPypi {
inherit pname version;
sha256 = "1vcvw4lkw1spiq322pm1256kail8nck6bbgpdxx3pqa905wd6q2s";
sha256 = "1i1mx5y9hkyfi9jrrkcw804hmkcglxi6rmf7vin7jfnbr2bf4q64";
};
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src;
sourceRoot = "${pname}-${version}/${cargoRoot}";
name = "${pname}-${version}";
hash = "sha256-PS562W4L1NimqDV2H0jl5vYhL08H9est/pbIxSdYVfo=";
};
cargoRoot = "src/rust";
outputs = [ "out" "dev" ];
nativeBuildInputs = lib.optionals (!isPyPy) [
cffi
];
rustPlatform.cargoSetupHook
setuptools-rust
] ++ (with rustPlatform; [ rust.cargo rust.rustc ]);
buildInputs = [ openssl ]
++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
@ -42,8 +53,6 @@ buildPythonPackage rec {
six
] ++ lib.optionals (!isPyPy) [
cffi
] ++ lib.optionals isPy27 [
ipaddress enum34
];
checkInputs = [
@ -52,6 +61,7 @@ buildPythonPackage rec {
iso8601
pretend
pytest
pytest-subtests
pytz
];

View File

@ -7,7 +7,7 @@ buildPythonPackage rec {
src = fetchPypi {
inherit pname version;
sha256 = "1yhaps0f3h2yjb6lmz953z1l1d84y9swk4k3gj9nqyk4vbx5m7cc";
sha256 = "0i888rrfn7116lj7f2nr4amd2z45sk6866zizjfpsn5wh2713cls";
};
# No tests included

View File

@ -1,12 +1,10 @@
{ lib
, rustPlatform
, fetchFromGitHub
, fetchurl
, pipInstallHook
, buildPythonPackage
, rustPlatform
, setuptools-rust
, wheel
, numpy
, python
, datasets
, pytestCheckHook
, requests
@ -49,7 +47,7 @@ let
url = "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt";
sha256 = "09a754pm4djjglv3x5pkgwd6f79i2rq8ydg0f7c3q1wmwqdbba8f";
};
in rustPlatform.buildRustPackage rec {
in buildPythonPackage rec {
pname = "tokenizers";
version = "0.10.0";
@ -60,19 +58,22 @@ in rustPlatform.buildRustPackage rec {
hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
};
cargoSha256 = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
cargoDeps = rustPlatform.fetchCargoTarball {
inherit src sourceRoot;
name = "${pname}-${version}";
hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
};
sourceRoot = "source/bindings/python";
nativeBuildInputs = [
pipInstallHook
setuptools-rust
wheel
];
nativeBuildInputs = [ setuptools-rust ] ++ (with rustPlatform; [
cargoSetupHook
rust.cargo
rust.rustc
]);
propagatedBuildInputs = [
numpy
python
];
installCheckInputs = [
@ -99,14 +100,6 @@ in rustPlatform.buildRustPackage rec {
ln -s ${openaiMerges} openai-gpt-merges.txt )
'';
buildPhase = ''
${python.interpreter} setup.py bdist_wheel
'';
installPhase = ''
pipInstallPhase
'';
preCheck = ''
HOME=$TMPDIR
'';