Merge pull request #112500 from danieldk/cargo-setup-hook
This commit is contained in:
commit
674bfaf5e5
|
@ -237,6 +237,104 @@ rustPlatform.buildRustPackage rec {
|
|||
}
|
||||
```
|
||||
|
||||
## Compiling non-Rust packages that include Rust code
|
||||
|
||||
Several non-Rust packages incorporate Rust code for performance- or
|
||||
security-sensitive parts. `rustPlatform` exposes several functions and
|
||||
hooks that can be used to integrate Cargo in non-Rust packages.
|
||||
|
||||
### Vendoring of dependencies
|
||||
|
||||
Since network access is not allowed in sandboxed builds, Rust crate
|
||||
dependencies need to be retrieved using a fetcher. `rustPlatform`
|
||||
provides the `fetchCargoTarball` fetcher, which vendors all
|
||||
dependencies of a crate. This fetcher can be used jointly with
|
||||
`cargoSetupHook` to vendor dependencies in derivations that do not use
|
||||
`buildRustPackage`.
|
||||
|
||||
In the following partial example, `fetchCargoTarball` and
|
||||
`cargoSetupHook` are used to vendor dependencies in the Python
|
||||
`tokenizers` derivation. The `tokenizers` Python package is in the
|
||||
`source/bindings/python` directory of the project's source archive. We
|
||||
use `fetchCargoTarball` to retrieve the dependencies specified in
|
||||
`source/bidings/Cargo.{lock,toml}`. The resulting path is assigned to
|
||||
the `cargoDeps` attribute, which is used by `cargoSetupHook` to
|
||||
configure Cargo.
|
||||
|
||||
```nix
|
||||
{ fetchFromGitHub
|
||||
, buildPythonPackage
|
||||
, rustPlatform
|
||||
, setuptools-rust
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "tokenizers";
|
||||
version = "0.10.0";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "huggingface";
|
||||
repo = pname;
|
||||
rev = "python-v${version}";
|
||||
hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
|
||||
};
|
||||
|
||||
cargoDeps = rustPlatform.fetchCargoTarball {
|
||||
inherit src sourceRoot;
|
||||
name = "${pname}-${version}";
|
||||
hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
|
||||
};
|
||||
|
||||
sourceRoot = "source/bindings/python";
|
||||
|
||||
nativeBuildInputs = [ setuptools-rust ] ++ (with rustPlatform; [
|
||||
cargoSetupHook
|
||||
rust.cargo
|
||||
rust.rustc
|
||||
]);
|
||||
|
||||
# ...
|
||||
}
|
||||
```
|
||||
|
||||
In some projects, the Rust crate is not in the main source directory
|
||||
of the projects. In such cases, the `cargoRoot` attribute can be used
|
||||
to specify the crate's directory relative to `sourceRoot`. In the
|
||||
following example, the crate is in `src/rust`, as specified in the
|
||||
`cargoRoot` attribute. Note that we also need to specify the correct
|
||||
path for `fetchCargoTarball`.
|
||||
|
||||
```nix
|
||||
|
||||
{ buildPythonPackage
|
||||
, fetchPypi
|
||||
, rustPlatform
|
||||
, setuptools-rust
|
||||
, openssl
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "cryptography";
|
||||
version = "3.4.2"; # Also update the hash in vectors.nix
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "1i1mx5y9hkyfi9jrrkcw804hmkcglxi6rmf7vin7jfnbr2bf4q64";
|
||||
};
|
||||
|
||||
cargoDeps = rustPlatform.fetchCargoTarball {
|
||||
inherit src;
|
||||
sourceRoot = "${pname}-${version}/${cargoRoot}";
|
||||
name = "${pname}-${version}";
|
||||
hash = "sha256-PS562W4L1NimqDV2H0jl5vYhL08H9est/pbIxSdYVfo=";
|
||||
};
|
||||
|
||||
cargoRoot = "src/rust";
|
||||
|
||||
# ...
|
||||
}
|
||||
```
|
||||
|
||||
## Compiling Rust crates using Nix instead of Cargo
|
||||
|
||||
### Simple operation
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
, buildPackages
|
||||
, cacert
|
||||
, cargo
|
||||
, diffutils
|
||||
, cargoSetupHook
|
||||
, fetchCargoTarball
|
||||
, runCommandNoCC
|
||||
, rustPlatform
|
||||
|
@ -71,19 +71,6 @@ let
|
|||
# against the src fixed-output derivation to check consistency.
|
||||
validateCargoDeps = !(cargoHash == "" && cargoSha256 == "");
|
||||
|
||||
# Some cargo builds include build hooks that modify their own vendor
|
||||
# dependencies. This copies the vendor directory into the build tree and makes
|
||||
# it writable. If we're using a tarball, the unpackFile hook already handles
|
||||
# this for us automatically.
|
||||
setupVendorDir = if cargoVendorDir == null
|
||||
then (''
|
||||
unpackFile "$cargoDeps"
|
||||
cargoDepsCopy=$(stripHash $cargoDeps)
|
||||
'')
|
||||
else ''
|
||||
cargoDepsCopy="$sourceRoot/${cargoVendorDir}"
|
||||
'';
|
||||
|
||||
targetIsJSON = lib.hasSuffix ".json" target;
|
||||
useSysroot = targetIsJSON && !__internal_dontAddSysroot;
|
||||
|
||||
|
@ -106,11 +93,6 @@ let
|
|||
releaseDir = "target/${shortTarget}/${buildType}";
|
||||
tmpDir = "${releaseDir}-tmp";
|
||||
|
||||
# Specify the stdenv's `diff` by abspath to ensure that the user's build
|
||||
# inputs do not cause us to find the wrong `diff`.
|
||||
# The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available.
|
||||
diff = "${diffutils.nativeDrv or diffutils}/bin/diff";
|
||||
|
||||
in
|
||||
|
||||
# Tests don't currently work for `no_std`, and all custom sysroots are currently built without `std`.
|
||||
|
@ -124,7 +106,7 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u
|
|||
|
||||
patchRegistryDeps = ./patch-registry-deps;
|
||||
|
||||
nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo rustc ];
|
||||
nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo cargoSetupHook rustc ];
|
||||
buildInputs = buildInputs ++ lib.optional stdenv.hostPlatform.isMinGW windows.pthreads;
|
||||
|
||||
patches = cargoPatches ++ patches;
|
||||
|
@ -135,72 +117,9 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u
|
|||
postUnpack = ''
|
||||
eval "$cargoDepsHook"
|
||||
|
||||
${setupVendorDir}
|
||||
|
||||
mkdir .cargo
|
||||
config="$(pwd)/$cargoDepsCopy/.cargo/config";
|
||||
if [[ ! -e $config ]]; then
|
||||
config=${./fetchcargo-default-config.toml};
|
||||
fi;
|
||||
substitute $config .cargo/config \
|
||||
--subst-var-by vendor "$(pwd)/$cargoDepsCopy"
|
||||
|
||||
cat >> .cargo/config <<'EOF'
|
||||
[target."${rust.toRustTarget stdenv.buildPlatform}"]
|
||||
"linker" = "${ccForBuild}"
|
||||
${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) ''
|
||||
[target."${shortTarget}"]
|
||||
"linker" = "${ccForHost}"
|
||||
${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633
|
||||
lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) ''
|
||||
"rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ]
|
||||
''}
|
||||
''}
|
||||
EOF
|
||||
|
||||
export RUST_LOG=${logLevel}
|
||||
'' + (args.postUnpack or "");
|
||||
|
||||
# After unpacking and applying patches, check that the Cargo.lock matches our
|
||||
# src package. Note that we do this after the patchPhase, because the
|
||||
# patchPhase may create the Cargo.lock if upstream has not shipped one.
|
||||
postPatch = (args.postPatch or "") + lib.optionalString validateCargoDeps ''
|
||||
cargoDepsLockfile=$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock
|
||||
srcLockfile=$NIX_BUILD_TOP/$sourceRoot/Cargo.lock
|
||||
|
||||
echo "Validating consistency between $srcLockfile and $cargoDepsLockfile"
|
||||
if ! ${diff} $srcLockfile $cargoDepsLockfile; then
|
||||
|
||||
# If the diff failed, first double-check that the file exists, so we can
|
||||
# give a friendlier error msg.
|
||||
if ! [ -e $srcLockfile ]; then
|
||||
echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile"
|
||||
echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [ -e $cargoDepsLockfile ]; then
|
||||
echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "ERROR: cargoSha256 is out of date"
|
||||
echo
|
||||
echo "Cargo.lock is not the same in $cargoDepsCopy"
|
||||
echo
|
||||
echo "To fix the issue:"
|
||||
echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value'
|
||||
echo "2. Build the derivation and wait for it to fail with a hash mismatch"
|
||||
echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field"
|
||||
echo
|
||||
|
||||
exit 1
|
||||
fi
|
||||
'' + ''
|
||||
unset cargoDepsCopy
|
||||
'';
|
||||
|
||||
configurePhase = args.configurePhase or ''
|
||||
runHook preConfigure
|
||||
runHook postConfigure
|
||||
|
|
|
@ -0,0 +1,84 @@
|
|||
cargoSetupPostUnpackHook() {
|
||||
echo "Executing cargoSetupPostUnpackHook"
|
||||
|
||||
# Some cargo builds include build hooks that modify their own vendor
|
||||
# dependencies. This copies the vendor directory into the build tree and makes
|
||||
# it writable. If we're using a tarball, the unpackFile hook already handles
|
||||
# this for us automatically.
|
||||
if [ -z $cargoVendorDir ]; then
|
||||
unpackFile "$cargoDeps"
|
||||
export cargoDepsCopy=$(stripHash $cargoDeps)
|
||||
else
|
||||
cargoDepsCopy="$sourceRoot/${cargoRoot:+$cargoRoot/}${cargoVendorDir}"
|
||||
fi
|
||||
|
||||
if [ ! -d .cargo ]; then
|
||||
mkdir .cargo
|
||||
fi
|
||||
|
||||
config="$(pwd)/$cargoDepsCopy/.cargo/config";
|
||||
if [[ ! -e $config ]]; then
|
||||
config=@defaultConfig@
|
||||
fi;
|
||||
|
||||
tmp_config=$(mktemp)
|
||||
substitute $config $tmp_config \
|
||||
--subst-var-by vendor "$(pwd)/$cargoDepsCopy"
|
||||
cat ${tmp_config} >> .cargo/config
|
||||
|
||||
cat >> .cargo/config <<'EOF'
|
||||
@rustTarget@
|
||||
EOF
|
||||
|
||||
echo "Finished cargoSetupPostUnpackHook"
|
||||
}
|
||||
|
||||
# After unpacking and applying patches, check that the Cargo.lock matches our
|
||||
# src package. Note that we do this after the patchPhase, because the
|
||||
# patchPhase may create the Cargo.lock if upstream has not shipped one.
|
||||
cargoSetupPostPatchHook() {
|
||||
echo "Executing cargoSetupPostPatchHook"
|
||||
|
||||
cargoDepsLockfile="$NIX_BUILD_TOP/$cargoDepsCopy/Cargo.lock"
|
||||
srcLockfile="$NIX_BUILD_TOP/$sourceRoot/${cargoRoot:+$cargoRoot/}/Cargo.lock"
|
||||
|
||||
echo "Validating consistency between $srcLockfile and $cargoDepsLockfile"
|
||||
if ! @diff@ $srcLockfile $cargoDepsLockfile; then
|
||||
|
||||
# If the diff failed, first double-check that the file exists, so we can
|
||||
# give a friendlier error msg.
|
||||
if ! [ -e $srcLockfile ]; then
|
||||
echo "ERROR: Missing Cargo.lock from src. Expected to find it at: $srcLockfile"
|
||||
echo "Hint: You can use the cargoPatches attribute to add a Cargo.lock manually to the build."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [ -e $cargoDepsLockfile ]; then
|
||||
echo "ERROR: Missing lockfile from cargo vendor. Expected to find it at: $cargoDepsLockfile"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "ERROR: cargoSha256 is out of date"
|
||||
echo
|
||||
echo "Cargo.lock is not the same in $cargoDepsCopy"
|
||||
echo
|
||||
echo "To fix the issue:"
|
||||
echo '1. Use "0000000000000000000000000000000000000000000000000000" as the cargoSha256 value'
|
||||
echo "2. Build the derivation and wait for it to fail with a hash mismatch"
|
||||
echo "3. Copy the 'got: sha256:' value back into the cargoSha256 field"
|
||||
echo
|
||||
|
||||
exit 1
|
||||
fi
|
||||
|
||||
unset cargoDepsCopy
|
||||
|
||||
echo "Finished cargoSetupPostPatchHook"
|
||||
}
|
||||
|
||||
postUnpackHooks+=(cargoSetupPostUnpackHook)
|
||||
|
||||
if [ -z ${cargoVendorDir-} ]; then
|
||||
postPatchHooks+=(cargoSetupPostPatchHook)
|
||||
fi
|
|
@ -0,0 +1,49 @@
|
|||
{ buildPackages
|
||||
, callPackage
|
||||
, diffutils
|
||||
, lib
|
||||
, makeSetupHook
|
||||
, rust
|
||||
, stdenv
|
||||
, target ? rust.toRustTargetSpec stdenv.hostPlatform
|
||||
}:
|
||||
|
||||
let
|
||||
targetIsJSON = lib.hasSuffix ".json" target;
|
||||
|
||||
# see https://github.com/rust-lang/cargo/blob/964a16a28e234a3d397b2a7031d4ab4a428b1391/src/cargo/core/compiler/compile_kind.rs#L151-L168
|
||||
# the "${}" is needed to transform the path into a /nix/store path before baseNameOf
|
||||
shortTarget = if targetIsJSON then
|
||||
(lib.removeSuffix ".json" (builtins.baseNameOf "${target}"))
|
||||
else target;
|
||||
ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc";
|
||||
ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc";
|
||||
in {
|
||||
cargoSetupHook = callPackage ({ }:
|
||||
makeSetupHook {
|
||||
name = "cargo-setup-hook.sh";
|
||||
deps = [ ];
|
||||
substitutions = {
|
||||
defaultConfig = ../fetchcargo-default-config.toml;
|
||||
|
||||
# Specify the stdenv's `diff` by abspath to ensure that the user's build
|
||||
# inputs do not cause us to find the wrong `diff`.
|
||||
# The `.nativeDrv` stanza works like nativeBuildInputs and ensures cross-compiling has the right version available.
|
||||
diff = "${diffutils.nativeDrv or diffutils}/bin/diff";
|
||||
|
||||
# Target platform
|
||||
rustTarget = ''
|
||||
[target."${rust.toRustTarget stdenv.buildPlatform}"]
|
||||
"linker" = "${ccForBuild}"
|
||||
${lib.optionalString (stdenv.buildPlatform.config != stdenv.hostPlatform.config) ''
|
||||
[target."${shortTarget}"]
|
||||
"linker" = "${ccForHost}"
|
||||
${# https://github.com/rust-lang/rust/issues/46651#issuecomment-433611633
|
||||
lib.optionalString (stdenv.hostPlatform.isMusl && stdenv.hostPlatform.isAarch64) ''
|
||||
"rustflags" = [ "-C", "target-feature=+crt-static", "-C", "link-arg=-lgcc" ]
|
||||
''}
|
||||
''}
|
||||
'';
|
||||
};
|
||||
} ./cargo-setup-hook.sh) {};
|
||||
}
|
|
@ -12,7 +12,7 @@ rec {
|
|||
};
|
||||
|
||||
buildRustPackage = callPackage ../../../build-support/rust {
|
||||
inherit rustc cargo fetchCargoTarball;
|
||||
inherit rustc cargo cargoSetupHook fetchCargoTarball;
|
||||
};
|
||||
|
||||
rustcSrc = callPackage ./rust-src.nix {
|
||||
|
@ -22,4 +22,7 @@ rec {
|
|||
rustLibSrc = callPackage ./rust-lib-src.nix {
|
||||
inherit rustc;
|
||||
};
|
||||
|
||||
# Hooks
|
||||
inherit (callPackage ../../../build-support/rust/hooks { }) cargoSetupHook;
|
||||
}
|
||||
|
|
|
@ -2,8 +2,8 @@
|
|||
, buildPythonPackage
|
||||
, fetchPypi
|
||||
, fetchpatch
|
||||
, isPy27
|
||||
, ipaddress
|
||||
, rustPlatform
|
||||
, setuptools-rust
|
||||
, openssl
|
||||
, cryptography_vectors
|
||||
, darwin
|
||||
|
@ -13,27 +13,38 @@
|
|||
, isPyPy
|
||||
, cffi
|
||||
, pytest
|
||||
, pytest-subtests
|
||||
, pretend
|
||||
, iso8601
|
||||
, pytz
|
||||
, hypothesis
|
||||
, enum34
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "cryptography";
|
||||
version = "3.3.2"; # Also update the hash in vectors.nix
|
||||
version = "3.4.2"; # Also update the hash in vectors.nix
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "1vcvw4lkw1spiq322pm1256kail8nck6bbgpdxx3pqa905wd6q2s";
|
||||
sha256 = "1i1mx5y9hkyfi9jrrkcw804hmkcglxi6rmf7vin7jfnbr2bf4q64";
|
||||
};
|
||||
|
||||
cargoDeps = rustPlatform.fetchCargoTarball {
|
||||
inherit src;
|
||||
sourceRoot = "${pname}-${version}/${cargoRoot}";
|
||||
name = "${pname}-${version}";
|
||||
hash = "sha256-PS562W4L1NimqDV2H0jl5vYhL08H9est/pbIxSdYVfo=";
|
||||
};
|
||||
|
||||
cargoRoot = "src/rust";
|
||||
|
||||
outputs = [ "out" "dev" ];
|
||||
|
||||
nativeBuildInputs = lib.optionals (!isPyPy) [
|
||||
cffi
|
||||
];
|
||||
rustPlatform.cargoSetupHook
|
||||
setuptools-rust
|
||||
] ++ (with rustPlatform; [ rust.cargo rust.rustc ]);
|
||||
|
||||
buildInputs = [ openssl ]
|
||||
++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security;
|
||||
|
@ -42,8 +53,6 @@ buildPythonPackage rec {
|
|||
six
|
||||
] ++ lib.optionals (!isPyPy) [
|
||||
cffi
|
||||
] ++ lib.optionals isPy27 [
|
||||
ipaddress enum34
|
||||
];
|
||||
|
||||
checkInputs = [
|
||||
|
@ -52,6 +61,7 @@ buildPythonPackage rec {
|
|||
iso8601
|
||||
pretend
|
||||
pytest
|
||||
pytest-subtests
|
||||
pytz
|
||||
];
|
||||
|
||||
|
|
|
@ -7,7 +7,7 @@ buildPythonPackage rec {
|
|||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
sha256 = "1yhaps0f3h2yjb6lmz953z1l1d84y9swk4k3gj9nqyk4vbx5m7cc";
|
||||
sha256 = "0i888rrfn7116lj7f2nr4amd2z45sk6866zizjfpsn5wh2713cls";
|
||||
};
|
||||
|
||||
# No tests included
|
||||
|
|
|
@ -1,12 +1,10 @@
|
|||
{ lib
|
||||
, rustPlatform
|
||||
, fetchFromGitHub
|
||||
, fetchurl
|
||||
, pipInstallHook
|
||||
, buildPythonPackage
|
||||
, rustPlatform
|
||||
, setuptools-rust
|
||||
, wheel
|
||||
, numpy
|
||||
, python
|
||||
, datasets
|
||||
, pytestCheckHook
|
||||
, requests
|
||||
|
@ -49,7 +47,7 @@ let
|
|||
url = "https://s3.amazonaws.com/models.huggingface.co/bert/openai-gpt-merges.txt";
|
||||
sha256 = "09a754pm4djjglv3x5pkgwd6f79i2rq8ydg0f7c3q1wmwqdbba8f";
|
||||
};
|
||||
in rustPlatform.buildRustPackage rec {
|
||||
in buildPythonPackage rec {
|
||||
pname = "tokenizers";
|
||||
version = "0.10.0";
|
||||
|
||||
|
@ -60,19 +58,22 @@ in rustPlatform.buildRustPackage rec {
|
|||
hash = "sha256-rQ2hRV52naEf6PvRsWVCTN7B1oXAQGmnpJw4iIdhamw=";
|
||||
};
|
||||
|
||||
cargoSha256 = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
|
||||
cargoDeps = rustPlatform.fetchCargoTarball {
|
||||
inherit src sourceRoot;
|
||||
name = "${pname}-${version}";
|
||||
hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0=";
|
||||
};
|
||||
|
||||
sourceRoot = "source/bindings/python";
|
||||
|
||||
nativeBuildInputs = [
|
||||
pipInstallHook
|
||||
setuptools-rust
|
||||
wheel
|
||||
];
|
||||
nativeBuildInputs = [ setuptools-rust ] ++ (with rustPlatform; [
|
||||
cargoSetupHook
|
||||
rust.cargo
|
||||
rust.rustc
|
||||
]);
|
||||
|
||||
propagatedBuildInputs = [
|
||||
numpy
|
||||
python
|
||||
];
|
||||
|
||||
installCheckInputs = [
|
||||
|
@ -99,14 +100,6 @@ in rustPlatform.buildRustPackage rec {
|
|||
ln -s ${openaiMerges} openai-gpt-merges.txt )
|
||||
'';
|
||||
|
||||
buildPhase = ''
|
||||
${python.interpreter} setup.py bdist_wheel
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
pipInstallPhase
|
||||
'';
|
||||
|
||||
preCheck = ''
|
||||
HOME=$TMPDIR
|
||||
'';
|
||||
|
|
Loading…
Reference in New Issue