From a8efb2053fd9c1e2fea97699754b5071e94c1649 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Thu, 11 Feb 2021 17:32:47 +0100 Subject: [PATCH 1/8] buildRustPackage: factor out build phase to cargoBuildHook - API change: remove the `target` argument of `buildRustPackage`, the target should always be in sync with the C/C++ compiler that is used. - Gathering of binaries has moved from `buildPhase` to `installPhase`, this simplifies the hook and orders this functionality logically with the installation logic. --- pkgs/build-support/rust/default.nix | 61 ++++++------------- .../rust/hooks/cargo-build-hook.sh | 33 ++++++++++ pkgs/build-support/rust/hooks/default.nix | 20 +++++- .../compilers/rust/make-rust-platform.nix | 6 +- 4 files changed, 75 insertions(+), 45 deletions(-) create mode 100644 pkgs/build-support/rust/hooks/cargo-build-hook.sh diff --git a/pkgs/build-support/rust/default.nix b/pkgs/build-support/rust/default.nix index 6741b329a30..19ec71261be 100644 --- a/pkgs/build-support/rust/default.nix +++ b/pkgs/build-support/rust/default.nix @@ -3,6 +3,7 @@ , buildPackages , cacert , cargo +, cargoBuildHook , cargoSetupHook , fetchCargoTarball , runCommandNoCC @@ -37,7 +38,6 @@ , cargoBuildFlags ? [] , buildType ? "release" , meta ? {} -, target ? rust.toRustTargetSpec stdenv.hostPlatform , cargoVendorDir ? null , checkType ? buildType , depsExtraArgs ? {} @@ -71,6 +71,7 @@ let # against the src fixed-output derivation to check consistency. validateCargoDeps = !(cargoHash == "" && cargoSha256 == ""); + target = rust.toRustTargetSpec stdenv.hostPlatform; targetIsJSON = lib.hasSuffix ".json" target; useSysroot = targetIsJSON && !__internal_dontAddSysroot; @@ -86,10 +87,6 @@ let originalCargoToml = src + /Cargo.toml; # profile info is later extracted }; - ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc"; - cxxForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++"; - ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; - cxxForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++"; releaseDir = "target/${shortTarget}/${buildType}"; tmpDir = "${releaseDir}-tmp"; @@ -102,11 +99,17 @@ assert useSysroot -> !(args.doCheck or true); stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs useSysroot { RUSTFLAGS = "--sysroot ${sysroot} " + (args.RUSTFLAGS or ""); } // { - inherit cargoDeps; + inherit buildAndTestSubdir cargoDeps releaseDir tmpDir; + + cargoBuildFlags = lib.concatStringsSep " " cargoBuildFlags; + + cargoBuildType = "--${buildType}"; patchRegistryDeps = ./patch-registry-deps; - nativeBuildInputs = nativeBuildInputs ++ [ cacert git cargo cargoSetupHook rustc ]; + nativeBuildInputs = nativeBuildInputs ++ + [ cacert git cargo cargoBuildHook cargoSetupHook rustc ]; + buildInputs = buildInputs ++ lib.optional stdenv.hostPlatform.isMinGW windows.pthreads; patches = cargoPatches ++ patches; @@ -125,38 +128,6 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u runHook postConfigure ''; - buildPhase = with builtins; args.buildPhase or '' - ${lib.optionalString (buildAndTestSubdir != null) "pushd ${buildAndTestSubdir}"} - runHook preBuild - - ( - set -x - env \ - "CC_${rust.toRustTarget stdenv.buildPlatform}"="${ccForBuild}" \ - "CXX_${rust.toRustTarget stdenv.buildPlatform}"="${cxxForBuild}" \ - "CC_${rust.toRustTarget stdenv.hostPlatform}"="${ccForHost}" \ - "CXX_${rust.toRustTarget stdenv.hostPlatform}"="${cxxForHost}" \ - cargo build -j $NIX_BUILD_CORES \ - ${lib.optionalString (buildType == "release") "--release"} \ - --target ${target} \ - --frozen ${concatStringsSep " " cargoBuildFlags} - ) - - runHook postBuild - - ${lib.optionalString (buildAndTestSubdir != null) "popd"} - - # This needs to be done after postBuild: packages like `cargo` do a pushd/popd in - # the pre/postBuild-hooks that need to be taken into account before gathering - # all binaries to install. - mkdir -p $tmpDir - cp -r $releaseDir/* $tmpDir/ - bins=$(find $tmpDir \ - -maxdepth 1 \ - -type f \ - -executable ! \( -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \)) - ''; - checkPhase = args.checkPhase or (let argstr = "${lib.optionalString (checkType == "release") "--release"} --target ${target} --frozen"; threads = if cargoParallelTestThreads then "$NIX_BUILD_CORES" else "1"; @@ -173,11 +144,19 @@ stdenv.mkDerivation ((removeAttrs args ["depsExtraArgs"]) // lib.optionalAttrs u strictDeps = true; - inherit releaseDir tmpDir; - installPhase = args.installPhase or '' runHook preInstall + # This needs to be done after postBuild: packages like `cargo` do a pushd/popd in + # the pre/postBuild-hooks that need to be taken into account before gathering + # all binaries to install. + mkdir -p $tmpDir + cp -r $releaseDir/* $tmpDir/ + bins=$(find $tmpDir \ + -maxdepth 1 \ + -type f \ + -executable ! \( -regex ".*\.\(so.[0-9.]+\|so\|a\|dylib\)" \)) + # rename the output dir to a architecture independent one mapfile -t targets < <(find "$NIX_BUILD_TOP" -type d | grep '${tmpDir}$') for target in "''${targets[@]}"; do diff --git a/pkgs/build-support/rust/hooks/cargo-build-hook.sh b/pkgs/build-support/rust/hooks/cargo-build-hook.sh new file mode 100644 index 00000000000..55585233413 --- /dev/null +++ b/pkgs/build-support/rust/hooks/cargo-build-hook.sh @@ -0,0 +1,33 @@ +cargoBuildHook() { + echo "Executing cargoBuildHook" + + runHook preBuild + + if [ ! -z "${buildAndTestSubdir-}" ]; then + pushd "${buildAndTestSubdir}" + fi + + ( + set -x + env \ + "CC_@rustBuildPlatform@=@ccForBuild@" \ + "CXX_@rustBuildPlatform@=@cxxForBuild@" \ + "CC_@rustTargetPlatform@=@ccForHost@" \ + "CXX_@rustTargetPlatform@=@cxxForHost@" \ + cargo build -j $NIX_BUILD_CORES \ + --target @rustTargetPlatformSpec@ \ + --frozen \ + ${cargoBuildType} \ + ${cargoBuildFlags} + ) + + runHook postBuild + + if [ ! -z "${buildAndTestSubdir-}" ]; then + popd + fi + + echo "Finished cargoBuildHook" +} + +buildPhase=cargoBuildHook diff --git a/pkgs/build-support/rust/hooks/default.nix b/pkgs/build-support/rust/hooks/default.nix index dea749f9393..e33d5813a13 100644 --- a/pkgs/build-support/rust/hooks/default.nix +++ b/pkgs/build-support/rust/hooks/default.nix @@ -1,5 +1,6 @@ { buildPackages , callPackage +, cargo , diffutils , lib , makeSetupHook @@ -16,9 +17,24 @@ let shortTarget = if targetIsJSON then (lib.removeSuffix ".json" (builtins.baseNameOf "${target}")) else target; - ccForBuild="${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc"; - ccForHost="${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; + ccForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}cc"; + cxxForBuild = "${buildPackages.stdenv.cc}/bin/${buildPackages.stdenv.cc.targetPrefix}c++"; + ccForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}cc"; + cxxForHost = "${stdenv.cc}/bin/${stdenv.cc.targetPrefix}c++"; + rustBuildPlatform = rust.toRustTarget stdenv.buildPlatform; + rustTargetPlatform = rust.toRustTarget stdenv.hostPlatform; + rustTargetPlatformSpec = rust.toRustTargetSpec stdenv.hostPlatform; in { + cargoBuildHook = callPackage ({ }: + makeSetupHook { + name = "cargo-build-hook.sh"; + deps = [ cargo ]; + substitutions = { + inherit ccForBuild ccForHost cxxForBuild cxxForHost + rustBuildPlatform rustTargetPlatform rustTargetPlatformSpec; + }; + } ./cargo-build-hook.sh) {}; + cargoSetupHook = callPackage ({ }: makeSetupHook { name = "cargo-setup-hook.sh"; diff --git a/pkgs/development/compilers/rust/make-rust-platform.nix b/pkgs/development/compilers/rust/make-rust-platform.nix index c7c7a8cdb45..728f459d887 100644 --- a/pkgs/development/compilers/rust/make-rust-platform.nix +++ b/pkgs/development/compilers/rust/make-rust-platform.nix @@ -12,7 +12,7 @@ rec { }; buildRustPackage = callPackage ../../../build-support/rust { - inherit rustc cargo cargoSetupHook fetchCargoTarball; + inherit rustc cargo cargoBuildHook cargoSetupHook fetchCargoTarball; }; rustcSrc = callPackage ./rust-src.nix { @@ -24,5 +24,7 @@ rec { }; # Hooks - inherit (callPackage ../../../build-support/rust/hooks { }) cargoSetupHook; + inherit (callPackage ../../../build-support/rust/hooks { + inherit cargo; + }) cargoBuildHook cargoSetupHook; } From 7876d1c252af514423296473143964112ef88148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Thu, 11 Feb 2021 19:12:43 +0100 Subject: [PATCH 2/8] fetchCargoTarball: set default sourceRoot to the empty string This avoids that non-buildRustPackage derivations need to specify sourceRoot when the fetcher performs root stripping. --- pkgs/build-support/rust/fetchCargoTarball.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/build-support/rust/fetchCargoTarball.nix b/pkgs/build-support/rust/fetchCargoTarball.nix index c30e88d99b8..3b36554e707 100644 --- a/pkgs/build-support/rust/fetchCargoTarball.nix +++ b/pkgs/build-support/rust/fetchCargoTarball.nix @@ -21,7 +21,7 @@ in , src ? null , srcs ? [] , patches ? [] -, sourceRoot +, sourceRoot ? "" , hash ? "" , sha256 ? "" , cargoUpdateHook ? "" From 160cf87086694f6d2007e7d427a6dee089266e93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Thu, 11 Feb 2021 19:17:06 +0100 Subject: [PATCH 3/8] rustPlatform.maturinBuildHook: init This build hook can be used to build Python packages using maturin. --- pkgs/build-support/rust/hooks/default.nix | 11 ++++++ .../rust/hooks/maturin-build-hook.sh | 39 +++++++++++++++++++ .../compilers/rust/make-rust-platform.nix | 2 +- 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 pkgs/build-support/rust/hooks/maturin-build-hook.sh diff --git a/pkgs/build-support/rust/hooks/default.nix b/pkgs/build-support/rust/hooks/default.nix index e33d5813a13..d4b2cc15605 100644 --- a/pkgs/build-support/rust/hooks/default.nix +++ b/pkgs/build-support/rust/hooks/default.nix @@ -4,6 +4,7 @@ , diffutils , lib , makeSetupHook +, maturin , rust , stdenv , target ? rust.toRustTargetSpec stdenv.hostPlatform @@ -62,4 +63,14 @@ in { ''; }; } ./cargo-setup-hook.sh) {}; + + maturinBuildHook = callPackage ({ }: + makeSetupHook { + name = "maturin-build-hook.sh"; + deps = [ cargo maturin ]; + substitutions = { + inherit ccForBuild ccForHost cxxForBuild cxxForHost + rustBuildPlatform rustTargetPlatform rustTargetPlatformSpec; + }; + } ./maturin-build-hook.sh) {}; } diff --git a/pkgs/build-support/rust/hooks/maturin-build-hook.sh b/pkgs/build-support/rust/hooks/maturin-build-hook.sh new file mode 100644 index 00000000000..7e2599d9224 --- /dev/null +++ b/pkgs/build-support/rust/hooks/maturin-build-hook.sh @@ -0,0 +1,39 @@ +maturinBuildHook() { + echo "Executing maturinBuildHook" + + runHook preBuild + + if [ ! -z "${buildAndTestSubdir-}" ]; then + pushd "${buildAndTestSubdir}" + fi + + ( + set -x + env \ + "CC_@rustBuildPlatform@=@ccForBuild@" \ + "CXX_@rustBuildPlatform@=@cxxForBuild@" \ + "CC_@rustTargetPlatform@=@ccForHost@" \ + "CXX_@rustTargetPlatform@=@cxxForHost@" \ + maturin build \ + --cargo-extra-args="-j $NIX_BUILD_CORES --frozen" \ + --target @rustTargetPlatformSpec@ \ + --manylinux off \ + --strip \ + --release \ + ${maturinBuildFlags-} + ) + + runHook postBuild + + if [ ! -z "${buildAndTestSubdir-}" ]; then + popd + fi + + # Move the wheel to dist/ so that regular Python tooling can find it. + mkdir -p dist + mv target/wheels/*.whl dist/ + + echo "Finished maturinBuildHook" +} + +buildPhase=maturinBuildHook diff --git a/pkgs/development/compilers/rust/make-rust-platform.nix b/pkgs/development/compilers/rust/make-rust-platform.nix index 728f459d887..1ea97b48c44 100644 --- a/pkgs/development/compilers/rust/make-rust-platform.nix +++ b/pkgs/development/compilers/rust/make-rust-platform.nix @@ -26,5 +26,5 @@ rec { # Hooks inherit (callPackage ../../../build-support/rust/hooks { inherit cargo; - }) cargoBuildHook cargoSetupHook; + }) cargoBuildHook cargoSetupHook maturinBuildHook; } From a24453fe94f19d7e7d5fca6d5d3dc6aac635019b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Thu, 11 Feb 2021 19:18:02 +0100 Subject: [PATCH 4/8] python3Packages.retworkx: switch to buildPythonPackage + cargo hooks --- .../python-modules/retworkx/default.nix | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/pkgs/development/python-modules/retworkx/default.nix b/pkgs/development/python-modules/retworkx/default.nix index 02a3ef64c57..8cce139facb 100644 --- a/pkgs/development/python-modules/retworkx/default.nix +++ b/pkgs/development/python-modules/retworkx/default.nix @@ -1,16 +1,14 @@ { lib +, buildPythonPackage , rustPlatform -, python , fetchFromGitHub -, pipInstallHook -, maturin -, pip + # Check inputs , pytestCheckHook , numpy }: -rustPlatform.buildRustPackage rec { +buildPythonPackage rec { pname = "retworkx"; version = "0.6.0"; @@ -21,11 +19,15 @@ rustPlatform.buildRustPackage rec { sha256 = "11n30ldg3y3y6qxg3hbj837pnbwjkqw3nxq6frds647mmmprrd20"; }; - cargoSha256 = "1vg4yf0k6yypqf9z46zz818mz7fdrgxj7zl6zjf7pnm2r8mq3qw5"; + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src; + name = "${pname}-${version}"; + hash = "sha256-heOBK8qi2nuc/Ib+I/vLzZ1fUUD/G/KTw9d7M4Hz5O0="; + }; - propagatedBuildInputs = [ python ]; + format = "pyproject"; - nativeBuildInputs = [ pipInstallHook maturin pip ]; + nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook ]; # Needed b/c need to check AFTER python wheel is installed (using Rust Build, not buildPythonPackage) doCheck = false; @@ -33,17 +35,6 @@ rustPlatform.buildRustPackage rec { installCheckInputs = [ pytestCheckHook numpy ]; - buildPhase = '' - runHook preBuild - maturin build --release --manylinux off --strip - runHook postBuild - ''; - - installPhase = '' - install -Dm644 -t dist target/wheels/*.whl - pipInstallPhase - ''; - preCheck = '' export TESTDIR=$(mktemp -d) cp -r tests/ $TESTDIR From 73bf3bdd893323d3c0424fc2d2b25ce75db6c15d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Thu, 11 Feb 2021 19:30:30 +0100 Subject: [PATCH 5/8] python3Packages.wasmer: switch to buildPythonPackage + cargo hooks --- .../python-modules/wasmer/default.nix | 69 ++++++------------- 1 file changed, 20 insertions(+), 49 deletions(-) diff --git a/pkgs/development/python-modules/wasmer/default.nix b/pkgs/development/python-modules/wasmer/default.nix index 62c8a2a06b7..28c9b8b5c16 100644 --- a/pkgs/development/python-modules/wasmer/default.nix +++ b/pkgs/development/python-modules/wasmer/default.nix @@ -1,63 +1,34 @@ { lib , rustPlatform , fetchFromGitHub -, maturin , buildPythonPackage -, isPy38 -, python }: let pname = "wasmer"; version = "1.0.0"; - - wheel = rustPlatform.buildRustPackage rec { - inherit pname version; - - src = fetchFromGitHub { - owner = "wasmerio"; - repo = "wasmer-python"; - rev = version; - hash = "sha256-I1GfjLaPYMIHKh2m/5IQepUsJNiVUEJg49wyuuzUYtY="; - }; - - cargoHash = "sha256-txOOia1C4W+nsXuXp4EytEn82CFfSmiOYwRLC4WPImc="; - - nativeBuildInputs = [ maturin python ]; - - preBuild = '' - cd packages/api - ''; - - buildPhase = '' - runHook preBuild - maturin build --release --manylinux off --strip - runHook postBuild - ''; - - postBuild = '' - cd ../.. - ''; - - doCheck = false; - - installPhase = '' - runHook preInstall - install -Dm644 -t $out target/wheels/*.whl - runHook postInstall - ''; - }; - -in -buildPythonPackage rec { +in buildPythonPackage rec { inherit pname version; - format = "wheel"; - src = wheel; + src = fetchFromGitHub { + owner = "wasmerio"; + repo = "wasmer-python"; + rev = version; + hash = "sha256-I1GfjLaPYMIHKh2m/5IQepUsJNiVUEJg49wyuuzUYtY="; + }; - unpackPhase = '' - mkdir -p dist - cp $src/*.whl dist - ''; + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src; + name = "${pname}-${version}"; + hash = "sha256-txOOia1C4W+nsXuXp4EytEn82CFfSmiOYwRLC4WPImc="; + }; + + format = "pyproject"; + + nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook ]; + + buildAndTestSubdir = "packages/api"; + + doCheck = false; pythonImportsCheck = [ "wasmer" ]; From 15c32ae8b1cceaead1194ae1bc0b732212955b66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Thu, 11 Feb 2021 19:35:54 +0100 Subject: [PATCH 6/8] python3Packages.adblock: switch to buildPythonPackage + cargo hooks --- .../python-modules/adblock/default.nix | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/pkgs/development/python-modules/adblock/default.nix b/pkgs/development/python-modules/adblock/default.nix index de6d72a61ec..c608f48fdf9 100644 --- a/pkgs/development/python-modules/adblock/default.nix +++ b/pkgs/development/python-modules/adblock/default.nix @@ -1,10 +1,9 @@ { stdenv , lib -, rustPlatform , fetchFromGitHub -, pipInstallHook +, buildPythonPackage +, rustPlatform , pythonImportsCheckHook -, maturin , pkg-config , openssl , publicsuffix-list @@ -13,7 +12,7 @@ , Security }: -rustPlatform.buildRustPackage rec { +buildPythonPackage rec { pname = "adblock"; version = "0.4.0"; disabled = isPy27; @@ -25,33 +24,27 @@ rustPlatform.buildRustPackage rec { rev = version; sha256 = "10d6ks2fyzbizq3kb69q478idj0h86k6ygjb6wl3zq3mf65ma4zg"; }; + + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src; + name = "${pname}-${version}"; + hash = "sha256-gEFmj3/KvhvvsOK2nX2L1RUD4Wfp3nYzEzVnQZIsIDY="; + }; + format = "pyproject"; - cargoSha256 = "0di05j942rrm2crpdpp9czhh65fmidyrvdp2n3pipgnagy7nchc0"; - - nativeBuildInputs = [ pipInstallHook maturin pkg-config pythonImportsCheckHook ]; + nativeBuildInputs = [ pkg-config pythonImportsCheckHook ] + ++ (with rustPlatform; [ cargoSetupHook maturinBuildHook ]); buildInputs = [ openssl ] ++ lib.optionals stdenv.isDarwin [ CoreFoundation Security ]; PSL_PATH = "${publicsuffix-list}/share/publicsuffix/public_suffix_list.dat"; - buildPhase = '' - runHook preBuild - maturin build --release --manylinux off --strip - runHook postBuild - ''; - # There are no rust tests doCheck = false; - pythonImportsCheck = [ "adblock" ]; - installPhase = '' - runHook preInstall - install -Dm644 -t dist target/wheels/*.whl - pipInstallPhase - runHook postInstall - ''; + pythonImportsCheck = [ "adblock" ]; passthru.meta = with lib; { description = "Python wrapper for Brave's adblocking library, which is written in Rust"; From 17bc42353495e1fb6285997e4b2659a221d3dd33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Thu, 11 Feb 2021 19:36:43 +0100 Subject: [PATCH 7/8] python3Packages.johnnycanencrypt: switch to buildPythonPackage + cargo hooks --- .../johnnycanencrypt/default.nix | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/pkgs/development/python-modules/johnnycanencrypt/default.nix b/pkgs/development/python-modules/johnnycanencrypt/default.nix index 77789fa44db..a0c82902d3e 100644 --- a/pkgs/development/python-modules/johnnycanencrypt/default.nix +++ b/pkgs/development/python-modules/johnnycanencrypt/default.nix @@ -1,14 +1,13 @@ { lib , stdenv -, rustPlatform , fetchFromGitHub -, pipInstallHook +, buildPythonPackage +, rustPlatform , llvmPackages , pkg-config , maturin , pcsclite , nettle -, python , requests , vcrpy , numpy @@ -17,7 +16,7 @@ , PCSC }: -rustPlatform.buildRustPackage rec { +buildPythonPackage rec { pname = "johnnycanencrypt"; version = "0.5.0"; disabled = pythonOlder "3.7"; @@ -28,7 +27,16 @@ rustPlatform.buildRustPackage rec { rev = "v${version}"; sha256 = "192wfrlyylrpzq70yki421mi1smk8q2cyki2a1d03q7h6apib3j4"; }; - cargoPatches = [ ./Cargo.lock.patch ]; + + cargoDeps = rustPlatform.fetchCargoTarball { + inherit patches src; + name = "${pname}-${version}"; + hash = "sha256-2XhXCKyXVlFgbcOoMy/A5ajiIVxBii56YeI29mO720U="; + }; + + format = "pyproject"; + + patches = [ ./Cargo.lock.patch ]; cargoSha256 = "0ifvpdizcdp2c5x2x2j1bhhy5a75q0pk7a63dmh52mlpmh45fy6r"; @@ -42,10 +50,10 @@ rustPlatform.buildRustPackage rec { nativeBuildInputs = [ llvmPackages.clang pkg-config - python - maturin - pipInstallHook - ]; + ] ++ (with rustPlatform; [ + cargoSetupHook + maturinBuildHook + ]); buildInputs = [ pcsclite @@ -67,17 +75,6 @@ rustPlatform.buildRustPackage rec { sed '/project-url = /d' -i Cargo.toml ''; - buildPhase = '' - runHook preBuild - maturin build --release --manylinux off --strip --cargo-extra-args="-j $NIX_BUILD_CORES --frozen" - runHook postBuild - ''; - - installPhase = '' - install -Dm644 -t dist target/wheels/*.whl - pipInstallPhase - ''; - preCheck = '' export TESTDIR=$(mktemp -d) cp -r tests/ $TESTDIR From dbc8633daf5fd40bd673a74b6df9f140416a5454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Fri, 12 Feb 2021 08:35:50 +0100 Subject: [PATCH 8/8] doc: describe cargoBuildHook and maturinBuildHook in the Rust section --- doc/languages-frameworks/rust.section.md | 116 ++++++++++++++++++++--- 1 file changed, 102 insertions(+), 14 deletions(-) diff --git a/doc/languages-frameworks/rust.section.md b/doc/languages-frameworks/rust.section.md index e9850111376..c53818f8157 100644 --- a/doc/languages-frameworks/rust.section.md +++ b/doc/languages-frameworks/rust.section.md @@ -248,18 +248,65 @@ hooks that can be used to integrate Cargo in non-Rust packages. Since network access is not allowed in sandboxed builds, Rust crate dependencies need to be retrieved using a fetcher. `rustPlatform` provides the `fetchCargoTarball` fetcher, which vendors all -dependencies of a crate. This fetcher can be used jointly with -`cargoSetupHook` to vendor dependencies in derivations that do not use -`buildRustPackage`. +dependencies of a crate. For example, given a source path `src` +containing `Cargo.toml` and `Cargo.lock`, `fetchCargoTarball` +can be used as follows: -In the following partial example, `fetchCargoTarball` and -`cargoSetupHook` are used to vendor dependencies in the Python -`tokenizers` derivation. The `tokenizers` Python package is in the -`source/bindings/python` directory of the project's source archive. We -use `fetchCargoTarball` to retrieve the dependencies specified in -`source/bidings/Cargo.{lock,toml}`. The resulting path is assigned to -the `cargoDeps` attribute, which is used by `cargoSetupHook` to -configure Cargo. +```nix +cargoDeps = rustPlatform.fetchCargoTarball { + inherit src; + hash = "sha256-BoHIN/519Top1NUBjpB/oEMqi86Omt3zTQcXFWqrek0="; +}; +``` + +The `src` attribute is required, as well as a hash specified through +one of the `sha256` or `hash` attributes. The following optional +attributes can also be used: + +* `name`: the name that is used for the dependencies tarball. If + `name` is not specified, then the name `cargo-deps` will be used. +* `sourceRoot`: when the `Cargo.lock`/`Cargo.toml` are in a + subdirectory, `sourceRoot` specifies the relative path to these + files. +* `patches`: patches to apply before vendoring. This is useful when + the `Cargo.lock`/`Cargo.toml` files need to be patched before + vendoring. + +### Hooks + +`rustPlatform` provides the following hooks to automate Cargo builds: + +* `cargoSetupHook`: configure Cargo to use depenencies vendored + through `fetchCargoTarball`. This hook uses the `cargoDeps` + environment variable to find the vendored dependencies. If a project + already vendors its dependencies, the variable `cargoVendorDir` can + be used instead. When the `Cargo.toml`/`Cargo.lock` files are not in + `sourceRoot`, then the optional `cargoRoot` is used to specify the + Cargo root directory relative to `sourceRoot`. +* `cargoBuildHook`: use Cargo to build a crate. If the crate to be + built is a crate in e.g. a Cargo workspace, the relative path to the + crate to build can be set through the optional `buildAndTestSubdir` + environment variable. Additional Cargo build flags can be passed + through `cargoBuildFlags`. +* `maturinBuildHook`: use [Maturin](https://github.com/PyO3/maturin) + to build a Python wheel. Similar to `cargoBuildHook`, the optional + variable `buildAndTestSubdir` can be used to build a crate in a + Cargo workspace. Additional maturin flags can be passed through + `maturinBuildFlags`. + +### Examples + +#### Python package using `setuptools-rust` + +For Python packages using `setuptools-rust`, you can use +`fetchCargoTarball` and `cargoSetupHook` to retrieve and set up Cargo +dependencies. The build itself is then performed by +`buildPythonPackage`. + +The following example outlines how the `tokenizers` Python package is +built. Since the Python package is in the `source/bindings/python` +directory of the *tokenizers* project's source archive, we use +`sourceRoot` to point the tooling to this directory: ```nix { fetchFromGitHub @@ -297,9 +344,9 @@ buildPythonPackage rec { } ``` -In some projects, the Rust crate is not in the main source directory -of the projects. In such cases, the `cargoRoot` attribute can be used -to specify the crate's directory relative to `sourceRoot`. In the +In some projects, the Rust crate is not in the main Python source +directory. In such cases, the `cargoRoot` attribute can be used to +specify the crate's directory relative to `sourceRoot`. In the following example, the crate is in `src/rust`, as specified in the `cargoRoot` attribute. Note that we also need to specify the correct path for `fetchCargoTarball`. @@ -335,6 +382,47 @@ buildPythonPackage rec { } ``` +#### Python package using `maturin` + +Python packages that use [Maturin](https://github.com/PyO3/maturin) +can be built with `fetchCargoTarball`, `cargoSetupHook`, and +`maturinBuildHook`. For example, the following (partial) derivation +builds the `retworkx` Python package. `fetchCargoTarball` and +`cargoSetupHook` are used to fetch and set up the crate dependencies. +`maturinBuildHook` is used to perform the build. + +```nix +{ lib +, buildPythonPackage +, rustPlatform +, fetchFromGitHub +}: + +buildPythonPackage rec { + pname = "retworkx"; + version = "0.6.0"; + + src = fetchFromGitHub { + owner = "Qiskit"; + repo = "retworkx"; + rev = version; + sha256 = "11n30ldg3y3y6qxg3hbj837pnbwjkqw3nxq6frds647mmmprrd20"; + }; + + cargoDeps = rustPlatform.fetchCargoTarball { + inherit src; + name = "${pname}-${version}"; + hash = "sha256-heOBK8qi2nuc/Ib+I/vLzZ1fUUD/G/KTw9d7M4Hz5O0="; + }; + + format = "pyproject"; + + nativeBuildInputs = with rustPlatform; [ cargoSetupHook maturinBuildHook ]; + + # ... +} +``` + ## Compiling Rust crates using Nix instead of Cargo ### Simple operation