diff --git a/pkgs/build-support/rust/default.nix b/pkgs/build-support/rust/default.nix index 2a890a0d232..ddf2fce3cae 100644 --- a/pkgs/build-support/rust/default.nix +++ b/pkgs/build-support/rust/default.nix @@ -11,8 +11,16 @@ let sha256 = depsSha256; }; + # The following is the directory name cargo creates when the registry index + # URL is file:///dev/null + # + # It's OK to use /dev/null as the URL because by the time we do this, cargo + # won't attempt to update the registry anymore, so the URL is more or less + # irrelevant + registryIndexDirName = "-ba82b75dd6681d6f"; + in stdenv.mkDerivation (args // { - inherit cargoDeps rustRegistry cargoUpdateHook; + inherit cargoDeps rustRegistry; patchRegistryDeps = ./patch-registry-deps; @@ -22,21 +30,26 @@ in stdenv.mkDerivation (args // { postUnpack = '' echo "Using cargo deps from $cargoDeps" - cp -r $cargoDeps deps + + cp -r "$cargoDeps" deps chmod +w deps -R - export CARGO_HOME=$(realpath deps) + cat < deps/config + [registry] + index = "file:///dev/null" + EOF echo "Using rust registry from $rustRegistry" + + ln -s "$rustRegistry" "deps/registry/index/${registryIndexDirName}" + + export CARGO_HOME="$(realpath deps)" + + # Retrieved the Cargo.lock file which we saved during the fetch + mv deps/Cargo.lock $sourceRoot/ + ( cd $sourceRoot - ln -s $rustRegistry ./cargo-rust-registry - - substituteInPlace Cargo.lock \ - --replace "registry+https://github.com/rust-lang/crates.io-index" \ - "registry+file:///proc/self/cwd/cargo-rust-registry" - - eval "$cargoUpdateHook" cargo fetch cargo clean @@ -46,10 +59,10 @@ in stdenv.mkDerivation (args // { prePatch = '' # Patch registry dependencies, using the scripts in $patchRegistryDeps ( - cd ../deps/registry/src/* - set -euo pipefail + cd ../deps/registry/src/* + for script in $patchRegistryDeps/*; do # Run in a subshell so that directory changes and shell options don't # affect any following commands diff --git a/pkgs/build-support/rust/fetch-cargo-deps b/pkgs/build-support/rust/fetch-cargo-deps index f0a21e673cd..c7799fb1b1f 100755 --- a/pkgs/build-support/rust/fetch-cargo-deps +++ b/pkgs/build-support/rust/fetch-cargo-deps @@ -12,52 +12,108 @@ echo "Fetching $src to $out" mkdir $out # Configure cargo to fetch from a local copy of the crates.io registry -# -# Unfortunately, `cargo fetch` will create an output directory named after a -# hash of the registry index URL. -# -# This makes things difficult for us because we don't want our output to change -# just because the path to the registry changed, otherwise we'd have to update -# all deps' SHA256 hashes whenever we simply update the registry to a newer -# commit. -# -# Also, since cargo doesn't seem to support relative URLs in the format -# file://../path, we use a hack to make sure the registry index path/URL is -# always the same: we'll create a symlink in the current working directory to -# the real registry path, while pointing cargo to the following fixed absolute -# path: -# -# file:///proc/self/cwd/symlink-name -ln -s $rustRegistry $src/cargo-rust-registry +echo "Using rust registry from $rustRegistry" -# TODO: replace /proc/self/cwd hack with normal relative path. Probably -# needs cargo fix. cat < $out/config [registry] -index = "file:///proc/self/cwd/cargo-rust-registry" +index = "file://$rustRegistry" EOF export CARGO_HOME=$out cd $src +if [[ ! -f Cargo.lock ]]; then + echo "ERROR: The Cargo.lock file doesn't exist" + echo + echo "Cargo.lock is needed to make sure that depsSha256 doesn't change" + echo "when the registry is updated." + + exit 1 +fi + +# We need to do the following string replacement so that 'cargo fetch' +# doesn't ignore the versions specified in Cargo.lock set +u substituteInPlace Cargo.lock \ --replace "registry+https://github.com/rust-lang/crates.io-index" \ - "registry+file:///proc/self/cwd/cargo-rust-registry" + "registry+file://$rustRegistry" set -u +# Do any possible 'cargo update -p --precise ' ad-hoc updates eval "$cargoUpdateHook" +# Do the fetch cargo fetch --verbose -# TODO: check that Cargo.lock exists, and hasn't changed -# TODO: this should be done by cargo itself +# Now that we have fetched everything, let's make the output deterministic -# Make it deterministic +# Cargo uses the following directory structure for fetched data, where +# $indexHash is a hash of the registry index URL: +# +# +# /config: +# +# Cargo config file. We'll delete this because it's not deterministic, +# and instead recreate it just before running 'cargo build'. +# +# /registry/cache/$indexHash/: +# +# This is where tarballs of registry package dependencies are kept +# We'll need to keep this, but make sure $indexHash is a fixed name. +# +# /registry/index/$indexHash/: +# +# A copy of the registry index is kept here. We can delete this, and +# instead, just before running 'cargo build', we'll symlink this +# directory to our static copy of the registry in the Nix store. +# +# /registry/src/$indexHash/{pkgName-pkgVersion}/: +# +# Here cargo keeps extracted sources of the cached tarballs. +# We'll just delete this because cargo will re-populate them from the +# tarballs. +# +# /git/db/{domain-hash}/: +# +# Here cargo keeps the `.git` directories of git dependencies. +# We'll need to keep these, but make them deterministic. +# +# /git/checkouts/{domain-hash}/{branchName}/: +# +# Here cargo keeps checked-out sources of the git dependencies. +# We can delete this, because cargo will re-populate them from the above +# `.git` directories. +# +# Let's start + +# Remove cargo config file, which points to the ever-changing registry +rm $out/config + +# Save the Cargo.lock file into the output, so that we don't have to do another +# 'cargo update' during the build (which would try to access the network) for +# any ad-hoc package updates (through $cargoUpdateHook). +# +# We need to replace the rustRegistry URL with something deterministic. +# Since the URL won't actually be accessed anymore, it's fine to use /dev/null. + +set +u +substituteInPlace Cargo.lock \ + --replace "registry+file://$rustRegistry" \ + "registry+file:///dev/null" +set -u +mv Cargo.lock $out/ + +# The following is the $indexHash cargo uses for the registry index when +# its URL is file:///dev/null, which is the registry index URL we use to make +# sure our output is deterministic. + +registryIndexDirName="-ba82b75dd6681d6f" +mv $out/registry/cache/* $out/registry/cache/$registryIndexDirName # The registry index changes all the time, so it's not deterministic -rm -rf $out/registry/index +# We'll symlink it before running 'cargo build' +rm -rf $out/registry/index/* # Make git DBs deterministic # TODO: test with git submodules @@ -83,31 +139,22 @@ rm -rf $out/registry/index git branch -rD "$branch" >&2 done - # Remove tags that don't point to any HEAD + # Remove all tags git tag | while read tag; do - rev="$(git rev-parse $tag)" - if [[ $revs != *" $rev"* ]]; then - git tag -d "$tag" >&2 - fi + git tag -d "$tag" >&2 done - # Remove branches that don't point to any HEAD + # Remove all local branches branchrefs=() eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)" for branchref in "${branchrefs[@]}"; do - echo "Examining $branchref" - rev="$(git rev-parse "$branchref")" - echo "Has rev $rev" - echo "List of revs: $revs" - if [[ $revs != *" $rev"* ]]; then - echo "Deleting $branchref" - git update-ref -d "$branchref" >&2 - fi + git update-ref -d "$branchref" >&2 done + # Create ad-hoc branches for the revs we need echo "$revs" | while read rev; do - echo "git branch b_$rev $rev" + echo "Creating git branch b_$rev $rev" git branch b_$rev $rev done diff --git a/pkgs/development/tools/build-managers/cargo/default.nix b/pkgs/development/tools/build-managers/cargo/default.nix index 0e70cb9232b..05e07b4afa2 100644 --- a/pkgs/development/tools/build-managers/cargo/default.nix +++ b/pkgs/development/tools/build-managers/cargo/default.nix @@ -23,7 +23,7 @@ buildRustPackage rec { cargo update -p threadpool --precise 0.1.4 ''; - depsSha256 = "12d2v4b85qabagrypvqiam2iybd4jwcg0sky0gqarfhjh2dhwfm6"; + depsSha256 = "1gj3mnjj17h5p0r1jcm3m3pm3p3l1rbfdz3l7v1cykng78dsabnq"; buildInputs = [ file curl pkgconfig python openssl cmake zlib ]; diff --git a/pkgs/development/tools/rust/racer/default.nix b/pkgs/development/tools/rust/racer/default.nix index baa29f356ed..4887736a859 100644 --- a/pkgs/development/tools/rust/racer/default.nix +++ b/pkgs/development/tools/rust/racer/default.nix @@ -11,7 +11,7 @@ buildRustPackage rec { sha256 = "0a768gvjry86l0xa5q0122iyq7zn2h9adfniglsgrbs4fan49xyn"; }; - depsSha256 = "0x1rq012k04ci18w5fll56jn011f1yyprs38pb3r223bag94ivsy"; + depsSha256 = "13rnxr5inrfnmg5mg2vap17fd2jp32g2a70pfi1sshi6vjr8kpap"; buildInputs = [ makeWrapper ];