fetch-cargo-deps: factor into fetchCargoDeps function

This commit is contained in:
Nikolay Amiantov 2016-02-25 14:05:14 +03:00
parent 34023d867d
commit 4f74a4aacb
2 changed files with 143 additions and 146 deletions

View File

@ -1,183 +1,177 @@
source $stdenv/setup fetchCargoDeps() {
src=$(realpath $1)
out=$(realpath $2)
set -euo pipefail echo "Fetching $src to $out"
src=$(realpath $1) mkdir $out
out=$(realpath $2)
echo "Fetching $src to $out" # Configure cargo to fetch from a local copy of the crates.io registry
mkdir $out echo "Using rust registry from $rustRegistry"
# Configure cargo to fetch from a local copy of the crates.io registry cat <<EOF > $out/config
echo "Using rust registry from $rustRegistry"
cat <<EOF > $out/config
[registry] [registry]
index = "file://$rustRegistry" index = "file://$rustRegistry"
EOF EOF
export CARGO_HOME=$out export CARGO_HOME=$out
cd $src cd $src
if [[ ! -f Cargo.lock ]]; then if [[ ! -f Cargo.lock ]]; then
echo echo
echo "ERROR: The Cargo.lock file doesn't exist" echo "ERROR: The Cargo.lock file doesn't exist"
echo echo
echo "Cargo.lock is needed to make sure that depsSha256 doesn't change" echo "Cargo.lock is needed to make sure that depsSha256 doesn't change"
echo "when the registry is updated." echo "when the registry is updated."
echo echo
exit 1 exit 1
fi fi
# We need to do the following string replacement so that 'cargo fetch' # We need to do the following string replacement so that 'cargo fetch'
# doesn't ignore the versions specified in Cargo.lock # doesn't ignore the versions specified in Cargo.lock
set +u substituteInPlace Cargo.lock \
substituteInPlace Cargo.lock \ --replace "registry+https://github.com/rust-lang/crates.io-index" \
--replace "registry+https://github.com/rust-lang/crates.io-index" \ "registry+file://$rustRegistry"
"registry+file://$rustRegistry"
set -u
# Do any possible 'cargo update -p <pkgName> --precise <version>' ad-hoc updates # Do any possible 'cargo update -p <pkgName> --precise <version>' ad-hoc updates
eval "$cargoUpdateHook" eval "$cargoUpdateHook"
# Do the fetch # Do the fetch
cargo fetch --verbose cargo fetch --verbose
# Now that we have fetched everything, let's make the output deterministic # Now that we have fetched everything, let's make the output deterministic
# Cargo uses the following directory structure for fetched data, where # Cargo uses the following directory structure for fetched data, where
# $indexHash is a hash of the registry index URL: # $indexHash is a hash of the registry index URL:
# #
# #
# /config: # /config:
# #
# Cargo config file. We'll delete this because it's not deterministic, # Cargo config file. We'll delete this because it's not deterministic,
# and instead recreate it just before running 'cargo build'. # and instead recreate it just before running 'cargo build'.
# #
# /registry/cache/$indexHash/: # /registry/cache/$indexHash/:
# #
# This is where tarballs of registry package dependencies are kept # This is where tarballs of registry package dependencies are kept
# We'll need to keep this, but make sure $indexHash is a fixed name. # We'll need to keep this, but make sure $indexHash is a fixed name.
# #
# /registry/index/$indexHash/: # /registry/index/$indexHash/:
# #
# A copy of the registry index is kept here. We can delete this, and # A copy of the registry index is kept here. We can delete this, and
# instead, just before running 'cargo build', we'll symlink this # instead, just before running 'cargo build', we'll symlink this
# directory to our static copy of the registry in the Nix store. # directory to our static copy of the registry in the Nix store.
# #
# /registry/src/$indexHash/{pkgName-pkgVersion}/: # /registry/src/$indexHash/{pkgName-pkgVersion}/:
# #
# Here cargo keeps extracted sources of the cached tarballs. # Here cargo keeps extracted sources of the cached tarballs.
# We'll just delete this because cargo will re-populate them from the # We'll just delete this because cargo will re-populate them from the
# tarballs. # tarballs.
# #
# /git/db/{domain-hash}/: # /git/db/{domain-hash}/:
# #
# Here cargo keeps the `.git` directories of git dependencies. # Here cargo keeps the `.git` directories of git dependencies.
# We'll need to keep these, but make them deterministic. # We'll need to keep these, but make them deterministic.
# #
# /git/checkouts/{domain-hash}/{branchName}/: # /git/checkouts/{domain-hash}/{branchName}/:
# #
# Here cargo keeps checked-out sources of the git dependencies. # Here cargo keeps checked-out sources of the git dependencies.
# We can delete this, because cargo will re-populate them from the above # We can delete this, because cargo will re-populate them from the above
# `.git` directories. # `.git` directories.
# #
# Let's start # Let's start
# Remove cargo config file, which points to the ever-changing registry # Remove cargo config file, which points to the ever-changing registry
rm $out/config rm $out/config
# Save the Cargo.lock file into the output, so that we don't have to do another # Save the Cargo.lock file into the output, so that we don't have to do another
# 'cargo update' during the build (which would try to access the network) for # 'cargo update' during the build (which would try to access the network) for
# any ad-hoc package updates (through $cargoUpdateHook). # any ad-hoc package updates (through $cargoUpdateHook).
# #
# We need to replace the rustRegistry URL with something deterministic. # We need to replace the rustRegistry URL with something deterministic.
# Since the URL won't actually be accessed anymore, it's fine to use /dev/null. # Since the URL won't actually be accessed anymore, it's fine to use /dev/null.
set +u substituteInPlace Cargo.lock \
substituteInPlace Cargo.lock \ --replace "registry+file://$rustRegistry" \
--replace "registry+file://$rustRegistry" \ "registry+file:///dev/null"
"registry+file:///dev/null" mv Cargo.lock $out/
set -u
mv Cargo.lock $out/
# Let's replace $indexHash with something more deterministic # Let's replace $indexHash with something more deterministic
mv $out/registry/cache/* $out/registry/cache/HASH mv $out/registry/cache/* $out/registry/cache/HASH
# The registry index changes all the time, so it's not deterministic # The registry index changes all the time, so it's not deterministic
# We'll symlink it before running 'cargo build' # We'll symlink it before running 'cargo build'
rm -rf $out/registry/index/* rm -rf $out/registry/index/*
# Make git DBs deterministic # Make git DBs deterministic
# TODO: test with git submodules # TODO: test with git submodules
[[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do [[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do
revs="" revs=""
cd "$out/git/checkouts/$name" cd "$out/git/checkouts/$name"
while read dir; do while read dir; do
# extract substring: [dir = "./xxx/yyy/.git"] => [branch = "xxx/yyy"] # extract substring: [dir = "./xxx/yyy/.git"] => [branch = "xxx/yyy"]
branch="${dir:2:$((${#dir}-7))}" branch="${dir:2:$((${#dir}-7))}"
cd "$out/git/checkouts/$name/$branch" cd "$out/git/checkouts/$name/$branch"
rev="$(git rev-parse HEAD)" rev="$(git rev-parse HEAD)"
revs="$revs $rev" revs="$revs $rev"
done < <(find . -type d -name .git -print) done < <(find . -type d -name .git -print)
echo "List of revs to keep for git db $name: $revs" echo "List of revs to keep for git db $name: $revs"
( (
# The following code was adapted from nix-prefetch-git # The following code was adapted from nix-prefetch-git
cd "$out/git/db/$name" cd "$out/git/db/$name"
export GIT_DIR=. export GIT_DIR=.
# Remove all remote branches # Remove all remote branches
git branch -r | while read branch; do git branch -r | while read branch; do
git branch -rD "$branch" >&2 git branch -rD "$branch" >&2
done done
# Remove all tags # Remove all tags
git tag | while read tag; do git tag | while read tag; do
git tag -d "$tag" >&2 git tag -d "$tag" >&2
done done
# Remove all local branches # Remove all local branches
branchrefs=() branchrefs=()
eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)" eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)"
for branchref in "${branchrefs[@]}"; do for branchref in "${branchrefs[@]}"; do
git update-ref -d "$branchref" >&2 git update-ref -d "$branchref" >&2
done done
# Create ad-hoc branches for the revs we need # Create ad-hoc branches for the revs we need
echo "$revs" | while read rev; do echo "$revs" | while read rev; do
echo "Creating git branch b_$rev $rev" echo "Creating git branch b_$rev $rev"
git branch b_$rev $rev git branch b_$rev $rev
done done
# Remove files that have timestamps or otherwise have non-deterministic # Remove files that have timestamps or otherwise have non-deterministic
# properties. # properties.
rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
# Do a full repack. Must run single-threaded, or else we lose determinism. # Do a full repack. Must run single-threaded, or else we lose determinism.
git config pack.threads 1 git config pack.threads 1
git repack -A -d -f git repack -A -d -f
rm -f config rm -f config
# Garbage collect unreferenced objects. # Garbage collect unreferenced objects.
git gc --prune=all git gc --prune=all
) )
done) done)
# Remove unneeded outputs # Remove unneeded outputs
[[ ! -d $out/registry/src ]] || rm -rf $out/registry/src [[ ! -d $out/registry/src ]] || rm -rf $out/registry/src
[[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts [[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts
# XXX: provide some debugging output to see find out why we are seeing # XXX: provide some debugging output to see find out why we are seeing
# sporadic hash mismatches # sporadic hash mismatches
find $out ! -type f find $out ! -type f
find $out -type f -exec sha256sum {} + find $out -type f -exec sha256sum {} +
}

View File

@ -9,8 +9,11 @@ stdenv.mkDerivation {
phases = "unpackPhase installPhase"; phases = "unpackPhase installPhase";
installPhase = '' installPhase = ''
source ${./fetch-cargo-deps}
export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt export SSL_CERT_FILE=${cacert}/etc/ssl/certs/ca-bundle.crt
bash ${./fetch-cargo-deps} . "$out"
fetchCargoDeps . "$out"
''; '';
outputHashAlgo = "sha256"; outputHashAlgo = "sha256";