Add support for building cargo'ed Rust programs

pkgs/build-support/fetchgit/nix-prefetch-git

@@ -224,7 +224,7 @@ make_deterministic_repo(){
         fi
     done
 
-    # Do a full repack. Must run single-threaded, or else we loose determinism.
+    # Do a full repack. Must run single-threaded, or else we lose determinism.
     git config pack.threads 1
     git repack -A -d -f
     rm -f .git/config

pkgs/build-support/rust/default.nix

@@ -0,0 +1,52 @@
+{ stdenv, cacert, git, rustc, cargo, rustRegistry }:
+{ name, src, depsSha256, buildInputs ? [], ... } @ args:
+
+let
+  fetchDeps = import ./fetchcargo.nix {
+    inherit stdenv cacert git rustc cargo rustRegistry;
+  };
+
+  cargoDeps = fetchDeps {
+    inherit name src;
+    sha256 = depsSha256;
+  };
+
+in stdenv.mkDerivation (args // {
+  inherit cargoDeps rustRegistry;
+
+  buildInputs = [ git cargo rustc ] ++ buildInputs;
+
+  configurePhase = args.configurePhase or "true";
+
+  postUnpack = ''
+    echo "Using rust registry from $rustRegistry"
+    (
+        cd $sourceRoot
+        ln -s $rustRegistry ./cargo-rust-registry
+        cargo clean
+        cargo fetch
+    )
+  '' + (args.postUnpack or "");
+
+  # TODO: Probably not the best way to do this, but it should work for now
+  prePatch = ''
+    for dir in ../deps/registry/src/*/pkg-config-*; do
+        [ -d "$dir" ] || continue
+
+        substituteInPlace "$dir/src/lib.rs" \
+            --replace '"/usr"' '"/nix/store/"'
+    done
+  '' + (args.prePatch or "");
+
+  buildPhase = args.buildPhase or ''
+    echo "Running cargo build"
+    cargo build --release
+  '';
+
+  installPhase = args.installPhase or ''
+    mkdir -p $out/bin
+    for f in $(find target/release -maxdepth 1 -type f); do
+      cp $f $out/bin
+    done;
+  '';
+})

pkgs/build-support/rust/fetch-builder.sh

@@ -0,0 +1,17 @@
+source $stdenv/setup
+
+# cargo-fetch needs to write to Cargo.lock, even to do nothing. We
+# create a fake checkout with symlinks and and editable Cargo.lock.
+mkdir copy
+cd copy
+for f in $(ls $src); do
+  ln -s $src/"$f" .
+done
+rm Cargo.lock
+cp $src/Cargo.lock .
+chmod +w Cargo.lock
+
+$fetcher . $out
+
+cd ..
+rm -rf copy

pkgs/build-support/rust/fetch-cargo-deps

@@ -0,0 +1,119 @@
+#! /bin/sh -eu
+
+set -o pipefail
+
+src=$(realpath $1)
+out=$(realpath $2)
+
+echo "Fetching $src to $out"
+
+mkdir $out
+
+# Configure cargo to fetch from a local copy of the crates.io registry
+#
+# Unfortunately, `cargo fetch` will create an output directory named after a
+# hash of the registry index URL.
+#
+# This makes things difficult for us because we don't want our output to change
+# just because the path to the registry changed, otherwise we'd have to update
+# all deps' SHA256 hashes whenever we simply update the registry to a newer
+# commit.
+#
+# Also, since cargo doesn't seem to support relative URLs in the format
+# file://../path, we use a hack to make sure the registry index path/URL is
+# always the same: we'll create a symlink in the current working directory to
+# the real registry path, while pointing cargo to the following fixed absolute
+# path:
+#
+# file:///proc/self/cwd/symlink-name
+
+ln -s $rustRegistry $src/cargo-rust-registry
+
+# TODO: replace /proc/self/cwd hack with normal relative path. Probably
+# needs cargo fix.
+cat <<EOF > $out/config
+[registry]
+index = "file:///proc/self/cwd/cargo-rust-registry"
+EOF
+
+export CARGO_HOME=$out
+cd $src
+cargo fetch --verbose
+
+# TODO: check that Cargo.lock exists, and hasn't changed
+# TODO: this should be done by cargo itself
+
+# Make it deterministic
+
+# The registry index changes all the time, so it's not deterministic
+rm -rf $out/registry/index
+
+# Make git DBs deterministic
+# TODO: test with git submodules
+[[ ! -d $out/git/checkouts ]] || (cd $out/git/checkouts && for name in *; do
+    cd "$out/git/checkouts/$name"
+    revs=""
+    for branch in *; do
+        cd "$branch"
+        rev="$(git rev-parse HEAD)"
+        revs="$revs $rev"
+        cd ..
+    done
+
+    (
+        # The following code was adapted from nix-prefetch-git
+
+        cd "$out/git/db/$name"
+
+        export GIT_DIR=.
+
+        # Remove all remote branches
+        git branch -r | while read branch; do
+            git branch -rD "$branch" >&2
+        done
+
+        # Remove tags that don't point to any HEAD
+        git tag | while read tag; do
+            rev="$(git rev-parse $tag)"
+            if [[ $revs != *" $rev"* ]]; then
+                git tag -d "$tag" >&2
+            fi
+        done
+
+        # Remove branches that don't point to any HEAD
+        branchrefs=()
+        eval "$(git for-each-ref --shell --format='branchrefs+=(%(refname))' refs/heads/)"
+
+        for branchref in "${branchrefs[@]}"; do
+            echo "Examining $branchref"
+            rev="$(git rev-parse "$branchref")"
+            echo "Has rev $rev"
+            echo "List of revs: $revs"
+            if [[ $revs != *" $rev"* ]]; then
+                echo "Deleting $branchref"
+                git update-ref -d "$branchref" >&2
+            fi
+        done
+
+        echo "$revs" | while read rev; do
+            echo "git branch b_$rev $rev"
+            git branch b_$rev $rev
+        done
+
+        # Remove files that have timestamps or otherwise have non-deterministic
+        # properties.
+        rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
+
+        # Do a full repack. Must run single-threaded, or else we lose determinism.
+        git config pack.threads 1
+        git repack -A -d -f
+        rm -f config
+
+        # Garbage collect unreferenced objects.
+        git gc --prune=all
+    )
+done)
+
+# Remove unneeded outputs
+[[ ! -d $out/registry/src ]] || rm -rf $out/registry/src
+[[ ! -d $out/git/checkouts ]] || rm -rf $out/git/checkouts

pkgs/build-support/rust/fetchcargo.nix

@@ -0,0 +1,19 @@
+{ stdenv, cacert, git, rustc, cargo, rustRegistry }:
+{ name ? "cargo-deps", src, sha256 }:
+
+stdenv.mkDerivation {
+  name = "${name}-fetch";
+  buildInputs = [ rustc cargo git ];
+  builder = ./fetch-builder.sh;
+  fetcher = ./fetch-cargo-deps;
+  inherit src rustRegistry;
+
+  outputHashAlgo = "sha256";
+  outputHashMode = "recursive";
+  outputHash = sha256;
+
+  SSL_CERT_FILE = "${cacert}/etc/ca-bundle.crt";
+
+  impureEnvVars = [ "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" ];
+  preferLocalBuild = true;
+}

pkgs/development/tools/build-managers/cargo/common.nix

@@ -3,6 +3,16 @@
 {
   inherit version;
 
+  name = "cargo-${version}";
+
+  postInstall = ''
+    rm "$out/lib/rustlib/components" \
+       "$out/lib/rustlib/install.log" \
+       "$out/lib/rustlib/rust-installer-version" \
+       "$out/lib/rustlib/uninstall.sh" \
+       "$out/lib/rustlib/manifest-cargo"
+  '';
+
   platform = if stdenv.system == "i686-linux"
     then "i686-unknown-linux-gnu"
     else if stdenv.system == "x86_64-linux"
@@ -20,5 +30,5 @@
     platforms = platforms.linux;
   };
 
-  name = "cargo-${version}";
+  setupHook = ./setup-hook.sh;
 }

pkgs/development/tools/build-managers/cargo/default.nix

@@ -0,0 +1,32 @@
+{ stdenv, fetchgit, rustPlatform, file, curl, python, pkgconfig, openssl
+, cmake, zlib }:
+
+with ((import ./common.nix) { inherit stdenv; version = "2015-04-14"; });
+
+with rustPlatform;
+
+buildRustPackage rec {
+  inherit name version meta setupHook;
+
+  src = fetchgit {
+    url = "https://github.com/rust-lang/cargo.git";
+    rev = "d49b44358ed800351647571144257d35ac0886cf";
+    sha256 = "1kaims28237mvp1qpw2cfgb3684jr54ivkdag0lw8iv9xap4i35y";
+    leaveDotGit = true;
+  };
+
+  depsSha256 = "1yi39asmnrya8w83jrjxym658cf1a5ffp8ym8502rqqvx30y0yx4";
+
+  buildInputs = [ file curl pkgconfig python openssl cmake zlib ];
+
+  configurePhase = ''
+    ./configure --prefix=$out --local-cargo=${cargo}/bin/cargo
+  '';
+
+  buildPhase = "make";
+
+  installPhase = ''
+    make install
+    ${postInstall}
+  '';
+}

pkgs/development/tools/build-managers/cargo/setup-hook.sh

@@ -0,0 +1,6 @@
+if [[ -n "$cargoDeps" ]]; then
+   echo "Using cargo deps from $cargoDeps"
+   cp -r $cargoDeps deps
+   chmod +w deps -R
+   export CARGO_HOME=$(realpath deps)
+fi

pkgs/development/tools/build-managers/cargo/snapshot.nix

@@ -19,11 +19,8 @@ let snapshotHash = if stdenv.system == "i686-linux"
     snapshotName = "cargo-nightly-${platform}.tar.gz";
 in
 
-
 stdenv.mkDerivation {
-  inherit name;
+  inherit name version meta setupHook;
-  inherit version;
-  inherit meta;
 
   src = fetchurl {
     url = "https://static-rust-lang-org.s3.amazonaws.com/cargo-dist/${snapshotDate}/${snapshotName}";
@@ -35,10 +32,8 @@ stdenv.mkDerivation {
   installPhase = ''
     mkdir -p "$out"
     ./install.sh "--prefix=$out"
-    rm "$out/lib/rustlib/components" \
+
-       "$out/lib/rustlib/install.log" \
+    ${postInstall}
-       "$out/lib/rustlib/rust-installer-version" \
-       "$out/lib/rustlib/uninstall.sh"
   '' + (if stdenv.isLinux then ''
     patchelf --interpreter "${stdenv.glibc}/lib/${stdenv.cc.dynamicLinker}" \
              --set-rpath "${stdenv.cc.cc}/lib/:${stdenv.cc.cc}/lib64/:${zlib}/lib" \

pkgs/development/tools/rust/racer/default.nix

@@ -1,6 +1,8 @@
-{stdenv, fetchgit, rustc, cargo, makeWrapper }:
+{stdenv, fetchgit, rustPlatform, makeWrapper }:
 
-stdenv.mkDerivation rec {
+with rustPlatform;
+
+buildRustPackage rec {
   #TODO add emacs support
   name = "racer-git-2015-04-12";
   src = fetchgit {
@@ -9,11 +11,9 @@ stdenv.mkDerivation rec {
     sha256 = "0a768gvjry86l0xa5q0122iyq7zn2h9adfniglsgrbs4fan49xyn";
   };
 
-  buildInputs = [ rustc cargo makeWrapper ];
+  depsSha256 = "0x1rq012k04ci18w5fll56jn011f1yyprs38pb3r223bag94ivsy";
 
-  buildPhase = ''
+  buildInputs = [ makeWrapper ];
-    CARGO_HOME="$NIX_BUILD_TOP/.cargo" cargo build --release
-  '';
 
   installPhase = ''
     mkdir -p $out/bin

pkgs/top-level/all-packages.nix

@@ -4337,6 +4337,24 @@ let
   rustcMaster = callPackage ../development/compilers/rustc/head.nix {};
   rustc = rustcBeta;
 
+  rustPlatform = rustStable;
+
+  rustStable = recurseIntoAttrs (makeRustPlatform rustc cargo rustStable);
+  rustUnstable = recurseIntoAttrs (makeRustPlatform rustcMaster cargo rustUnstable);
+
+  # rust platform to build cargo itself (with cargoSnapshot)
+  rustCargoPlatform = makeRustPlatform rustcMaster cargoSnapshot rustCargoPlatform;
+
+  makeRustPlatform = rustc: cargo: self:
+    let
+      callPackage = newScope self;
+    in {
+      inherit rustc cargo;
+
+      rustRegistry = callPackage ./rust-packages.nix { };
+
+      buildRustPackage = callPackage ../build-support/rust { };
+    };
 
   sbclBootstrap = callPackage ../development/compilers/sbcl/bootstrap.nix {};
   sbcl = callPackage ../development/compilers/sbcl {
@@ -4938,6 +4956,11 @@ let
 
   byacc = callPackage ../development/tools/parsing/byacc { };
 
+  cargo = callPackage ../development/tools/build-managers/cargo {
+    # cargo needs to be built with rustCargoPlatform, which uses cargoSnapshot
+    rustPlatform = rustCargoPlatform;
+  };
+
   cargoSnapshot = callPackage ../development/tools/build-managers/cargo/snapshot.nix { };
 
   casperjs = callPackage ../development/tools/casperjs { };
@@ -5263,8 +5286,8 @@ let
   premake = premake4;
 
   racerRust = callPackage ../development/tools/rust/racer {
-    rustc = rustcMaster;
+    # racerRust still uses unstable features from the standard library
-    cargo = cargoSnapshot;
+    rustPlatform = rustUnstable;
   };
 
   radare = callPackage ../development/tools/analysis/radare {

pkgs/top-level/rust-packages.nix

@@ -0,0 +1,19 @@
+# This file defines the source of Rust / cargo's crates registry
+#
+# buildRustPackage will automatically download dependencies from the registry
+# version that we define here. If you're having problems downloading / finding
+# a Rust library, try updating this to a newer commit.
+
+{ fetchgit }:
+
+fetchgit {
+  url = git://github.com/rust-lang/crates.io-index.git;
+
+  # 2015-04-20
+  rev = "c7112fed5f973e438bb600946016c5083e66b1c9";
+  sha256 = "0vyrz7d6zvh79hx5fg557g93r9qm40wx1g4hx7304lina4smk30h";
+
+  # cargo needs the 'master' branch to exist
+  leaveDotGit = true;
+  branchName = "master";
+}