From fa6c5c3dc76ae4428b9643b3f0ab05bb9049181e Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Wed, 19 Feb 2020 14:50:12 +0100 Subject: [PATCH 1/8] buildBazelPackage: prime fixed output derivations All bazel fixed output derivations should be specific to the bazel version that was used to generate them. There is not guarantee that the build will still succeed or reproduces (without the cached fixed output) if the fetch phase wasn't rerun with a different bazel version. In the past bazel had been bumped but not all those packages that have fixed outputs from bazel builds. This lead to compiling and somewhat working TF versions that couldn't be reproduced without the cached fixed outputs. --- pkgs/build-support/build-bazel-package/default.nix | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkgs/build-support/build-bazel-package/default.nix b/pkgs/build-support/build-bazel-package/default.nix index 058f42aa8d5..47c72bf9260 100644 --- a/pkgs/build-support/build-bazel-package/default.nix +++ b/pkgs/build-support/build-bazel-package/default.nix @@ -122,6 +122,8 @@ in stdenv.mkDerivation (fBuildAttrs // { cp -r $bazelOut/external $out + echo '${bazel.name}' > $out/.nix-bazel-version + runHook postInstall ''; @@ -143,6 +145,14 @@ in stdenv.mkDerivation (fBuildAttrs // { preConfigure = '' mkdir -p "$bazelOut" + + test "${bazel.name}" = "$(<$deps/.nix-bazel-version)" || { + echo "fixed output derivation was built for a different bazel version" >&2 + echo " got: $(<$deps/.nix-bazel-version)" >&2 + echo "expected: ${bazel.name}" >&2 + exit 1 + } + cp -r $deps $bazelOut/external chmod -R +w $bazelOut find $bazelOut -type l | while read symlink; do From abd312b6b1cd17b6def045ca9007f7d17e984820 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Wed, 19 Feb 2020 14:54:44 +0100 Subject: [PATCH 2/8] gvisor: update fixed output hash --- pkgs/applications/virtualization/gvisor/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/applications/virtualization/gvisor/default.nix b/pkgs/applications/virtualization/gvisor/default.nix index 2d99fb3bf57..3d0ae35f560 100644 --- a/pkgs/applications/virtualization/gvisor/default.nix +++ b/pkgs/applications/virtualization/gvisor/default.nix @@ -76,7 +76,7 @@ in buildBazelPackage rec { rm -f "$bazelOut"/java.log "$bazelOut"/java.log.* ''; - sha256 = "122qk6iv8hd7g2a84y9aqqhij4r0m47vpxzbqhhh6k5livc73qd6"; + sha256 = "1bn7nhv5pag8fdm8l8nvgg3fzvhpy2yv9yl2slrb16lckxzha3v6"; }; buildAttrs = { From 9683826ffa9b17f812d1baff556e2ca2ac4334ad Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Wed, 19 Feb 2020 14:58:10 +0100 Subject: [PATCH 3/8] bazel-watcher: update fixed output hash --- pkgs/development/tools/bazel-watcher/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/tools/bazel-watcher/default.nix b/pkgs/development/tools/bazel-watcher/default.nix index 62ecc47b17b..29276fb6bfb 100644 --- a/pkgs/development/tools/bazel-watcher/default.nix +++ b/pkgs/development/tools/bazel-watcher/default.nix @@ -62,7 +62,7 @@ buildBazelPackage rec { sed -e '/^FILE:@bazel_gazelle_go_repository_tools.*/d' -i $bazelOut/external/\@*.marker ''; - sha256 = "0g2y283glx2ykxxqc3vsg520a6s2w5d937wndhgpfajc5yjgiz43"; + sha256 = "0cmj186n2y1g9kkdhcivmh2qvigvpnbp03m575b7hgsxi1cp3ssj"; }; buildAttrs = { From fa7b4d513c56e39e41e92cb73766f7adc341ab9e Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Wed, 19 Feb 2020 15:04:34 +0100 Subject: [PATCH 4/8] tensorflow-probabilit: update fixed output hash --- .../python-modules/tensorflow-probability/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/tensorflow-probability/default.nix b/pkgs/development/python-modules/tensorflow-probability/default.nix index b585ab4f81b..99a3a978a4e 100644 --- a/pkgs/development/python-modules/tensorflow-probability/default.nix +++ b/pkgs/development/python-modules/tensorflow-probability/default.nix @@ -42,7 +42,7 @@ let bazelTarget = ":pip_pkg"; fetchAttrs = { - sha256 = "0sjjj9z1dhilhpc8pq4154czrb79z9cm044jvn75kxcjv6v5l2m5"; + sha256 = "0135nxxvkmjzpd80r1g9fdkk9h62g0xlvp32g5zgk0hkma5kq0bx"; }; buildAttrs = { From 0ff376c03f1459e12ae5e28b7ef298e1311fce36 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Wed, 19 Feb 2020 15:09:48 +0100 Subject: [PATCH 5/8] bazel-remote: update fixed output hash --- .../tools/build-managers/bazel/bazel-remote/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/tools/build-managers/bazel/bazel-remote/default.nix b/pkgs/development/tools/build-managers/bazel/bazel-remote/default.nix index ffdeb01c2bb..88dd7f45ab5 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel-remote/default.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel-remote/default.nix @@ -64,7 +64,7 @@ buildBazelPackage rec { sed -e '/^FILE:@bazel_gazelle_go_repository_tools.*/d' -i $bazelOut/external/\@*.marker ''; - sha256 = "1n66hg1w5jv2rc8q4sjlaf0agvxr713aa40mbkhgjv57x9j7bgn0"; + sha256 = "0lnwc7ya6sjlx6jk5dwws166vli1lzdfc256f9g6dy1fci3sc17y"; }; buildAttrs = { From 84b36780723ad90052a87b151c8fafbf1763f56d Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Mon, 24 Feb 2020 13:36:39 +0100 Subject: [PATCH 6/8] pythonPackages.dm-sonned: update fixed output hash --- pkgs/development/python-modules/dm-sonnet/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/dm-sonnet/default.nix b/pkgs/development/python-modules/dm-sonnet/default.nix index 4ce61680f03..603d900b810 100644 --- a/pkgs/development/python-modules/dm-sonnet/default.nix +++ b/pkgs/development/python-modules/dm-sonnet/default.nix @@ -36,7 +36,7 @@ let bazelTarget = ":install"; fetchAttrs = { - sha256 = "0mxma7jajm42v1hv6agl909xra0azihj588032ivhlmmh403x6wg"; + sha256 = "0wb2gh9ji8bgq4s9ci9x017dybxqzjhncpw33b1wjksm2yhbkvlz"; }; bazelFlags = [ From c539f937c539b99e8c8b763b2568b52e39d3e9c4 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Mon, 2 Mar 2020 17:22:35 +0100 Subject: [PATCH 7/8] pythonPackages.tensorflow-estimator_1_15_1: init at 1.15.1 TF 1.15 still needs an older version of the tensorflow-estimator package. --- .../tensorflow-estimator/1_15_1.nix | 27 +++++++++++++++++++ .../python-modules/tensorflow/default.nix | 4 +-- pkgs/top-level/python-packages.nix | 1 + 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 pkgs/development/python-modules/tensorflow-estimator/1_15_1.nix diff --git a/pkgs/development/python-modules/tensorflow-estimator/1_15_1.nix b/pkgs/development/python-modules/tensorflow-estimator/1_15_1.nix new file mode 100644 index 00000000000..39c667357c8 --- /dev/null +++ b/pkgs/development/python-modules/tensorflow-estimator/1_15_1.nix @@ -0,0 +1,27 @@ +{ stdenv, fetchPypi, buildPythonPackage +, numpy +, absl-py +, mock +}: + +buildPythonPackage rec { + pname = "tensorflow-estimator"; + version = "1.15.1"; + format = "wheel"; + + src = fetchPypi { + pname = "tensorflow_estimator"; + inherit version format; + sha256 = "1fc61wmc0w22frs79j2x4g6wnv5g21xc6rix1g4bsvy9qfvvylw8"; + }; + + propagatedBuildInputs = [ mock numpy absl-py ]; + + meta = with stdenv.lib; { + description = "TensorFlow Estimator is a high-level API that encapsulates model training, evaluation, prediction, and exporting."; + homepage = http://tensorflow.org; + license = licenses.asl20; + maintainers = with maintainers; [ jyp ]; + }; +} + diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 9e5acca4749..5422503173c 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -7,7 +7,7 @@ , future, setuptools, wheel, keras-preprocessing, keras-applications, google-pasta , functools32 , opt-einsum -, termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator +, termcolor, grpcio, six, wrapt, protobuf, tensorflow-estimator_1_15_1 # Common deps , git, swig, which, binutils, glibcLocales, cython # Common libraries @@ -374,7 +374,7 @@ in buildPythonPackage { numpy six protobuf - tensorflow-estimator + tensorflow-estimator_1_15_1 termcolor wrapt grpcio diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index ebe69a1b14c..90e80890c6f 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -6657,6 +6657,7 @@ in { zerobin = callPackage ../development/python-modules/zerobin { }; tensorflow-estimator = callPackage ../development/python-modules/tensorflow-estimator { }; + tensorflow-estimator_1_15_1 = callPackage ../development/python-modules/tensorflow-estimator/1_15_1.nix { }; tensorflow-probability = callPackage ../development/python-modules/tensorflow-probability { }; From 642bea6cecf5ee1258b2a317f6e56278c7a355b9 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Mon, 24 Feb 2020 13:32:36 +0100 Subject: [PATCH 8/8] tensorflow: update fixed output hashes --- pkgs/development/python-modules/tensorflow/default.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/tensorflow/default.nix b/pkgs/development/python-modules/tensorflow/default.nix index 5422503173c..cdf659a74dc 100644 --- a/pkgs/development/python-modules/tensorflow/default.nix +++ b/pkgs/development/python-modules/tensorflow/default.nix @@ -296,10 +296,12 @@ let TF_SYSTEM_LIBS = null; # cudaSupport causes fetch of ncclArchive, resulting in different hashes + # FIXME: can't (re)produce this output with current bazel. + # FIXME: build log: https://gist.github.com/andir/eff3e9c8eda5b56c8ea84903aed9cc35 sha256 = if cudaSupport then - "1p544yk7jcspgc4qr4amw11ds16c2an5yxvagx5pmwawz0s083pf" + "0gyhjvzshgj59mbns8njlfl9qpz4sdg4j0xs2dva0w2nql7cr7im" else - "1dqbw3k3avqiy9xpgs44l6z65ab5rjjlxwig8z7gcl7fw9h6sbq9"; + "04jvg3mc2si4xdbszc1vnw1rmf22p7snbjphmnklp7bc39jxkcrz"; }; buildAttrs = {