From 11e2009821decf55aa20589447076aab5f998464 Mon Sep 17 00:00:00 2001 From: Matthieu Coudron Date: Thu, 6 Sep 2018 00:11:02 +0900 Subject: [PATCH 1/7] haskellPackages.tensorflow-mnist-input-data: fetch Prefetch data so that the package doesn't try to connect to the internet. --- .../configuration-tensorflow.nix | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pkgs/development/haskell-modules/configuration-tensorflow.nix b/pkgs/development/haskell-modules/configuration-tensorflow.nix index dfc93686405..d1ca474ba4f 100644 --- a/pkgs/development/haskell-modules/configuration-tensorflow.nix +++ b/pkgs/development/haskell-modules/configuration-tensorflow.nix @@ -62,6 +62,30 @@ in { mkDerivation, base, bytestring, Cabal, cryptonite, directory , filepath, HTTP, network-uri, stdenv }: + + let + urlPrefix = "http://yann.lecun.com/exdb/mnist/"; + + # File names relative to 'urlPrefix' and their sha256. + fileInfos = [ + [ "train-images-idx3-ubyte.gz" + "440fcabf73cc546fa21475e81ea370265605f56be210a4024d2ca8f203523609" + ] + + [ "train-labels-idx1-ubyte.gz" + "3552534a0a558bbed6aed32b30c495cca23d567ec52cac8be1a0730e8010255c" + ] + + [ "t10k-images-idx3-ubyte.gz" + "8d422c7b0a1c1c79245a5bcf07fe86e33eeafee792b84584aec276f5a2dbc4e6" + ] + + [ "t10k-labels-idx1-ubyte.gz" + "f7ae60f92e00ec6debd23a6088c31dbd2371eca3ffa0defaefb259924204aec6" + ] + ]; + downloads = map (x: pkgs.fetchurl { url = urlPrefix + builtins.head x; sha256= builtins.tail x;}) fileInfos; + in mkDerivation { pname = "tensorflow-mnist-input-data"; version = "0.1.0.0"; @@ -71,6 +95,9 @@ in base bytestring Cabal cryptonite directory filepath HTTP network-uri ]; + preConfigure = pkgs.lib.strings.concatStringsSep "\n" ( + map (x: "cp ${x} data/$(stripHash ${x})") downloads + ); libraryHaskellDepends = [ base ]; homepage = "https://github.com/tensorflow/haskell#readme"; description = "Downloader of input data for training MNIST"; From 6bd8799448dd3d4a7f92f569202010e1fe5b9a33 Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Thu, 6 Sep 2018 08:43:04 +0200 Subject: [PATCH 2/7] haskellPackages.tensorflow-mnist-input-data: use an attribute set to specify the data files This is more understandable and safer than using a nested list. --- .../configuration-tensorflow.nix | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/pkgs/development/haskell-modules/configuration-tensorflow.nix b/pkgs/development/haskell-modules/configuration-tensorflow.nix index d1ca474ba4f..10ee30d81c8 100644 --- a/pkgs/development/haskell-modules/configuration-tensorflow.nix +++ b/pkgs/development/haskell-modules/configuration-tensorflow.nix @@ -64,27 +64,17 @@ in }: let - urlPrefix = "http://yann.lecun.com/exdb/mnist/"; - - # File names relative to 'urlPrefix' and their sha256. - fileInfos = [ - [ "train-images-idx3-ubyte.gz" - "440fcabf73cc546fa21475e81ea370265605f56be210a4024d2ca8f203523609" - ] - - [ "train-labels-idx1-ubyte.gz" - "3552534a0a558bbed6aed32b30c495cca23d567ec52cac8be1a0730e8010255c" - ] - - [ "t10k-images-idx3-ubyte.gz" - "8d422c7b0a1c1c79245a5bcf07fe86e33eeafee792b84584aec276f5a2dbc4e6" - ] - - [ "t10k-labels-idx1-ubyte.gz" - "f7ae60f92e00ec6debd23a6088c31dbd2371eca3ffa0defaefb259924204aec6" - ] - ]; - downloads = map (x: pkgs.fetchurl { url = urlPrefix + builtins.head x; sha256= builtins.tail x;}) fileInfos; + fileInfos = { + "train-images-idx3-ubyte.gz" = "440fcabf73cc546fa21475e81ea370265605f56be210a4024d2ca8f203523609"; + "train-labels-idx1-ubyte.gz" = "3552534a0a558bbed6aed32b30c495cca23d567ec52cac8be1a0730e8010255c"; + "t10k-images-idx3-ubyte.gz" = "8d422c7b0a1c1c79245a5bcf07fe86e33eeafee792b84584aec276f5a2dbc4e6"; + "t10k-labels-idx1-ubyte.gz" = "f7ae60f92e00ec6debd23a6088c31dbd2371eca3ffa0defaefb259924204aec6"; + }; + downloads = with pkgs.lib; flip mapAttrsToList fileInfos (name: sha256: + pkgs.fetchurl { + url = "http://yann.lecun.com/exdb/mnist/${name}"; + inherit sha256; + }); in mkDerivation { pname = "tensorflow-mnist-input-data"; From 641025d410ec7212996ba9d7cea4d12ce8218886 Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Thu, 6 Sep 2018 08:44:19 +0200 Subject: [PATCH 3/7] haskellPackages.tensorflow-mnist-input-data: link data files instead of copying This prevents duplication in the nix store. --- pkgs/development/haskell-modules/configuration-tensorflow.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/haskell-modules/configuration-tensorflow.nix b/pkgs/development/haskell-modules/configuration-tensorflow.nix index 10ee30d81c8..dedcaf0cbc7 100644 --- a/pkgs/development/haskell-modules/configuration-tensorflow.nix +++ b/pkgs/development/haskell-modules/configuration-tensorflow.nix @@ -86,7 +86,7 @@ in network-uri ]; preConfigure = pkgs.lib.strings.concatStringsSep "\n" ( - map (x: "cp ${x} data/$(stripHash ${x})") downloads + map (x: "ln -s ${x} data/$(stripHash ${x})") downloads ); libraryHaskellDepends = [ base ]; homepage = "https://github.com/tensorflow/haskell#readme"; From a5f08c17e5034ad00c170e1599d19f6d77aa19ca Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Thu, 6 Sep 2018 08:49:17 +0200 Subject: [PATCH 4/7] haskellPackages.tensorflow-mnist: fix missing dependency on tensorflow-mnist-input-data --- .../haskell-modules/configuration-tensorflow.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkgs/development/haskell-modules/configuration-tensorflow.nix b/pkgs/development/haskell-modules/configuration-tensorflow.nix index dedcaf0cbc7..43a3b82923b 100644 --- a/pkgs/development/haskell-modules/configuration-tensorflow.nix +++ b/pkgs/development/haskell-modules/configuration-tensorflow.nix @@ -55,9 +55,11 @@ in tensorflow-logging = super.tensorflow-logging.override { inherit proto-lens; }; - tensorflow-mnist = super.tensorflow-mnist.override { + tensorflow-mnist = overrideCabal (super.tensorflow-mnist.override { inherit proto-lens; - }; + # https://github.com/tensorflow/haskell/issues/215 + tensorflow-mnist-input-data = self.tensorflow-mnist-input-data; + }) (_drv: { broken = false; }); tensorflow-mnist-input-data = setSourceRoot "tensorflow-mnist-input-data" (super.callPackage ( { mkDerivation, base, bytestring, Cabal, cryptonite, directory , filepath, HTTP, network-uri, stdenv From 39d35e77ee51c24c688d028f0dc0b9fb761db1d7 Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Thu, 6 Sep 2018 08:54:12 +0200 Subject: [PATCH 5/7] Revert "libtensorflow: fix hashes for darwin and cuda downloads" This reverts commit 93ed13f86b624dc3214f58d09c3f3318f9900afa. libtensorflow is only used by the Haskell tensorflow packages and they don't work with tensorflow-1.10 yet. So the easiest solution is to just revert this commit and add it back when they do gain support. --- pkgs/development/libraries/libtensorflow/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/libraries/libtensorflow/default.nix b/pkgs/development/libraries/libtensorflow/default.nix index b4e616409c4..2bc83b31423 100644 --- a/pkgs/development/libraries/libtensorflow/default.nix +++ b/pkgs/development/libraries/libtensorflow/default.nix @@ -38,12 +38,12 @@ in stdenv.mkDerivation rec { sha256 = if system == "linux-x86_64" then if cudaSupport - then "0v66sscxpyixjrf9yjshl001nix233i6chc61akx0kx7ial4l1wn" + then "1q3mh06x344im25z7r3vgrfksfdsi8fh8ldn6y2mf86h4d11yxc3" else "11sbpcbgdzj8v17mdppfv7v1fn3nbzkdad60gc42y2j6knjbmwxb" else if system == "darwin-x86_64" then if cudaSupport then unavailable - else "11p0f77m4wycpc024mh7jx0kbdhgm0wp6ir6dsa8lkcpdb59bn59" + else "1qj0v1706w6mczycdsh38h2glyv5d25v62kdn98wxd5rw8f9v657" else unavailable; }; From 6e14b9fc2d738b18948d221367899c16db92f622 Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Thu, 6 Sep 2018 08:56:09 +0200 Subject: [PATCH 6/7] Revert "libtensorflow: 1.9.0 -> 1.10.0" This reverts commit 713991132eb7b01f5732a301da4d377bafe32e8b. libtensorflow is only used by the Haskell tensorflow packages and they don't work with tensorflow-1.10 yet. So the easiest solution is to just revert this commit and add it back when they do gain support. --- pkgs/development/libraries/libtensorflow/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/libraries/libtensorflow/default.nix b/pkgs/development/libraries/libtensorflow/default.nix index 2bc83b31423..e6cd140c4e4 100644 --- a/pkgs/development/libraries/libtensorflow/default.nix +++ b/pkgs/development/libraries/libtensorflow/default.nix @@ -31,7 +31,7 @@ let in stdenv.mkDerivation rec { pname = "libtensorflow"; - version = "1.10.0"; + version = "1.9.0"; name = "${pname}-${version}"; src = fetchurl { url = "https://storage.googleapis.com/tensorflow/${pname}/${pname}-${tfType}-${system}-${version}.tar.gz"; @@ -39,7 +39,7 @@ in stdenv.mkDerivation rec { if system == "linux-x86_64" then if cudaSupport then "1q3mh06x344im25z7r3vgrfksfdsi8fh8ldn6y2mf86h4d11yxc3" - else "11sbpcbgdzj8v17mdppfv7v1fn3nbzkdad60gc42y2j6knjbmwxb" + else "0l9ps115ng5ffzdwphlqmj3jhidps2v5afppdzrbpzmy41xz0z21" else if system == "darwin-x86_64" then if cudaSupport then unavailable From 1fc5a6d2b6bf3acfd5e4761fd2cb0c549c6045f0 Mon Sep 17 00:00:00 2001 From: Bas van Dijk Date: Thu, 6 Sep 2018 09:00:07 +0200 Subject: [PATCH 7/7] haskell: build the tensorflow packages on hydra since they now build successfully --- .../haskell-modules/configuration-hackage2nix.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkgs/development/haskell-modules/configuration-hackage2nix.yaml b/pkgs/development/haskell-modules/configuration-hackage2nix.yaml index 3aee4857f99..4a26fcbffc1 100644 --- a/pkgs/development/haskell-modules/configuration-hackage2nix.yaml +++ b/pkgs/development/haskell-modules/configuration-hackage2nix.yaml @@ -9054,12 +9054,6 @@ dont-distribute-packages: temporary-resourcet: [ i686-linux, x86_64-linux, x86_64-darwin ] tempus: [ i686-linux, x86_64-linux, x86_64-darwin ] tensor: [ i686-linux, x86_64-linux, x86_64-darwin ] - tensorflow-core-ops: [ i686-linux, x86_64-linux, x86_64-darwin ] - tensorflow-logging: [ i686-linux, x86_64-linux, x86_64-darwin ] - tensorflow-opgen: [ i686-linux, x86_64-linux, x86_64-darwin ] - tensorflow-ops: [ i686-linux, x86_64-linux, x86_64-darwin ] - tensorflow-proto: [ i686-linux, x86_64-linux, x86_64-darwin ] - tensorflow: [ i686-linux, x86_64-linux, x86_64-darwin ] term-rewriting: [ i686-linux, x86_64-linux, x86_64-darwin ] termbox-bindings: [ i686-linux, x86_64-linux, x86_64-darwin ] termcolor: [ i686-linux, x86_64-linux, x86_64-darwin ]