From 6206a342e0c409de25d3573f7f444434ab75d0ed Mon Sep 17 00:00:00 2001 From: Benjamin Hipple Date: Tue, 23 Oct 2018 17:33:49 -0400 Subject: [PATCH] mkl: include Intel's libiomp.so in the MKL RPM unpack Since Intel's default openmp implementation is available in the same src tarball, we can just include it in the package. This means that `mkl` now "just works" without any environment variables, fragile setup-hooks, or forced propagation. Since the openmp implementation is only needed at runtime (and for test cases), users can substitute a different one if they prefer by exporting it with `LD_PRELOAD`, which is how Intel recommends handling this. If they do not do so, `libiomp.so` lives next to `libmkl_rt.so` and thus will be in the RPATH as a sane default. Since this still comes from the same src tarball, we can ship it without losing the fixed-output derivation; likewise, since Hydra is not building or caching these, shipping these proprietary packages costs no bandwidth for the nix community. --- doc/languages-frameworks/python.section.md | 11 +++++- .../libraries/science/math/mkl/default.nix | 34 ++++++++----------- .../python-modules/numexpr/default.nix | 8 +---- .../python-modules/numpy/default.nix | 2 -- 4 files changed, 25 insertions(+), 30 deletions(-) diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index 3e4e12665bd..acd2bf769b0 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -1104,7 +1104,7 @@ on `numpy` will be built with `mkl`. The following is an overlay that configures `numpy` to use `mkl`: ```nix self: super: { - python36 = super.python36.override { + python37 = super.python37.override { packageOverrides = python-self: python-super: { numpy = python-super.numpy.override { blas = super.pkgs.mkl; @@ -1114,6 +1114,15 @@ self: super: { } ``` +`mkl` requires an `openmp` implementation when running with multiple processors. +By default, `mkl` will use Intel's `iomp` implementation if no other is +specified, but this is a runtime-only dependency and binary compatible with the +LLVM implementation. To use that one instead, Intel recommends users set it with +`LD_PRELOAD`. + +Note that `mkl` is only available on `x86_64-{linux,darwin}` platforms; +moreover, Hydra is not building and distributing pre-compiled binaries using it. + ## Contributing ### Contributing guidelines diff --git a/pkgs/development/libraries/science/math/mkl/default.nix b/pkgs/development/libraries/science/math/mkl/default.nix index 37814047f97..0eecb2012f1 100644 --- a/pkgs/development/libraries/science/math/mkl/default.nix +++ b/pkgs/development/libraries/science/math/mkl/default.nix @@ -1,21 +1,8 @@ { stdenvNoCC, writeText, fetchurl, rpmextract, undmg }: /* - Some (but not all) mkl functions require openmp, but Intel does not add these - to SO_NEEDED and instructs users to put openmp on their LD_LIBRARY_PATH. If - you are using mkl and your library/application is using some of the functions - that require openmp, add a setupHook like this to your package: - - setupHook = writeText "setup-hook.sh" '' - addOpenmp() { - addToSearchPath LD_LIBRARY_PATH ${openmp}/lib - } - addEnvHooks "$targetOffset" addOpenmp - ''; - - We do not add the setup hook here, because avoiding it allows this large - package to be a fixed-output derivation with better cache efficiency. - */ - + For details on using mkl as a blas provider for python packages such as numpy, + numexpr, scipy, etc., see the Python section of the NixPkgs manual. +*/ stdenvNoCC.mkDerivation rec { name = "mkl-${version}"; version = "${date}.${rel}"; @@ -43,16 +30,23 @@ stdenvNoCC.mkDerivation rec { '' else '' rpmextract rpm/intel-mkl-common-c-${date}-${rel}-${date}-${rel}.noarch.rpm rpmextract rpm/intel-mkl-core-rt-${date}-${rel}-${date}-${rel}.x86_64.rpm + rpmextract rpm/intel-openmp-19.0.0-${rel}-19.0.0-${rel}.x86_64.rpm ''; installPhase = if stdenvNoCC.isDarwin then '' mkdir -p $out/lib + cp -r compilers_and_libraries_${version}/mac/mkl/include $out/ - cp -r compilers_and_libraries_${version}/mac/mkl/lib/* $out/lib/ + cp -r compilers_and_libraries_${version}/licensing/mkl/en/license.txt $out/lib/ + cp -r compilers_and_libraries_${version}/mac/compiler/lib/* $out/lib/ + cp -r compilers_and_libraries_${version}/mac/mkl/lib/* $out/lib/ '' else '' mkdir -p $out/lib + cp -r opt/intel/compilers_and_libraries_${version}/linux/mkl/include $out/ + + cp -r opt/intel/compilers_and_libraries_${version}/linux/compiler/lib/intel64_lin/* $out/lib/ cp -r opt/intel/compilers_and_libraries_${version}/linux/mkl/lib/intel64_lin/* $out/lib/ cp license.txt $out/lib/ ''; @@ -66,8 +60,8 @@ stdenvNoCC.mkDerivation rec { outputHashAlgo = "sha256"; outputHashMode = "recursive"; outputHash = if stdenvNoCC.isDarwin - then "1224dln7n8px1rk8biiggf77wjhxh8mzw0hd8zlyjm8i6j8w7i12" - else "0d8ai0wi8drp071acqkm1wv6vyg12010y843y56zzi1pql81xqvx"; + then "0000000000000000000000000000000000000000000000000000" + else "1amagcaan0hk3x9v7gg03gkw02n066v4kmjb32yyzsy5rfrivb1a"; meta = with stdenvNoCC.lib; { description = "Intel Math Kernel Library"; @@ -78,7 +72,7 @@ stdenvNoCC.mkDerivation rec { threading models. ''; homepage = https://software.intel.com/en-us/mkl; - license = [ licenses.issl licenses.unfreeRedistributable ]; + license = licenses.issl; platforms = [ "x86_64-linux" "x86_64-darwin" ]; maintainers = [ maintainers.bhipple ]; }; diff --git a/pkgs/development/python-modules/numexpr/default.nix b/pkgs/development/python-modules/numexpr/default.nix index acf41fb539e..280b11706a1 100644 --- a/pkgs/development/python-modules/numexpr/default.nix +++ b/pkgs/development/python-modules/numexpr/default.nix @@ -3,7 +3,6 @@ , fetchPypi , python , numpy -, llvmPackages ? null }: buildPythonPackage rec { @@ -16,16 +15,11 @@ buildPythonPackage rec { }; # Remove existing site.cfg, use the one we built for numpy. - # Somehow openmp needs to be added to LD_LIBRARY_PATH - # https://software.intel.com/en-us/forums/intel-system-studio/topic/611682 preBuild = '' rm site.cfg ln -s ${numpy.cfg} site.cfg - export LD_LIBRARY_PATH=${llvmPackages.openmp}/lib ''; - buildInputs = [] ++ lib.optional (numpy.blasImplementation == "mkl") llvmPackages.openmp; - propagatedBuildInputs = [ numpy ]; # Run the test suite. @@ -47,4 +41,4 @@ buildPythonPackage rec { homepage = "https://github.com/pydata/numexpr"; license = lib.licenses.mit; }; -} \ No newline at end of file +} diff --git a/pkgs/development/python-modules/numpy/default.nix b/pkgs/development/python-modules/numpy/default.nix index 8a3879f1982..d66cbd77bbc 100644 --- a/pkgs/development/python-modules/numpy/default.nix +++ b/pkgs/development/python-modules/numpy/default.nix @@ -71,8 +71,6 @@ in buildPythonPackage rec { inherit blasImplementation cfg; }; - doCheck = blasImplementation != "mkl"; - # Disable two tests # - test_f2py: f2py isn't yet on path. # - test_large_file_support: takes a long time and can cause the machine to run out of disk space