From c420de6b05710cccee5b0d9e5573ae36c3e31c5e Mon Sep 17 00:00:00 2001 From: aszlig Date: Fri, 22 Jan 2016 18:49:33 +0100 Subject: [PATCH 1/5] gobject-introspection: Fix patching shared objects The gi-r-scanner is generating a list of shared libraries that are referenced in the shared-library attribute of the element of the GIR file. However, this attribute only contains the names of the libraries and not the full store paths, like for example while preparing to package libblockdev, the following items were included in the shared-library attribute: /nix/store/...-libblockdev-1.3/lib/libblockdev.so.0 libm.so.6 libdmraid.so.1.0.0.rc16 libbd_utils.so.0 Unfortunately, loading such a library without setting LD_LIBRARY_PATH is going to fail finding libm.so.6 and libdmraid.so.1.0.0.rc16. Now the first attempt at solving this was to put absolute paths of all the libraries referenced in the shared-library attribute, but this also led up to including paths of build-time shared objects into that attribute: /nix/store/...-libblockdev-1.3/lib/libblockdev.so.0 /nix/store/...-glibc-2.21/lib/libm.so.6 /nix/store/...-dmraid-1.0.0.rc16/lib/libdmraid.so.1.0.0.rc16 /tmp/nix-build-libblockdev-1.3.drv-0/.../utils/.libs/libbd_utils.so.0 This of course is not what we want, so the final solution is to only use the absolute path whenever it is a Nix path and leave the library name as-is if the path doesn't reside within the store, like this: /nix/store/...-libblockdev-1.3/lib/libblockdev.so.0 /nix/store/...-glibc-2.21/lib/libm.so.6 /nix/store/...-dmraid-1.0.0.rc16/lib/libdmraid.so.1.0.0.rc16 libbd_utils.so.0 The downside of this approach is that if not even the output path of the library is in LD_LIBRARY_PATH, even loading of libbd_utils.so.0 could fail, so we need to patch the loader as well. Signed-off-by: aszlig --- .../absolute_shlib_path.patch | 40 +++++++++++++++++-- .../gobject-introspection/default.nix | 9 ++++- pkgs/top-level/all-packages.nix | 4 +- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch b/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch index 04bcc42a032..2803d767dc7 100644 --- a/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch +++ b/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch @@ -1,6 +1,40 @@ ---- ./giscanner/utils.py.orig 2014-08-14 22:05:05.055334080 +0200 -+++ ./giscanner/utils.py 2014-08-14 22:05:24.687497334 +0200 -@@ -110,17 +110,11 @@ +diff --git a/giscanner/shlibs.py b/giscanner/shlibs.py +index 838d343..9dbdc0f 100644 +--- a/giscanner/shlibs.py ++++ b/giscanner/shlibs.py +@@ -53,10 +53,24 @@ def _resolve_libtool(options, binary, libraries): + # Match absolute paths on OS X to conform to how libraries are usually + # referenced on OS X systems. + def _ldd_library_pattern(library_name): ++ nix_store_dir = re.escape('@nixStoreDir@'.rstrip('/')) + pattern = "(? Date: Fri, 22 Jan 2016 22:37:00 +0100 Subject: [PATCH 2/5] gobject-introspection: Add fallback for libraries After patching up the shared libraries in c420de6 to use absolute paths, there are still some libraries left which do not get an absolute paths assigned. Those libraries are the ones which have an absolute path outside of the Nix store, so we assume that they're build products of the current build and make them absolute by prepending "$out/lib" or "$lib/lib" (depending on whether it's a multiple output derivation or not) to its basename. So for my test case, the resulting library paths now look like this: /nix/store/...-libblockdev-1.3/lib/libblockdev.so.0 /nix/store/...-glibc-2.21/lib/libm.so.6 /nix/store/...-dmraid-1.0.0.rc16/lib/libdmraid.so.1.0.0.rc16 /nix/store/...-libblockdev-1.3/lib/libbd_utils.so.0 Which is perfectly fine and everything gets resolved correctly after importing the library using GI. However, I didn't test it against other libraries and programs, so this still needs testing, especially for Darwin. Signed-off-by: aszlig --- .../absolute_shlib_path.patch | 65 ++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch b/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch index 2803d767dc7..84f303b42a0 100644 --- a/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch +++ b/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch @@ -1,5 +1,55 @@ +diff --git a/giscanner/scannermain.py b/giscanner/scannermain.py +index 89ec193..8ee92c3 100755 +--- a/giscanner/scannermain.py ++++ b/giscanner/scannermain.py +@@ -94,6 +94,34 @@ def get_windows_option_group(parser): + return group + + ++def _get_default_fallback_libpath(): ++ outputs = os.environ.get("outputs", "out").split() ++ if "lib" in outputs: ++ # For multiple output derivations let's try whether there is a $lib ++ # environment variable and use that as the base store path. ++ store_path = os.environ.get("lib") ++ elif "out" in outputs: ++ # Otherwise we have a single output derivation, so the libraries most ++ # certainly will end up in "$out/lib". ++ store_path = os.environ.get("out") ++ ++ if store_path is not None: ++ # Even if we have a $lib as output, there still should be a $lib/lib ++ # directory. ++ return os.path.join(store_path, 'lib') ++ else: ++ # If we haven't found a possible scenario, let's return an empty string ++ # so that the shared library won't be prepended with a path. ++ # ++ # Note that this doesn't mean that all hope is lost, because after all ++ # we can still use --fallback-library-path to set one. ++ # ++ # Also, we're not returning None, because that would make it very ++ # difficult to disable adding fallback paths altogether using something ++ # like: --fallback-library-path="" ++ return "" ++ ++ + def _get_option_parser(): + parser = optparse.OptionParser('%prog [options] sources') + parser.add_option('', "--quiet", +@@ -200,6 +228,10 @@ match the namespace prefix.""") + parser.add_option("", "--filelist", + action="store", dest="filelist", default=[], + help="file containing headers and sources to be scanned") ++ parser.add_option("", "--fallback-library-path", ++ action="store", dest="fallback_libpath", ++ default=_get_default_fallback_libpath(), ++ help="Path to prepend to unknown shared libraries") + + group = get_preprocessor_option_group(parser) + parser.add_option_group(group) diff --git a/giscanner/shlibs.py b/giscanner/shlibs.py -index 838d343..9dbdc0f 100644 +index 838d343..35b6dab 100644 --- a/giscanner/shlibs.py +++ b/giscanner/shlibs.py @@ -53,10 +53,24 @@ def _resolve_libtool(options, binary, libraries): @@ -30,6 +80,19 @@ index 838d343..9dbdc0f 100644 # This is a what we do for non-la files. We assume that we are on an +@@ -115,7 +129,11 @@ def _resolve_non_libtool(options, binary, libraries): + m = pattern.search(line) + if m: + del patterns[library] +- shlibs.append(m.group(1)) ++ match = m.group(1) ++ if not match.startswith('/') \ ++ and len(options.fallback_libpath) > 0: ++ match = os.path.join(options.fallback_libpath, match) ++ shlibs.append(match) + break + + if len(patterns) > 0: diff --git a/giscanner/utils.py b/giscanner/utils.py index 660081e..c9c767a 100644 --- a/giscanner/utils.py From 723f7f8f4fc44f0621227e1feaf61901fca12db9 Mon Sep 17 00:00:00 2001 From: aszlig Date: Fri, 22 Jan 2016 23:55:31 +0100 Subject: [PATCH 3/5] gobject-introspection: Don't hardcode /nix/store If no config.nix.storeDir has been set, don't fall back to "/nix/store" but use builtins.storeDir instead so we always should end up with the correct store path no matter whether config.nix.storeDir has been set. Thanks to @lethalman for pointing this out. Signed-off-by: aszlig --- pkgs/development/libraries/gobject-introspection/default.nix | 2 +- pkgs/top-level/all-packages.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/libraries/gobject-introspection/default.nix b/pkgs/development/libraries/gobject-introspection/default.nix index c2f60d460d7..2567975aa66 100644 --- a/pkgs/development/libraries/gobject-introspection/default.nix +++ b/pkgs/development/libraries/gobject-introspection/default.nix @@ -1,6 +1,6 @@ { stdenv, fetchurl, glib, flex, bison, pkgconfig, libffi, python , libintlOrEmpty, autoconf, automake, otool -, substituteAll, nixStoreDir ? "/nix/store" +, substituteAll, nixStoreDir ? builtins.storeDir }: # now that gobjectIntrospection creates large .gir files (eg gtk3 case) # it may be worth thinking about using multiple derivation outputs diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index b04d2c1457b..8d270ce7f86 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -6640,7 +6640,7 @@ let mpfr = callPackage ../development/libraries/mpfr/default.nix { }; gobjectIntrospection = callPackage ../development/libraries/gobject-introspection { - nixStoreDir = config.nix.storeDir or "/nix/store"; + nixStoreDir = config.nix.storeDir or builtins.storeDir; }; goocanvas = callPackage ../development/libraries/goocanvas { }; From b3b444a79e53d4362a5995d01401b732f0b67eef Mon Sep 17 00:00:00 2001 From: aszlig Date: Sat, 23 Jan 2016 00:14:53 +0100 Subject: [PATCH 4/5] gobject-introspection: Improve comment in patch As the comment needed explanation, that it's about temporary build files, this should do better. Thanks again to @lethalman for pointing that out. Signed-off-by: aszlig --- .../gobject-introspection/absolute_shlib_path.patch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch b/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch index 84f303b42a0..1cec9c1e1a5 100644 --- a/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch +++ b/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch @@ -49,7 +49,7 @@ index 89ec193..8ee92c3 100755 group = get_preprocessor_option_group(parser) parser.add_option_group(group) diff --git a/giscanner/shlibs.py b/giscanner/shlibs.py -index 838d343..35b6dab 100644 +index 838d343..ca7fc0d 100644 --- a/giscanner/shlibs.py +++ b/giscanner/shlibs.py @@ -53,10 +53,24 @@ def _resolve_libtool(options, binary, libraries): @@ -67,8 +67,8 @@ index 838d343..35b6dab 100644 + # First match Nix store paths because they need to be absolute. + (?:%s(?:/[^/]*)+) + # Everything else not a store path remains relative, because we -+ # would end up having absolute build paths in the resulting GIR -+ # file. ++ # would end up with temporary paths that are only valid during ++ # build time in the resulting GIR file. + | (?<=/) + ) + # And finally the library itself: From 740b30b937ee42a3ae3596e6c9ba0a08742db5d9 Mon Sep 17 00:00:00 2001 From: aszlig Date: Sat, 23 Jan 2016 16:24:03 +0100 Subject: [PATCH 5/5] gobject-introspection: Deal with $outputLib Once #7701 gets merged, we have another environment variable called $outputLib, which then points to another environment variable which is the final library output. This was brought up in discussion with @lethalman and @vcunat in: https://github.com/NixOS/nixpkgs/pull/12558#discussion_r50599813 The closure-size branch is not yet merged into master, so this is only a preparation and we're still falling back to $out and $lib whenever $outputLib isn't available. Signed-off-by: aszlig --- .../absolute_shlib_path.patch | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch b/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch index 1cec9c1e1a5..49e059befdb 100644 --- a/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch +++ b/pkgs/development/libraries/gobject-introspection/absolute_shlib_path.patch @@ -1,21 +1,26 @@ diff --git a/giscanner/scannermain.py b/giscanner/scannermain.py -index 89ec193..8ee92c3 100755 +index 89ec193..54f1d2e 100755 --- a/giscanner/scannermain.py +++ b/giscanner/scannermain.py -@@ -94,6 +94,34 @@ def get_windows_option_group(parser): +@@ -94,6 +94,39 @@ def get_windows_option_group(parser): return group +def _get_default_fallback_libpath(): -+ outputs = os.environ.get("outputs", "out").split() -+ if "lib" in outputs: -+ # For multiple output derivations let's try whether there is a $lib -+ # environment variable and use that as the base store path. -+ store_path = os.environ.get("lib") -+ elif "out" in outputs: -+ # Otherwise we have a single output derivation, so the libraries most -+ # certainly will end up in "$out/lib". -+ store_path = os.environ.get("out") ++ # Newer multiple-output-optimized stdenv has an environment variable ++ # $outputLib which in turn specifies another variable which then is used as ++ # the destination for the library contents (${!outputLib}/lib). ++ store_path = os.environ.get(os.environ.get("outputLib")) ++ if store_path is None: ++ outputs = os.environ.get("outputs", "out").split() ++ if "lib" in outputs: ++ # For multiple output derivations let's try whether there is a $lib ++ # environment variable and use that as the base store path. ++ store_path = os.environ.get("lib") ++ elif "out" in outputs: ++ # Otherwise we have a single output derivation, so the libraries ++ # most certainly will end up in "$out/lib". ++ store_path = os.environ.get("out") + + if store_path is not None: + # Even if we have a $lib as output, there still should be a $lib/lib @@ -37,7 +42,7 @@ index 89ec193..8ee92c3 100755 def _get_option_parser(): parser = optparse.OptionParser('%prog [options] sources') parser.add_option('', "--quiet", -@@ -200,6 +228,10 @@ match the namespace prefix.""") +@@ -200,6 +233,10 @@ match the namespace prefix.""") parser.add_option("", "--filelist", action="store", dest="filelist", default=[], help="file containing headers and sources to be scanned")