Merge pull request #17749 from abbradar/python-set

wrapPythonProgram: use site.addsitedir instead of PYTHONPATH

doc/languages-frameworks/python.md

@@ -503,9 +503,12 @@ and can be used as:
 
 The `buildPythonPackage` mainly does four things:
 
-* In the `buildPhase`, it calls `${python.interpreter} setup.py bdist_wheel` to build a wheel binary zipfile.
+* In the `buildPhase`, it calls `${python.interpreter} setup.py bdist_wheel` to
+  build a wheel binary zipfile.
 * In the `installPhase`, it installs the wheel file using `pip install *.whl`.
-* In the `postFixup` phase, the `wrapPythonPrograms` bash function is called to wrap all programs in the `$out/bin/*` directory to include `$PYTHONPATH` and `$PATH` environment variables.
+* In the `postFixup` phase, the `wrapPythonPrograms` bash function is called to
+  wrap all programs in the `$out/bin/*` directory to include `$PATH`
+  environment variable and add dependent libraries to script's `sys.path`.
 * In the `installCheck` phase, `${python.interpreter} setup.py test` is ran.
 
 As in Perl, dependencies on other Python packages can be specified in the

pkgs/applications/networking/instant-messengers/gajim/default.nix

@@ -39,7 +39,6 @@ stdenv.mkDerivation rec {
   postPatch = ''
     sed -i -e '0,/^[^#]/ {
       /^[^#]/i export \\\
-        PYTHONPATH="'"$PYTHONPATH\''${PYTHONPATH:+:}\$PYTHONPATH"'" \\\
         GST_PLUGIN_PATH="'"\$GST_PLUGIN_PATH''${GST_PLUGIN_PATH:+:}${""
         }$GST_PLUGIN_PATH"'"
     }' scripts/gajim.in
@@ -53,21 +52,38 @@ stdenv.mkDerivation rec {
   '';
 
   buildInputs = [
-    python intltool pkgconfig libX11
+    python libX11
+  ] ++ optionals enableJingle [ farstream gst_plugins_bad libnice ];
+
+  nativeBuildInputs = [
+    pythonPackages.wrapPython intltool pkgconfig
+  ];
+
+  propagatedBuildInputs = [
     pythonPackages.pygobject pythonPackages.pyGtkGlade
     pythonPackages.sqlite3 pythonPackages.pyasn1
     pythonPackages.pyxdg
     pythonPackages.nbxmpp
     pythonPackages.pyopenssl pythonPackages.dbus-python
-  ] ++ optionals enableJingle [ farstream gst_plugins_bad libnice ]
+  ] ++ optional enableE2E pythonPackages.pycrypto
-    ++ optional enableE2E pythonPackages.pycrypto
     ++ optional enableRST pythonPackages.docutils
     ++ optional enableNotifications pythonPackages.notify
     ++ extraPythonPackages pythonPackages;
 
-  postInstall = ''
+  postFixup = ''
     install -m 644 -t "$out/share/gajim/icons/hicolor" \
                       "icons/hicolor/index.theme"
+
+    buildPythonPath "$out"
+
+    for i in $out/bin/*; do
+      name="$(basename "$i")"
+      if [ "$name" = "gajim-history-manager" ]; then
+        name="history_manager"
+      fi
+
+      patchPythonScript "$out/share/gajim/src/$name.py"
+    done
   '';
 
   enableParallelBuilding = true;

pkgs/development/python-modules/generic/wrap.sh

@@ -4,6 +4,38 @@ wrapPythonPrograms() {
     wrapPythonProgramsIn $out "$out $pythonPath"
 }
 
+# Builds environment variables like PYTHONPATH and PATH walking through closure
+# of dependencies.
+buildPythonPath() {
+    local pythonPath="$1"
+    local python="@executable@"
+    local path
+
+    # Create an empty table of python paths (see doc on _addToPythonPath
+    # for how this is used). Build up the program_PATH and program_PYTHONPATH
+    # variables.
+    declare -A pythonPathsSeen=()
+    program_PYTHONPATH=
+    program_PATH=
+    pythonPathsSeen["@python@"]=1
+    addToSearchPath program_PATH @python@/bin
+    for path in $pythonPath; do
+        _addToPythonPath $path
+    done
+}
+
+# Patches a Python script so that it has correct libraries path and executable
+# name.
+patchPythonScript() {
+    local f="$1"
+
+    # The magicalSedExpression will invoke a "$(basename "$f")", so
+    # if you change $f to something else, be sure to also change it
+    # in pkgs/top-level/python-packages.nix!
+    # It also uses $program_PYTHONPATH.
+    sed -i "$f" -re '@magicalSedExpression@'
+}
+
 # Transforms any binaries generated by the setup.py script, replacing them
 # with an executable shell script which will set some environment variables
 # and then call into the original binary (which has been given a .wrapped
@@ -11,19 +43,9 @@ wrapPythonPrograms() {
 wrapPythonProgramsIn() {
     local dir="$1"
     local pythonPath="$2"
-    local python="@executable@"
-    local path
     local f
 
-    # Create an empty table of python paths (see doc on _addToPythonPath
+    buildPythonPath "$pythonPath"
-    # for how this is used). Build up the program_PATH and program_PYTHONPATH
-    # variables.
-    declare -A pythonPathsSeen=()
-    program_PYTHONPATH=
-    program_PATH=
-    for path in $pythonPath; do
-        _addToPythonPath $path
-    done
 
     # Find all regular files in the output directory that are executable.
     for f in $(find "$dir" -type f -perm -0100); do
@@ -39,16 +61,12 @@ wrapPythonProgramsIn() {
             # dont wrap EGG-INFO scripts since they are called from python
             if echo "$f" | grep -qv EGG-INFO/scripts; then
                 echo "wrapping \`$f'..."
-                # The magicalSedExpression will invoke a "$(basename "$f")", so
+                patchPythonScript "$f"
-                # if you change $f to something else, be sure to also change it
-                # in pkgs/top-level/python-packages.nix!
-                sed -i "$f" -re '@magicalSedExpression@'
                 # wrapProgram creates the executable shell script described
                 # above. The script will set PYTHONPATH and PATH variables.!
                 # (see pkgs/build-support/setup-hooks/make-wrapper.sh)
                 local -a wrap_args=("$f"
-                                 --prefix PYTHONPATH ':' "$program_PYTHONPATH"
+                                 --prefix PATH ':' "$program_PATH")
-                                 --prefix PATH ':' "$program_PATH:$dir/bin")
 
                 # Add any additional arguments provided by makeWrapperArgs
                 # argument to buildPythonPackage.

pkgs/top-level/python-packages.nix

@@ -42,6 +42,7 @@ in modules // {
     { deps = pkgs.makeWrapper;
       substitutions.libPrefix = python.libPrefix;
       substitutions.executable = python.interpreter;
+      substitutions.python = python;
       substitutions.magicalSedExpression = let
         # Looks weird? Of course, it's between single quoted shell strings.
         # NOTE: Order DOES matter here, so single character quotes need to be
@@ -53,19 +54,31 @@ in modules // {
           isSingle = elem quote [ "\"" "'\"'\"'" ];
           endQuote = if isSingle then "[^\\\\]${quote}" else quote;
         in ''
-          /^ *[a-z]?${quote}/ {
+          /^[a-z]?${quote}/ {
             /${quote}${quote}|${quote}.*${endQuote}/{n;br}
             :${label}; n; /^${quote}/{n;br}; /${endQuote}/{n;br}; b${label}
           }
         '';
 
+        # This preamble does two things:
+        # * Sets argv[0] to the original application's name; otherwise it would be .foo-wrapped.
+        #   Python doesn't support `exec -a`.
+        # * Adds all required libraries to sys.path via `site.addsitedir`. It also handles *.pth files.
+        preamble = ''
+          import sys
+          import site
+          import functools
+          sys.argv[0] = '"'$(basename "$f")'"'
+          functools.reduce(lambda k, p: site.addsitedir(p, k), ['"$([ -n "$program_PYTHONPATH" ] && (echo "'$program_PYTHONPATH'" | sed "s|:|','|g") || true)"'], site._init_pathinfo())
+        '';
+
       in ''
         1 {
-          /^#!/!b; :r
+          :r
-          /\\$/{N;br}
+          /\\$|,$/{N;br}
-          /__future__|^ *(#.*)?$/{n;br}
+          /__future__|^ |^ *(#.*)?$/{n;br}
           ${concatImapStrings mkStringSkipper quoteVariants}
-          /^ *[^# ]/i import sys; sys.argv[0] = '"'$(basename "$f")'"'
+          /^[^# ]/i ${replaceStrings ["\n"] [";"] preamble}
         }
       '';
     }
@@ -8521,20 +8534,7 @@ in modules // {
     installPhase = ''
       mkdir -p $out/${python.sitePackages}
       cp -r scfbuild $out/${python.sitePackages}
-      # Workaround for #16133
+      cp -r bin $out
-      mkdir -p $out/bin
-
-      cat >$out/bin/scfbuild <<EOF
-      #!/usr/bin/env python2
-
-      import sys
-      from scfbuild.main import main
-
-      if __name__ == '__main__':
-        sys.exit(main())
-      EOF
-
-      chmod +x $out/bin/scfbuild
     '';
 
     meta = with stdenv.lib; {