resholve: init at 0.4.0 (#85827)

resholve: init at 0.4.0

resholve attempts to resolve executables in shell scripts.
Includes Nix builder for resolving dependencies in Nix-built
shell projects.

pkgs/development/misc/resholve/README.md

@@ -0,0 +1,138 @@
+# Using resholve's Nix API
+
+resholve converts bare executable references in shell scripts to absolute
+paths. This will hopefully make its way into the Nixpkgs manual soon, but
+until then I'll outline how to use the `resholvePackage` function.
+
+> Fair warning: resholve does *not* aspire to resolving all valid Shell
+> scripts. It depends on the OSH/Oil parser, which aims to support most (but
+> not all) Bash, and aims to be a ~90% sort of solution.
+
+Let's start with a simple example from one of my own projects:
+
+```nix
+{ stdenv, lib, resholvePackage, fetchFromGitHub, bashup-events44, bashInteractive_5, doCheck ? true, shellcheck }:
+
+resholvePackage rec {
+  pname = "shellswain";
+  version = "unreleased";
+
+  src = fetchFromGitHub {
+    # ...
+  };
+
+  solutions = {
+    profile = {
+      # the only *required* arguments
+      scripts = [ "bin/shellswain.bash" ];
+      interpreter = "none";
+      inputs = [ bashup-events44 ];
+    };
+  };
+
+  makeFlags = [ "prefix=${placeholder "out"}" ];
+
+  inherit doCheck;
+  checkInputs = [ shellcheck ];
+
+  # ...
+}
+```
+
+I'll focus on the `solutions` attribute, since this is the only part
+that differs from other derivations.
+
+Each "solution" (k=v pair)
+describes one resholve invocation. For most shell packages, one
+invocation will probably be enough. resholve will make you be very
+explicit about your script's dependencies, and it may also need your
+help sorting out some references or problems that it can't safely
+handle on its own.
+
+If you have more than one script, and your scripts need conflicting
+directives, you can specify more than one solution to resolve the
+scripts separately, but still produce a single package.
+
+Let's take a closer look:
+
+```nix
+  solutions = {
+    # each solution has a short name; this is what you'd use to
+    # override the settings of this solution, and it may also show up
+    # in (some) error messages.
+    profile = {
+      # specify one or more $out-relative script paths (unlike many
+      # builders, resholve will modify the output files during fixup
+      # to correctly resolve scripts that source within the package)
+      scripts = [ "bin/shellswain.bash" ];
+      # "none" for no shebang, "${bash}/bin/bash" for bash, etc.
+      interpreter = "none";
+      # packages resholve should resolve executables from
+      inputs = [ bashup-events44 ];
+    };
+  };
+```
+
+resholve has a (growing) number of options for handling more complex
+scripts. I won't cover these in excruciating detail here. You can find
+more information about these in `man resholve` via `nixpkgs.resholve`.
+
+Instead, we'll look at the general form of the solutions attrset:
+
+```nix
+solutions = {
+  shortname = {
+    # required
+    # $out-relative paths to try resolving
+    scripts = [ "bin/shunit2" ];
+    # packages to resolve executables from
+    inputs = [ coreutils gnused gnugrep findutils ];
+    # path for shebang, or 'none' to omit shebang
+    interpreter = "${bash}/bin/bash";
+
+    # optional
+    fake = { fake directives };
+    fix = { fix directives };
+    keep = { keep directives };
+    # file to inject before first code-line of script
+    prologue = file;
+    # file to inject after last code-line of script
+    epilogue = file;
+    # extra command-line flags passed to resholve; generally this API
+    # should align with what resholve supports, but flags may help if
+    # you need to override the version of resholve.
+    flags = [ ];
+  };
+};
+```
+
+The main way you'll adjust how resholve handles your scripts are the
+fake, fix, and keep directives. The manpage covers their purpose and
+how to format them on the command-line, so I'll focus on how you'll
+need to translate them into Nix types.
+
+```nix
+# --fake 'f:setUp;tearDown builtin:setopt source:/etc/bashrc'
+fake = {
+    function = [ "setUp" "tearDown" ];
+    builtin = [ "setopt" ];
+    source = [ "/etc/bashrc" ];
+};
+
+# --fix 'aliases xargs:ls $GIT:gix'
+fix = {
+    # all single-word directives use `true` as value
+    aliases = true;
+    xargs = [ "ls" ];
+    "$GIT" = [ "gix" ];
+};
+
+# --keep 'which:git;ls .:$HOME $LS:exa /etc/bashrc ~/.bashrc'
+keep = {
+    which = [ "git" "ls" ];
+    "." = [ "$HOME" ];
+    "$LS" = [ "exa" ];
+    "/etc/bashrc" = true;
+    "~/.bashrc" = true;
+};
+```

pkgs/development/misc/resholve/default.nix

@@ -0,0 +1,9 @@
+{ callPackage
+, doCheck ? true
+}:
+
+rec {
+  resholve = callPackage ./resholve.nix { inherit doCheck; };
+  resholvePackage =
+    callPackage ./resholve-package.nix { inherit resholve; };
+}

pkgs/development/misc/resholve/deps.nix

@@ -0,0 +1,120 @@
+{ stdenv
+, python27Packages
+, fetchFromGitHub
+, makeWrapper
+, # re2c deps
+  autoreconfHook
+, # py-yajl deps
+  git
+, # oil deps
+  readline
+, cmark
+, file
+, glibcLocales
+, oilPatches ? [ ]
+}:
+
+/*
+Notes on specific dependencies:
+- if/when python2.7 is removed from nixpkgs, this may need to figure
+  out how to build oil's vendored python2
+- I'm not sure if glibcLocales is worth the addition here. It's to fix
+  a libc test oil runs. My oil fork just disabled the libc tests, but
+  I haven't quite decided if that's the right long-term call, so I
+  didn't add a patch for it here yet.
+*/
+
+rec {
+  # had to add this as well; 1.3 causes a break here; sticking
+  # to oil's official 1.0.3 dep for now.
+  re2c = stdenv.mkDerivation rec {
+    pname = "re2c";
+    version = "1.0.3";
+    sourceRoot = "${src.name}/re2c";
+    src = fetchFromGitHub {
+      owner = "skvadrik";
+      repo = "re2c";
+      rev = version;
+      sha256 = "0grx7nl9fwcn880v5ssjljhcb9c5p2a6xpwil7zxpmv0rwnr3yqi";
+    };
+    nativeBuildInputs = [ autoreconfHook ];
+    preCheck = ''
+      patchShebangs run_tests.sh
+    '';
+  };
+
+  py-yajl = python27Packages.buildPythonPackage rec {
+    pname = "oil-pyyajl-unstable";
+    version = "2019-12-05";
+    src = fetchFromGitHub {
+      owner = "oilshell";
+      repo = "py-yajl";
+      rev = "eb561e9aea6e88095d66abcc3990f2ee1f5339df";
+      sha256 = "17hcgb7r7cy8r1pwbdh8di0nvykdswlqj73c85k6z8m0filj3hbh";
+      fetchSubmodules = true;
+    };
+    # just for submodule IIRC
+    nativeBuildInputs = [ git ];
+  };
+
+  # resholve's primary dependency is this developer build of the oil shell.
+  oildev = python27Packages.buildPythonPackage rec {
+    pname = "oildev-unstable";
+    version = "2020-03-31";
+
+    src = fetchFromGitHub {
+      owner = "oilshell";
+      repo = "oil";
+      rev = "ea80cdad7ae1152a25bd2a30b87fe3c2ad32394a";
+      sha256 = "0pxn0f8qbdman4gppx93zwml7s5byqfw560n079v68qjgzh2brq2";
+
+      /*
+      It's not critical to drop most of these; the primary target is
+      the vendored fork of Python-2.7.13, which is ~ 55M and over 3200
+      files, dozens of which get interpreter script patches in fixup.
+      */
+      extraPostFetch = ''
+        rm -rf Python-2.7.13 benchmarks metrics py-yajl rfc gold web testdata services demo devtools cpp
+      '';
+    };
+
+    # TODO: not sure why I'm having to set this for nix-build...
+    #       can anyone tell if I'm doing something wrong?
+    SOURCE_DATE_EPOCH = 315532800;
+
+    # These aren't, strictly speaking, nix/nixpkgs specific, but I've
+    # had hell upstreaming them. Pulling from resholve source and
+    # passing in from resholve.nix
+    patches = oilPatches;
+
+    buildInputs = [ readline cmark py-yajl ];
+
+    nativeBuildInputs = [ re2c file makeWrapper ];
+
+    propagatedBuildInputs = with python27Packages; [ six typing ];
+
+    doCheck = true;
+
+    preBuild = ''
+      build/dev.sh all
+    '';
+
+    postPatch = ''
+      patchShebangs asdl build core doctools frontend native oil_lang
+    '';
+
+    _NIX_SHELL_LIBCMARK = "${cmark}/lib/libcmark${stdenv.hostPlatform.extensions.sharedLibrary}";
+
+    # See earlier note on glibcLocales
+    LOCALE_ARCHIVE = stdenv.lib.optionalString (stdenv.buildPlatform.libc == "glibc") "${glibcLocales}/lib/locale/locale-archive";
+
+    meta = {
+      description = "A new unix shell";
+      homepage = "https://www.oilshell.org/";
+      license = with stdenv.lib.licenses; [
+        psfl # Includes a portion of the python interpreter and standard library
+        asl20 # Licence for Oil itself
+      ];
+    };
+  };
+}

pkgs/development/misc/resholve/resholve-package.nix

@@ -0,0 +1,97 @@
+{ stdenv, lib, resholve }:
+
+{ pname
+, src
+, version
+, passthru ? { }
+, solutions
+, ...
+}@attrs:
+let
+  inherit stdenv;
+  /* These functions break up the work of partially validating the
+   * 'solutions' attrset and massaging it into env/cli args.
+   *
+   * Note: some of the left-most args do not *have* to be passed as
+   * deep as they are, but I've done so to provide more error context
+   */
+
+  # for brevity / line length
+  spaces = l: builtins.concatStringsSep " " l;
+  semicolons = l: builtins.concatStringsSep ";" l;
+
+  /* Throw a fit with dotted attr path context */
+  nope = path: msg:
+    throw "${builtins.concatStringsSep "." path}: ${msg}";
+
+  /* Special-case directive value representations by type */
+  makeDirective = solution: env: name: val:
+    if builtins.isInt val then builtins.toString val
+    else if builtins.isString val then name
+    else if true == val then name
+    else if false == val then "" # omit!
+    else if null == val then "" # omit!
+    else if builtins.isList val then "${name}:${semicolons val}"
+    else nope [ solution env name ] "unexpected type: ${builtins.typeOf val}";
+
+  /* Build fake/fix/keep directives from Nix types */
+  makeDirectives = solution: env: val:
+    lib.mapAttrsToList (makeDirective solution env) val;
+
+  /* Special-case value representation by type/name */
+  makeEnvVal = solution: env: val:
+    if env == "inputs" then lib.makeBinPath val
+    else if builtins.isString val then val
+    else if builtins.isList val then spaces val
+    else if builtins.isAttrs val then spaces (makeDirectives solution env val)
+    else nope [ solution env ] "unexpected type: ${builtins.typeOf val}";
+
+  /* Shell-format each env value */
+  shellEnv = solution: env: value:
+    lib.escapeShellArg (makeEnvVal solution env value);
+
+  /* Build a single ENV=val pair */
+  makeEnv = solution: env: value:
+    "RESHOLVE_${lib.toUpper env}=${shellEnv solution env value}";
+
+  /* Discard attrs claimed by makeArgs */
+  removeCliArgs = value:
+    removeAttrs value [ "scripts" "flags" ];
+
+  /* Verify required arguments are present */
+  validateSolution = { scripts, inputs, interpreter, ... }: true;
+
+  /* Pull out specific solution keys to build ENV=val pairs */
+  makeEnvs = solution: value:
+    spaces (lib.mapAttrsToList (makeEnv solution) (removeCliArgs value));
+
+  /* Pull out specific solution keys to build CLI argstring */
+  makeArgs = { flags ? [ ], scripts, ... }:
+    spaces (flags ++ scripts);
+
+  /* Build a single resholve invocation */
+  makeInvocation = solution: value:
+    if validateSolution value then
+      "${makeEnvs solution value} resholve --overwrite ${makeArgs value}"
+    else throw "invalid solution"; # shouldn't trigger for now
+
+  /* Build resholve invocation for each solution. */
+  makeCommands = solutions:
+    lib.mapAttrsToList makeInvocation solutions;
+
+  self = (stdenv.mkDerivation ((removeAttrs attrs [ "solutions" ])
+    // {
+    inherit pname version src;
+    buildInputs = [ resholve ];
+
+    # enable below for verbose debug info if needed
+    # supports default python.logging levels
+    # LOGLEVEL="INFO";
+    preFixup = ''
+      pushd "$out"
+      ${builtins.concatStringsSep "\n" (makeCommands solutions)}
+      popd
+    '';
+  }));
+in
+lib.extendDerivation true passthru self

pkgs/development/misc/resholve/resholve.nix

@@ -0,0 +1,74 @@
+{ stdenv
+, callPackage
+, python27Packages
+, installShellFiles
+, fetchFromGitHub
+, file
+, findutils
+, gettext
+, bats
+, bash
+, doCheck ? true
+}:
+let
+  version = "0.4.0";
+  rSrc = fetchFromGitHub {
+    owner = "abathur";
+    repo = "resholve";
+    rev = "v${version}";
+    hash = "sha256-wfxcX3wMZqoi5bWjXYRa21UDDJmTDfE+21p4mL2IJog=";
+  };
+  deps = callPackage ./deps.nix {
+    /*
+    resholve needs to patch Oil, but trying to avoid adding
+    them all *to* nixpkgs, since they aren't specific to
+    nix/nixpkgs.
+    */
+    oilPatches = [
+      "${rSrc}/0001-add_setup_py.patch"
+      "${rSrc}/0002-add_MANIFEST_in.patch"
+      "${rSrc}/0003-fix_codegen_shebang.patch"
+      "${rSrc}/0004-disable-internal-py-yajl-for-nix-built.patch"
+    ];
+  };
+in
+python27Packages.buildPythonApplication {
+  pname = "resholve";
+  inherit version;
+  src = rSrc;
+  format = "other";
+
+  nativeBuildInputs = [ installShellFiles ];
+
+  propagatedBuildInputs = [ deps.oildev python27Packages.ConfigArgParse ];
+
+  patchPhase = ''
+    for file in resholve; do
+      substituteInPlace $file --subst-var-by version ${version}
+    done
+  '';
+
+  installPhase = ''
+    install -Dm755 resholve $out/bin/resholve
+    installManPage resholve.1
+  '';
+
+  inherit doCheck;
+  checkInputs = [ bats ];
+  RESHOLVE_PATH = "${stdenv.lib.makeBinPath [ file findutils gettext ]}";
+
+  checkPhase = ''
+    # explicit interpreter for test suite
+    export INTERP="${bash}/bin/bash" PATH="$out/bin:$PATH"
+    patchShebangs .
+    ./test.sh
+  '';
+
+  meta = with stdenv.lib; {
+    description = "Resolve external shell-script dependencies";
+    homepage = "https://github.com/abathur/resholve";
+    license = with licenses; [ mit ];
+    maintainers = with maintainers; [ abathur ];
+    platforms = platforms.all;
+  };
+}

pkgs/top-level/all-packages.nix

@@ -7099,6 +7099,9 @@ in
 
   rescuetime = libsForQt5.callPackage ../applications/misc/rescuetime { };
 
+  inherit (callPackage ../development/misc/resholve { })
+    resholve resholvePackage;
+
   reuse = callPackage ../tools/package-management/reuse { };
 
   rewritefs = callPackage ../os-specific/linux/rewritefs { };