Merge #33057: stdenv meta checks: make them lazy

Closes #22277 - it's superseded;  I have some WIP on evaluation
performance, but best do that in a separate PR/thread.

lib/customisation.nix

@@ -36,7 +36,7 @@ rec {
   overrideDerivation = drv: f:
     let
       newDrv = derivation (drv.drvAttrs // (f drv));
-    in addPassthru newDrv (
+    in lib.flip (extendDerivation true) newDrv (
       { meta = drv.meta or {};
         passthru = if drv ? passthru then drv.passthru else {};
       }
@@ -131,8 +131,8 @@ rec {
 
 
   /* Add attributes to each output of a derivation without changing
-     the derivation itself. */
+     the derivation itself and check a given condition when evaluating. */
-  addPassthru = drv: passthru:
+  extendDerivation = condition: passthru: drv:
     let
       outputs = drv.outputs or [ "out" ];
 
@@ -142,13 +142,23 @@ rec {
       outputToAttrListElement = outputName:
         { name = outputName;
           value = commonAttrs // {
-            inherit (drv.${outputName}) outPath drvPath type outputName;
+            inherit (drv.${outputName}) type outputName;
+            drvPath = assert condition; drv.${outputName}.drvPath;
+            outPath = assert condition; drv.${outputName}.outPath;
           };
         };
 
       outputsList = map outputToAttrListElement outputs;
-  in commonAttrs // { outputUnspecified = true; };
+    in commonAttrs // {
+      outputUnspecified = true;
+      drvPath = assert condition; drv.drvPath;
+      outPath = assert condition; drv.outPath;
+    };
 
+  /* Add attributes to each output of a derivation without changing
+     the derivation itself. */
+  addPassthru = lib.warn "`addPassthru` is deprecated, replace with `extendDerivation true`"
+                         (extendDerivation true);
 
   /* Strip a derivation of all non-essential attributes, returning
      only those needed by hydra-eval-jobs. Also strictly evaluate the

lib/default.nix

@@ -87,7 +87,8 @@ let
     inherit (stringsWithDeps) textClosureList textClosureMap
       noDepEntry fullDepEntry packEntry stringAfter;
     inherit (customisation) overrideDerivation makeOverridable
-      callPackageWith callPackagesWith addPassthru hydraJob makeScope;
+      callPackageWith callPackagesWith extendDerivation addPassthru
+      hydraJob makeScope;
     inherit (meta) addMetaAttrs dontDistribute setName updateName
       appendToName mapDerivationAttrset lowPrio lowPrioSet hiPrio
       hiPrioSet;

nixos/doc/manual/release-notes/rl-1803.xml

@@ -113,7 +113,7 @@ following incompatible changes:</para>
   </listitem>
   <listitem>
     <para>
-      <literal>cc-wrapper</literal>has been split in two; there is now also a <literal>bintools-wrapper</literal>.
+      <literal>cc-wrapper</literal> has been split in two; there is now also a <literal>bintools-wrapper</literal>.
       The most commonly used files in <filename>nix-support</filename> are now split between the two wrappers.
       Some commonly used ones, like <filename>nix-support/dynamic-linker</filename>, are duplicated for backwards compatability, even though they rightly belong only in <literal>bintools-wrapper</literal>.
       Other more obscure ones are just moved.
@@ -131,6 +131,11 @@ following incompatible changes:</para>
       Other types dependencies should be unaffected.
     </para>
   </listitem>
+  <listitem>
+    <para>
+      <literal>lib.addPassthru</literal> is removed.  Use <literal>lib.extendDerivation true</literal> instead.  <emphasis role="strong">TODO: actually remove it before branching 18.03 off.</emphasis>
+    </para>
+  </listitem>
   <listitem>
     <para>
       The <literal>memcached</literal> service no longer accept dynamic socket

pkgs/os-specific/linux/kernel/generic.nix

@@ -134,10 +134,12 @@ let
     passthru = kernel.passthru // (removeAttrs passthru [ "passthru" ]);
   };
 
-  nativeDrv = lib.addPassthru kernel.nativeDrv passthru;
+  addPassthru' = lib.extendDerivation true passthru;
 
-  crossDrv = lib.addPassthru kernel.crossDrv passthru;
+  nativeDrv = addPassthru' kernel.nativeDrv;
+
+  crossDrv = addPassthru' kernel.crossDrv;
 
 in if kernel ? crossDrv
    then nativeDrv // { inherit nativeDrv crossDrv; }
-   else lib.addPassthru kernel passthru
+   else addPassthru' kernel

pkgs/stdenv/generic/check-meta.nix

@@ -1,6 +1,5 @@
-# Extend a derivation with checks for brokenness, license, etc.  Throw a
+# Checks derivation meta and attrs for problems (like brokenness,
-# descriptive error when the check fails; return `derivationArg` otherwise.
+# licenses, etc).
-# Note: no dependencies are checked in this step.
 
 { lib, config, system, meta, derivationArg, mkDerivationArg }:
 
@@ -154,12 +153,14 @@ let
 
     # Weirder stuff that doesn't appear in the documentation?
     knownVulnerabilities = listOf str;
+    name = str;
     version = str;
     tag = str;
     updateWalker = bool;
     executables = listOf str;
     outputsToInstall = listOf str;
     position = str;
+    evaluates = bool;
     repositories = attrsOf str;
     isBuildPythonPackage = platforms;
     schedulingPriority = int;
@@ -196,13 +197,11 @@ let
       { valid = false; reason = "unknown-meta"; errormsg = "has an invalid meta attrset:${lib.concatMapStrings (x: "\n\t - " + x) res}"; }
     else { valid = true; };
 
-  # Throw an error if trying to evaluate an non-valid derivation
+   validity = checkValidity attrs;
-  validityCondition =
-         let v = checkValidity attrs;
-         in if !v.valid
-           then handleEvalIssue (removeAttrs v ["valid"])
-           else true;
 
-in
+in validity // {
-  assert validityCondition;
+  # Throw an error if trying to evaluate an non-valid derivation
-  derivationArg
+  handled = if !validity.valid
+    then handleEvalIssue (removeAttrs validity ["valid"])
+    else true;
+}

pkgs/stdenv/generic/make-derivation.nix

@@ -81,6 +81,9 @@ rec {
       inherit erroneousHardeningFlags hardeningDisable hardeningEnable supportedHardeningFlags;
     })
     else let
+      references = nativeBuildInputs ++ buildInputs
+                ++ propagatedNativeBuildInputs ++ propagatedBuildInputs;
+
       dependencies = map (map lib.chooseDevOutputs) [
         [
           (map (drv: drv.__spliced.buildBuild or drv) depsBuildBuild)
@@ -140,9 +143,12 @@ rec {
               (lib.concatLists propagatedDependencies));
         in
         {
-          name = name + lib.optionalString
+          # A hack to make `nix-env -qa` and `nix search` ignore broken packages.
+          # TODO(@oxij): remove this assert when something like NixOS/nix#1771 gets merged into nix.
+          name = assert validity.handled; name + lib.optionalString
             (stdenv.hostPlatform != stdenv.buildPlatform)
             ("-" + stdenv.hostPlatform.config);
+
           builder = attrs.realBuilder or stdenv.shell;
           args = attrs.args or ["-e" (attrs.builder or ./default-builder.sh)];
           inherit stdenv;
@@ -197,46 +203,59 @@ rec {
           doInstallCheck = doInstallCheck && (stdenv.hostPlatform == stdenv.buildPlatform);
         });
 
+      validity = import ./check-meta.nix {
+        inherit lib config meta derivationArg;
+        mkDerivationArg = attrs;
+        # Nix itself uses the `system` field of a derivation to decide where
+        # to build it. This is a bit confusing for cross compilation.
+        inherit (stdenv) system;
+      };
+
       # The meta attribute is passed in the resulting attribute set,
       # but it's not part of the actual derivation, i.e., it's not
       # passed to the builder and is not a dependency.  But since we
       # include it in the result, it *is* available to nix-env for queries.
-      meta = { }
+      meta = {
+          # `name` above includes cross-compilation cruft (and is under assert),
+          # lets have a clean always accessible version here.
+          inherit name;
+
           # If the packager hasn't specified `outputsToInstall`, choose a default,
           # which is the name of `p.bin or p.out or p`;
           # if he has specified it, it will be overridden below in `// meta`.
           #   Note: This default probably shouldn't be globally configurable.
           #   Services and users should specify outputs explicitly,
           #   unless they are comfortable with this default.
-        // { outputsToInstall =
+          outputsToInstall =
-          let
+            let
-            outs = outputs'; # the value passed to derivation primitive
+              outs = outputs'; # the value passed to derivation primitive
-            hasOutput = out: builtins.elem out outs;
+              hasOutput = out: builtins.elem out outs;
-          in [( lib.findFirst hasOutput null (["bin" "out"] ++ outs) )];
+            in [( lib.findFirst hasOutput null (["bin" "out"] ++ outs) )];
         }
         // attrs.meta or {}
-          # Fill `meta.position` to identify the source location of the package.
+        # Fill `meta.position` to identify the source location of the package.
-        // lib.optionalAttrs (pos != null)
+        // lib.optionalAttrs (pos != null) {
-          { position = pos.file + ":" + toString pos.line; }
+          position = pos.file + ":" + toString pos.line;
-        ;
+        # Expose the result of the checks for everyone to see.
+        } // {
+          evaluates = validity.valid
+                   && (if config.checkMetaRecursively or false
+                       then lib.all (d: d.meta.evaluates or true) references
+                       else true);
+        };
 
     in
 
-      lib.addPassthru
+      lib.extendDerivation
-        (derivation (import ./check-meta.nix
+        validity.handled
-          {
+        ({
-            inherit lib config meta derivationArg;
+           overrideAttrs = f: mkDerivation (attrs // (f attrs));
-            mkDerivationArg = attrs;
+           inherit meta passthru;
-            # Nix itself uses the `system` field of a derivation to decide where
+         } //
-            # to build it. This is a bit confusing for cross compilation.
+         # Pass through extra attributes that are not inputs, but
-            inherit (stdenv) system;
+         # should be made available to Nix expressions using the
-          }))
+         # derivation (e.g., in assertions).
-        ( {
+         passthru)
-            overrideAttrs = f: mkDerivation (attrs // (f attrs));
+        (derivation derivationArg);
-            inherit meta passthru;
+
-          } //
-          # Pass through extra attributes that are not inputs, but
-          # should be made available to Nix expressions using the
-          # derivation (e.g., in assertions).
-          passthru);
 }