Merge pull request #34351 from teto/kernel_wip

[RDY] buildLinux: Add more overrides

pkgs/os-specific/linux/kernel/common-config.nix

@@ -16,7 +16,7 @@
 
 */
 
-{ stdenv, version, kernelPlatform, extraConfig, features }:
+{ stdenv, version, extraConfig, features }:
 
 with stdenv.lib;
 
@@ -682,6 +682,5 @@ with stdenv.lib;
     WW_MUTEX_SELFTEST? n
   ''}
 
-  ${kernelPlatform.kernelExtraConfig or ""}
   ${extraConfig}
 ''

pkgs/os-specific/linux/kernel/generate-config.pl

@@ -13,18 +13,18 @@ use strict;
 use IPC::Open2;
 use Cwd;
 
-my $wd = getcwd;
+# exported via nix
-
 my $debug = $ENV{'DEBUG'};
 my $autoModules = $ENV{'AUTO_MODULES'};
 my $preferBuiltin = $ENV{'PREFER_BUILTIN'};
-    
+my $ignoreConfigErrors = $ENV{'ignoreConfigErrors'};
+my $buildRoot = $ENV{'BUILD_ROOT'};
 $SIG{PIPE} = 'IGNORE';
 
 # Read the answers.
 my %answers;
 my %requiredAnswers;
-open ANSWERS, "<$ENV{KERNEL_CONFIG}" or die;
+open ANSWERS, "<$ENV{KERNEL_CONFIG}" or die "Could not open answer file";
 while (<ANSWERS>) {
     chomp;
     s/#.*//;
@@ -40,7 +40,7 @@ close ANSWERS;
 sub runConfig {
 
     # Run `make config'.
-    my $pid = open2(\*IN, \*OUT, "make -C $ENV{SRC} O=$wd config SHELL=bash ARCH=$ENV{ARCH}");
+    my $pid = open2(\*IN, \*OUT, "make -C $ENV{SRC} O=$buildRoot config SHELL=bash ARCH=$ENV{ARCH}");
 
     # Parse the output, look for questions and then send an
     # appropriate answer.
@@ -122,7 +122,7 @@ runConfig;
 # there.  `make config' often overrides answers if later questions
 # cause options to be selected.
 my %config;
-open CONFIG, "<.config" or die;
+open CONFIG, "<$buildRoot/.config" or die "Could not read .config";
 while (<CONFIG>) {
     chomp;
     if (/^CONFIG_([A-Za-z0-9_]+)="(.*)"$/) {
@@ -137,7 +137,7 @@ while (<CONFIG>) {
 close CONFIG;
 
 foreach my $name (sort (keys %answers)) {
-    my $f = $requiredAnswers{$name} && $ENV{'ignoreConfigErrors'} ne "1"
+    my $f = $requiredAnswers{$name} && $ignoreConfigErrors ne "1"
         ? sub { die "error: " . $_[0]; } : sub { warn "warning: " . $_[0]; };
     &$f("unused option: $name\n") unless defined $config{$name};
     &$f("option not set correctly: $name (wanted '$answers{$name}', got '$config{$name}')\n")

pkgs/os-specific/linux/kernel/generic.nix

@@ -1,3 +1,11 @@
+{ buildPackages, runCommand, nettools, bc, perl, gmp, libmpc, mpfr, openssl
+, ncurses
+, libelf
+, utillinux
+, writeTextFile, ubootTools
+, callPackage
+}:
+
 { stdenv, buildPackages, perl, buildLinux
 
 , # The kernel source tarball.
@@ -28,7 +36,7 @@
 , extraMeta ? {}
 , hostPlatform
 , ...
-}:
+} @ args:
 
 assert stdenv.isLinux;
 
@@ -45,8 +53,10 @@ let
   } // features) kernelPatches;
 
   config = import ./common-config.nix {
-    inherit stdenv version extraConfig;
+    inherit stdenv version ;
-    kernelPlatform = hostPlatform;
+    # append extraConfig for backwards compatibility but also means the user can't override the kernelExtraConfig part
+    extraConfig = extraConfig + lib.optionalString (hostPlatform ? kernelExtraConfig ) hostPlatform.kernelExtraConfig;
+
     features = kernelFeatures; # Ensure we know of all extra patches, etc.
   };
 
@@ -68,7 +78,9 @@ let
     nativeBuildInputs = [ perl ];
 
     platformName = hostPlatform.platform.name;
+    # e.g. "defconfig"
     kernelBaseConfig = hostPlatform.platform.kernelBaseConfig;
+    # e.g. "bzImage"
     kernelTarget = hostPlatform.platform.kernelTarget;
     autoModules = hostPlatform.platform.kernelAutoModules;
     preferBuiltin = hostPlatform.platform.kernelPreferBuiltin or false;
@@ -83,25 +95,25 @@ let
     inherit (kernel) src patches preUnpack;
 
     buildPhase = ''
-      cd $buildRoot
+      export buildRoot="''${buildRoot:-build}"
 
       # Get a basic config file for later refinement with $generateConfig.
-      make HOSTCC=${buildPackages.stdenv.cc.targetPrefix}gcc -C ../$sourceRoot O=$PWD $kernelBaseConfig ARCH=$arch
+      make HOSTCC=${buildPackages.stdenv.cc.targetPrefix}gcc -C . O="$buildRoot" $kernelBaseConfig ARCH=$arch
 
       # Create the config file.
       echo "generating kernel configuration..."
-      echo "$kernelConfig" > kernel-config
+      echo "$kernelConfig" > "$buildRoot/kernel-config"
-      DEBUG=1 ARCH=$arch KERNEL_CONFIG=kernel-config AUTO_MODULES=$autoModules \
+      DEBUG=1 ARCH=$arch KERNEL_CONFIG="$buildRoot/kernel-config" AUTO_MODULES=$autoModules \
-           PREFER_BUILTIN=$preferBuiltin SRC=../$sourceRoot perl -w $generateConfig
+           PREFER_BUILTIN=$preferBuiltin BUILD_ROOT="$buildRoot" SRC=. perl -w $generateConfig
     '';
 
-    installPhase = "mv .config $out";
+    installPhase = "mv $buildRoot/.config $out";
 
     enableParallelBuilding = true;
   };
 
-  kernel = buildLinux {
+  kernel = (callPackage ./manual-config.nix {}) {
-    inherit version modDirVersion src kernelPatches stdenv extraMeta configfile;
+    inherit version modDirVersion src kernelPatches stdenv extraMeta configfile hostPlatform;
 
     config = { CONFIG_MODULES = "y"; CONFIG_FW_LOADER = "m"; };
   };

pkgs/os-specific/linux/kernel/linux-4.13.nix

@@ -1,6 +1,6 @@
 { stdenv, buildPackages, hostPlatform, fetchurl, perl, buildLinux, ... } @ args:
 
-import ./generic.nix (args // rec {
+buildLinux (args // rec {
   version = "4.13.16";
   extraMeta.branch = "4.13";
 

pkgs/os-specific/linux/kernel/linux-4.14.nix

@@ -2,7 +2,7 @@
 
 with stdenv.lib;
 
-import ./generic.nix (args // rec {
+buildLinux (args // rec {
   version = "4.14.17";
 
   # branchVersion needs to be x.y

pkgs/os-specific/linux/kernel/linux-4.15.nix

@@ -2,7 +2,7 @@
 
 with stdenv.lib;
 
-import ./generic.nix (args // rec {
+buildLinux (args // rec {
   version = "4.15.1";
 
   # modDirVersion needs to be x.y.z, will automatically add .0 if needed

pkgs/os-specific/linux/kernel/linux-4.4.nix

@@ -1,6 +1,6 @@
 { stdenv, buildPackages, hostPlatform, fetchurl, perl, buildLinux, ... } @ args:
 
-import ./generic.nix (args // rec {
+buildLinux (args // rec {
   version = "4.4.115";
   extraMeta.branch = "4.4";
 

pkgs/os-specific/linux/kernel/linux-4.9.nix

@@ -1,6 +1,6 @@
 { stdenv, buildPackages, hostPlatform, fetchurl, perl, buildLinux, ... } @ args:
 
-import ./generic.nix (args // rec {
+buildLinux (args // rec {
   version = "4.9.80";
   extraMeta.branch = "4.9";
 

pkgs/os-specific/linux/kernel/linux-beagleboard.nix

@@ -4,7 +4,7 @@ let
   modDirVersion = "4.14.12";
   tag = "r23";
 in
-stdenv.lib.overrideDerivation (import ./generic.nix (args // rec {
+stdenv.lib.overrideDerivation (buildLinux (args // rec {
   version = "${modDirVersion}-ti-${tag}";
   inherit modDirVersion;
 

pkgs/os-specific/linux/kernel/linux-hardened-copperhead.nix

@@ -15,7 +15,7 @@ let
 
   modDirVersion = "${modVersion}-hardened";
 in
-import ./generic.nix (args // {
+buildLinux (args // {
   inherit modDirVersion;
 
   version = "${version}-${revision}";

pkgs/os-specific/linux/kernel/linux-mptcp.nix

@@ -1,9 +1,10 @@
 { stdenv, buildPackages, hostPlatform, fetchFromGitHub, perl, buildLinux, ... } @ args:
 
-import ./generic.nix (rec {
+buildLinux (rec {
   mptcpVersion = "0.93";
   modDirVersion = "4.9.60";
   version = "${modDirVersion}-mptcp_v${mptcpVersion}";
+  # autoModules= true;
 
   extraMeta = {
     branch = "4.4";
@@ -43,4 +44,4 @@ import ./generic.nix (rec {
     TCP_CONG_BALIA m
 
   '' + (args.extraConfig or "");
-} // args // (args.argsOverride or {}))
+} // args)

pkgs/os-specific/linux/kernel/linux-rpi.nix

@@ -4,7 +4,7 @@ let
   modDirVersion = "4.9.59";
   tag = "1.20171029";
 in
-stdenv.lib.overrideDerivation (import ./generic.nix (args // rec {
+stdenv.lib.overrideDerivation (buildLinux (args // rec {
   version = "${modDirVersion}-${tag}";
   inherit modDirVersion;
 

pkgs/os-specific/linux/kernel/linux-samus-4.12.nix

@@ -1,6 +1,6 @@
 { stdenv, buildPackages, hostPlatform, fetchFromGitHub, perl, buildLinux, ncurses, ... } @ args:
 
-import ./generic.nix (args // rec {
+buildLinux (args // rec {
   version = "4.12.2";
   extraMeta.branch = "4.12-2";
 

pkgs/os-specific/linux/kernel/linux-testing-bcachefs.nix

@@ -1,6 +1,6 @@
 { stdenv, buildPackages, hostPlatform, fetchgit, perl, buildLinux, ... } @ args:
 
-import ./generic.nix (args // rec {
+buildLinux (args // rec {
   version = "4.11.2017.08.23";
   modDirVersion = "4.11.0";
   extraMeta.branch = "master";

pkgs/os-specific/linux/kernel/linux-testing.nix

@@ -1,6 +1,6 @@
 { stdenv, buildPackages, hostPlatform, fetchurl, perl, buildLinux, libelf, utillinux, ... } @ args:
 
-import ./generic.nix (args // rec {
+buildLinux (args // rec {
   version = "4.15-rc9";
   modDirVersion = "4.15.0-rc9";
   extraMeta.branch = "4.15";

pkgs/os-specific/linux/kernel/manual-config.nix

@@ -1,8 +1,8 @@
 { buildPackages, runCommand, nettools, bc, perl, gmp, libmpc, mpfr, openssl
+, ncurses ? null
 , libelf
 , utillinux
 , writeTextFile, ubootTools
-, hostPlatform
 }:
 
 let
@@ -34,7 +34,9 @@ in {
   # Use defaultMeta // extraMeta
   extraMeta ? {},
   # Whether to utilize the controversial import-from-derivation feature to parse the config
-  allowImportFromDerivation ? false
+  allowImportFromDerivation ? false,
+
+  hostPlatform
 }:
 
 let
@@ -86,8 +88,6 @@ let
       inherit src;
 
       preUnpack = ''
-        mkdir build
-        export buildRoot="$(pwd)/build"
       '';
 
       patches = map (p: p.patch) kernelPatches;
@@ -102,7 +102,25 @@ let
 
       configurePhase = ''
         runHook preConfigure
+
+        mkdir build
+        export buildRoot="$(pwd)/build"
+
+        echo "manual-config configurePhase buildRoot=$buildRoot pwd=$PWD"
+
+        if [[ -z "$buildRoot" || ! -d "$buildRoot" ]]; then
+          echo "set $buildRoot to the build folder please"
+          exit 1
+        fi
+
+        if [ -f "$buildRoot/.config" ]; then
+          echo "Could not link $buildRoot/.config : file exists"
+          exit 1
+        fi
         ln -sv ${configfile} $buildRoot/.config
+
+        # reads the existing .config file and prompts the user for options in
+        # the current kernel source that are not found in the file.
         make $makeFlags "''${makeFlagsArray[@]}" oldconfig
         runHook postConfigure
 
@@ -115,6 +133,8 @@ let
 
         # Note: we can get rid of this once http://permalink.gmane.org/gmane.linux.kbuild.devel/13800 is merged.
         buildFlagsArray+=("KBUILD_BUILD_TIMESTAMP=$(date -u -d @$SOURCE_DATE_EPOCH)")
+
+        cd $buildRoot
       '';
 
       buildFlags = [
@@ -136,7 +156,7 @@ let
 
       postInstall = ''
         mkdir -p $dev
-        cp $buildRoot/vmlinux $dev/
+        cp vmlinux $dev/
       '' + (optionalString installsFirmware ''
         mkdir -p $out/lib/firmware
       '') + (if (platform ? kernelDTB && platform.kernelDTB) then ''
@@ -151,7 +171,7 @@ let
         unlink $out/lib/modules/${modDirVersion}/source
 
         mkdir -p $dev/lib/modules/${modDirVersion}/build
-        cp -dpR ../$sourceRoot $dev/lib/modules/${modDirVersion}/source
+        cp -dpR .. $dev/lib/modules/${modDirVersion}/source
         cd $dev/lib/modules/${modDirVersion}/source
 
         cp $buildRoot/{.config,Module.symvers} $dev/lib/modules/${modDirVersion}/build
@@ -170,7 +190,7 @@ let
         # from drivers/ in the future; it adds 50M to keep all of its
         # headers on 3.10 though.
 
-        chmod u+w -R ../source
+        chmod u+w -R ..
         arch=$(cd $dev/lib/modules/${modDirVersion}/build/arch; ls)
 
         # Remove unused arches

pkgs/os-specific/linux/kernel/update.sh

@@ -50,9 +50,9 @@ ls $NIXPKGS/pkgs/os-specific/linux/kernel | while read FILE; do
   # Rewrite the expression
   sed -i -e '/version = /d' -e '/modDirVersion = /d' $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
   if grep -q '^[0-9]\+.[0-9]\+$' <<< "$V"; then
-    sed -i "\#import ./generic.nix (args // rec {#a \  modDirVersion = \"${V}.0\";" $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
+    sed -i "\#buildLinux (args // rec {#a \  modDirVersion = \"${V}.0\";" $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
   fi
-  sed -i "\#import ./generic.nix (args // rec {#a \  version = \"$V\";" $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
+  sed -i "\#buildLinux (args // rec {#a \  version = \"$V\";" $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE
 
   # Commit the changes
   git add -u $NIXPKGS/pkgs/os-specific/linux/kernel/$FILE

pkgs/top-level/all-packages.nix

@@ -13199,7 +13199,7 @@ with pkgs;
 
   # A function to build a manually-configured kernel
   linuxManualConfig = pkgs.buildLinux;
-  buildLinux = makeOverridable (callPackage ../os-specific/linux/kernel/manual-config.nix {});
+  buildLinux = makeOverridable (callPackage ../os-specific/linux/kernel/generic.nix {});
 
   keyutils = callPackage ../os-specific/linux/keyutils { };