Merge pull request #32271 from orivej/parallel-building

Enable parallel building for CMake based projects by default

doc/stdenv.xml

@@ -251,10 +251,17 @@ genericBuild
 
   <varlistentry>
     <term><varname>enableParallelBuilding</varname></term>
-    <listitem><para>If set, <literal>stdenv</literal> will pass specific
-    flags to <literal>make</literal> and other build tools to enable
-    parallel building with up to <literal>build-cores</literal>
-    workers.</para></listitem>
+    <listitem>
+      <para>If set to <literal>true</literal>, <literal>stdenv</literal> will
+      pass specific flags to <literal>make</literal> and other build tools to
+      enable parallel building with up to <literal>build-cores</literal>
+      workers.</para>
+
+      <para>Unless set to <literal>false</literal>, some build systems with good
+      support for parallel building including <literal>cmake</literal>,
+      <literal>meson</literal>, and <literal>qmake</literal> will set it to
+      <literal>true</literal>.</para>
+    </listitem>
   </varlistentry>
 
   <varlistentry>

pkgs/applications/altcoins/memorycoin.nix

@@ -31,6 +31,10 @@ stdenv.mkDerivation rec{
     then "install -D bitcoin-qt $out/bin/memorycoin-qt"
     else "install -D bitcoind $out/bin/memorycoind";
 
+  # `make build/version.o`:
+  # make: *** No rule to make target 'build/build.h', needed by 'build/version.o'.  Stop.
+  enableParallelBuilding = false;
+
   meta = {
     description = "Peer-to-peer, CPU-based electronic cash system";
     longDescription= ''

pkgs/applications/altcoins/primecoin.nix

@@ -31,6 +31,10 @@ stdenv.mkDerivation rec{
     then "install -D bitcoin-qt $out/bin/primecoin-qt"
     else "install -D bitcoind $out/bin/primecoind";
 
+  # `make build/version.o`:
+  # make: *** No rule to make target 'build/build.h', needed by 'build/version.o'.  Stop.
+  enableParallelBuilding = false;
+
   meta = {
     description = "A new type cryptocurrency which is proof-of-work based on searching for prime numbers";
     longDescription= ''

pkgs/applications/gis/qgis/default.nix

@@ -14,8 +14,9 @@ stdenv.mkDerivation rec {
 
   nativeBuildInputs = [ cmake makeWrapper ];
 
+  # `make -f src/providers/wms/CMakeFiles/wmsprovider_a.dir/build.make src/providers/wms/CMakeFiles/wmsprovider_a.dir/qgswmssourceselect.cpp.o`:
   # fatal error: ui_qgsdelimitedtextsourceselectbase.h: No such file or directory
-  #enableParallelBuilding = true;
+  enableParallelBuilding = false;
 
   # To handle the lack of 'local' RPATH; required, as they call one of
   # their built binaries requiring their libs, in the build process.

pkgs/applications/graphics/awesomebump/default.nix

@@ -15,6 +15,7 @@ let
     name = "qtnproperty";
     inherit src;
     sourceRoot = "AwesomeBump/Sources/utils/QtnProperty";
+    patches = [ ./qtnproperty-parallel-building.patch ];
     buildInputs = [ qtscript qtbase qtdeclarative ];
     nativeBuildInputs = [ qmake flex bison ];
     postInstall = ''
@@ -46,6 +47,10 @@ in stdenv.mkDerivation rec {
         --run "cd $d"
   '';
 
+  passthru = {
+    inherit qtnproperty;
+  };
+
   meta = {
     homepage = https://github.com/kmkolasinski/AwesomeBump;
     description = "A program to generate normal, height, specular or ambient occlusion textures from a single image";

pkgs/applications/graphics/awesomebump/qtnproperty-parallel-building.patch

@@ -0,0 +1,9 @@
+--- a/PEG/Flex.pri
++++ b/PEG/Flex.pri
+@@ -1,5 +1,6 @@
+ flex.name = Flex ${QMAKE_FILE_IN}
+ flex.input = FLEX_SOURCES
++flex.depends = ${QMAKE_FILE_PATH}/${QMAKE_FILE_BASE}.parser.cpp
+ flex.output = ${QMAKE_FILE_PATH}/${QMAKE_FILE_BASE}.lexer.cpp
+ 
+ win32:flex.commands = win_flex --wincompat -o ${QMAKE_FILE_PATH}/${QMAKE_FILE_BASE}.lexer.cpp ${QMAKE_FILE_IN}

pkgs/applications/misc/golden-cheetah/default.nix

@@ -30,6 +30,10 @@ stdenv.mkDerivation rec {
 
     runHook postInstall
   '';
+
+  # RCC: Error in 'Resources/application.qrc': Cannot find file 'translations/gc_fr.qm'
+  enableParallelBuilding = false;
+
   meta = {
     description = "Performance software for cyclists, runners and triathletes";
     platforms = stdenv.lib.platforms.linux;

pkgs/applications/networking/instant-messengers/dino/default.nix

@@ -1,5 +1,5 @@
 { stdenv, fetchFromGitHub
-, vala, cmake, wrapGAppsHook, pkgconfig, gettext
+, vala, cmake, ninja, wrapGAppsHook, pkgconfig, gettext
 , gobjectIntrospection, gnome3, glib, gdk_pixbuf, gtk3, glib_networking
 , xorg, libXdmcp, libxkbcommon
 , libnotify, libsoup
@@ -26,6 +26,7 @@ stdenv.mkDerivation rec {
   nativeBuildInputs = [
     vala
     cmake
+    ninja
     pkgconfig
     wrapGAppsHook
   ];

pkgs/applications/networking/instant-messengers/ricochet/default.nix

@@ -46,6 +46,9 @@ stdenv.mkDerivation rec {
     cp icons/ricochet.png $out/share/pixmaps/ricochet.png
   '';
 
+  # RCC: Error in 'translation/embedded.qrc': Cannot find file 'ricochet_en.qm'
+  enableParallelBuilding = false;
+
   meta = with stdenv.lib; {
     description = "Anonymous peer-to-peer instant messaging";
     homepage = https://ricochet.im;

pkgs/applications/networking/ostinato/default.nix

@@ -54,6 +54,10 @@ stdenv.mkDerivation rec {
     EOF
   '';
 
+  # `cd common; qmake ostproto.pro; make pdmlreader.o`:
+  # pdmlprotocol.h:23:25: fatal error: protocol.pb.h: No such file or directory
+  enableParallelBuilding = false;
+
   meta = with stdenv.lib; {
     description = "A packet traffic generator and analyzer";
     homepage    = http://ostinato.org;

pkgs/applications/science/logic/stp/default.nix

@@ -23,6 +23,10 @@ stdenv.mkDerivation rec {
     )
   '';
 
+  # `make -f lib/Interface/CMakeFiles/cppinterface.dir/build.make lib/Interface/CMakeFiles/cppinterface.dir/cpp_interface.cpp.o`:
+  # include/stp/AST/UsefulDefs.h:41:29: fatal error: stp/AST/ASTKind.h: No such file or directory
+  enableParallelBuilding = false;
+
   meta = with stdenv.lib; {
     description = "Simple Theorem Prover";
     maintainers = with maintainers; [ mornfall ];

pkgs/applications/science/machine-learning/shogun/default.nix

@@ -1,4 +1,4 @@
-{ stdenv, lib, fetchFromGitHub, ccache, cmake, ctags, swig
+{ stdenv, lib, fetchFromGitHub, fetchpatch, ccache, cmake, ctags, swig
 # data, compression
 , bzip2, curl, hdf5, json_c, lzma, lzo, protobuf, snappy
 # maths
@@ -27,6 +27,12 @@ stdenv.mkDerivation rec {
     fetchSubmodules = true;
   };
 
+  patches = fetchpatch {
+    name = "Fix-meta-example-parser-bug-in-parallel-builds.patch";
+    url = "https://github.com/shogun-toolbox/shogun/commit/ecd6a8f11ac52748e89d27c7fab7f43c1de39f05.patch";
+    sha256 = "1hrwwrj78sxhwcvgaz7n4kvh5y9snfcc4jf5xpgji5hjymnl311n";
+  };
+
   CCACHE_DIR=".ccache";
 
   buildInputs = with lib; [

pkgs/applications/version-management/guitone/default.nix

@@ -16,6 +16,8 @@ stdenv.mkDerivation rec {
     branch = "net.venge.monotone.guitone";
   };
 
+  patches = [ ./parallel-building.patch ];
+
   nativeBuildInputs = [ pkgconfig ];
   buildInputs = [ qt4 qmake4Hook graphviz ];
 
@@ -24,6 +26,7 @@ stdenv.mkDerivation rec {
   meta = {
     description = "Qt4 based GUI for monotone";
     homepage = http://guitone.thomaskeller.biz;
+    downloadPage = https://code.monotone.ca/p/guitone/;
     inherit (qt4.meta) platforms;
   };
 }

pkgs/applications/version-management/guitone/parallel-building.patch

@@ -0,0 +1,7 @@
+Without this `make tmp/AttributesView.o` fails with
+src/view/dialogs/AddEditAttribute.h:22:35: fatal error: ui_add_edit_attribute.h: No such file or directory
+--- a/guitone.pro
++++ b/guitone.pro
+@@ -215 +215,2 @@ help.commands = @echo Available targets: $${QMAKE_EXTRA_TARGETS}
+ QMAKE_EXTRA_TARGETS += help
++CONFIG += depend_includepath

pkgs/desktops/gnome-3/core/totem/default.nix

@@ -9,7 +9,9 @@ stdenv.mkDerivation rec {
 
   doCheck = true;
 
-  enableParallelBuilding = true;
+  # https://bugzilla.gnome.org/show_bug.cgi?id=784236
+  # https://github.com/mesonbuild/meson/issues/1994
+  enableParallelBuilding = false;
 
   NIX_CFLAGS_COMPILE = "-I${gnome3.glib.dev}/include/gio-unix-2.0";
 

pkgs/development/libraries/gstreamer/bad/default.nix

@@ -70,4 +70,6 @@ stdenv.mkDerivation rec {
     ++ optional (!stdenv.isDarwin) wildmidi;
 
   LDFLAGS = optionalString stdenv.isDarwin "-lintl";
+
+  enableParallelBuilding = true;
 }

pkgs/development/libraries/qt-4.x/4.8/qmake-hook.sh

@@ -3,6 +3,11 @@ qmakeConfigurePhase() {
 
     $QMAKE PREFIX=$out $qmakeFlags
 
+    if ! [[ -v enableParallelBuilding ]]; then
+        enableParallelBuilding=1
+        echo "qmake4Hook: enabled parallel building"
+    fi
+
     runHook postConfigure
 }
 

pkgs/development/libraries/qt-5/hooks/qmake-hook.sh

@@ -10,6 +10,11 @@ qmakeConfigurePhase() {
           NIX_OUTPUT_PLUGIN=${!outputBin}/${qtPluginPrefix:?} \
           $qmakeFlags
 
+    if ! [[ -v enableParallelBuilding ]]; then
+        enableParallelBuilding=1
+        echo "qmake: enabled parallel building"
+    fi
+
     runHook postConfigure
 }
 

pkgs/development/tools/build-managers/cmake/setup-hook.sh

@@ -51,10 +51,19 @@ cmakeConfigurePhase() {
     # And build always Release, to ensure optimisation flags
     cmakeFlags="-DCMAKE_BUILD_TYPE=${cmakeBuildType:-Release} -DCMAKE_SKIP_BUILD_RPATH=ON $cmakeFlags"
 
+    if [ "$buildPhase" = ninjaBuildPhase ]; then
+        cmakeFlags="-GNinja $cmakeFlags"
+    fi
+
     echo "cmake flags: $cmakeFlags ${cmakeFlagsArray[@]}"
 
     cmake ${cmakeDir:-.} $cmakeFlags "${cmakeFlagsArray[@]}"
 
+    if ! [[ -v enableParallelBuilding ]]; then
+        enableParallelBuilding=1
+        echo "cmake: enabled parallel building"
+    fi
+
     runHook postConfigure
 }
 

pkgs/development/tools/build-managers/meson/setup-hook.sh

@@ -13,6 +13,11 @@ mesonConfigurePhase() {
     meson build $mesonFlags "${mesonFlagsArray[@]}"
     cd build
 
+    if ! [[ -v enableParallelBuilding ]]; then
+        enableParallelBuilding=1
+        echo "meson: enabled parallel building"
+    fi
+
     runHook postConfigure
 }
 

pkgs/development/tools/build-managers/ninja/setup-hook.sh

@@ -4,9 +4,16 @@ ninjaBuildPhase() {
     if [[ -z "$ninjaFlags" && ! ( -e build.ninja ) ]]; then
         echo "no build.ninja, doing nothing"
     else
+        local buildCores=1
+
+        # Parallel building is enabled by default.
+        if [ "${enableParallelBuilding-1}" ]; then
+            buildCores="$NIX_BUILD_CORES"
+        fi
+
         # shellcheck disable=SC2086
         local flagsArray=( \
-            ${enableParallelBuilding:+-j${NIX_BUILD_CORES} -l${NIX_BUILD_CORES}} \
+            -j"$buildCores" -l"$NIX_BUILD_CORES" \
             $ninjaFlags "${ninjaFlagsArray[@]}" \
             $buildFlags "${buildFlagsArray[@]}")
 

pkgs/development/tools/misc/editorconfig-core-c/default.nix

@@ -14,6 +14,10 @@ stdenv.mkDerivation rec {
   buildInputs = [ pcre ];
   nativeBuildInputs = [ cmake doxygen ];
 
+  # Multiple doxygen can not generate man pages in the same base directory in
+  # parallel: https://bugzilla.gnome.org/show_bug.cgi?id=791153
+  enableParallelBuilding = false;
+
   meta = with stdenv.lib; {
     homepage = http://editorconfig.org/;
     description = "EditorConfig core library written in C";

pkgs/games/chessx/default.nix

@@ -16,7 +16,7 @@ stdenv.mkDerivation rec {
   nativeBuildInputs = [ pkgconfig qmake ];
 
   # RCC: Error in 'resources.qrc': Cannot find file 'i18n/chessx_da.qm'
-  #enableParallelBuilding = true;
+  enableParallelBuilding = false;
 
   installPhase = ''
       runHook preInstall

pkgs/os-specific/linux/conky/default.nix

@@ -131,6 +131,10 @@ stdenv.mkDerivation rec {
     ++ optional nvidiaSupport       "-DBUILD_NVIDIA=ON"
     ;
 
+  # `make -f src/CMakeFiles/conky.dir/build.make src/CMakeFiles/conky.dir/conky.cc.o`:
+  # src/conky.cc:137:23: fatal error: defconfig.h: No such file or directory
+  enableParallelBuilding = false;
+
   meta = with stdenv.lib; {
     homepage = http://conky.sourceforge.net/;
     description = "Advanced, highly configurable system monitor based on torsmo";

pkgs/tools/misc/rockbox-utility/default.nix

@@ -39,6 +39,12 @@ stdenv.mkDerivation  rec {
     runHook postInstall
   '';
 
+  # `make build/rcc/qrc_rbutilqt-lang.cpp` fails with
+  #      RCC: Error in 'rbutilqt-lang.qrc': Cannot find file 'lang/rbutil_cs.qm'
+  # Do not add `lrelease rbutilqt.pro` into preConfigure, otherwise `make lrelease`
+  # may clobber the files read by the parallel `make build/rcc/qrc_rbutilqt-lang.cpp`.
+  enableParallelBuilding = false;
+
   meta = with stdenv.lib; {
     description = "Open source firmware for mp3 players";
     homepage = http://www.rockbox.org;