firefox: overhaul LTO

Enable LTO support on Linux by default again.

Add patch to fix dependentlibs.list generation under LTO. This is
necessary for fixing firefox-wayland crashing when built with LTO.

Add makeFlags which set ar, ranlib, and nm to be llvm-ar, llvm-ranlib
and llvm-nm when building with llvm-based LTO. (bmo#1480005)

pkgs/applications/networking/browsers/firefox/common.nix

@@ -23,8 +23,7 @@
 , ffmpegSupport ? true
 , gtk3Support ? true, gtk2, gtk3, wrapGAppsHook
 , waylandSupport ? true, libxkbcommon
-# LTO is disabled since it caused segfaults on wayland see https://github.com/NixOS/nixpkgs/issues/101429
-, ltoSupport ? false, overrideCC, buildPackages
+, ltoSupport ? stdenv.isLinux, overrideCC, buildPackages
 , gssSupport ? true, kerberos
 , pipewireSupport ? waylandSupport && webrtcSupport, pipewire
 
@@ -91,11 +90,19 @@ let
             then "/Applications/${binaryNameCapitalized}.app/Contents/MacOS"
             else "/bin";
 
+  # 78 ESR won't build with rustc 1.47
+  inherit (if lib.versionAtLeast ffversion "82" then rustPackages else rustPackages_1_45)
+    rustc cargo;
+
   # Darwin's stdenv provides the default llvmPackages version, match that since
   # clang LTO on Darwin is broken so the stdenv is not being changed.
+  # Target the LLVM version that rustc -Vv reports it is built with for LTO.
+  # rustPackages_1_45 -> LLVM 10, rustPackages -> LLVM 11
   llvmPackages = if stdenv.isDarwin
                  then buildPackages.llvmPackages
-                 else buildPackages.llvmPackages_10;
+                 else if lib.versionAtLeast rustc.llvm.version "11"
+                      then buildPackages.llvmPackages_11
+                      else buildPackages.llvmPackages_10;
 
   # When LTO for Darwin is fixed, the following will need updating as lld
   # doesn't work on it. For now it is fine since ltoSupport implies no Darwin.
@@ -103,10 +110,6 @@ let
                 then overrideCC stdenv llvmPackages.lldClang
                 else stdenv;
 
-  # 78 ESR won't build with rustc 1.47
-  inherit (if lib.versionAtLeast ffversion "82" then rustPackages else rustPackages_1_45)
-    rustc cargo;
-
   nss_pkg = if lib.versionOlder ffversion "83" then nss_3_53 else nss;
 in
 
@@ -121,13 +124,19 @@ buildStdenv.mkDerivation ({
   ] ++
   lib.optional (lib.versionOlder ffversion "83") ./no-buildconfig-ffx76.patch ++
   lib.optional (lib.versionAtLeast ffversion "84") ./no-buildconfig-ffx84.patch ++
+  lib.optional (ltoSupport && lib.versionOlder ffversion "84") ./lto-dependentlibs-generation-ffx83.patch ++
+  lib.optional (ltoSupport && lib.versionAtLeast ffversion "84" && lib.versionOlder ffversion "86")
+    (fetchpatch {
+      url = "https://hg.mozilla.org/mozilla-central/raw-rev/fdff20c37be3";
+      sha256 = "135n9brliqy42lj3nqgb9d9if7x6x9nvvn0z4anbyf89bikixw48";
+    })
 
   # there are two flavors of pipewire support
   # The patches for the ESR release and the patches for the current stable
   # release.
   # Until firefox upstream stabilizes pipewire support we will have to continue
   # tracking multiple versions here.
-  lib.optional (pipewireSupport && lib.versionOlder ffversion "83")
+  ++ lib.optional (pipewireSupport && lib.versionOlder ffversion "83")
     (fetchpatch {
       # https://src.fedoraproject.org/rpms/firefox/blob/master/f/firefox-pipewire-0-3.patch
       url = "https://src.fedoraproject.org/rpms/firefox/raw/e99b683a352cf5b2c9ff198756859bae408b5d9d/f/firefox-pipewire-0-3.patch";
@@ -316,6 +325,13 @@ buildStdenv.mkDerivation ({
     "MOZILLA_OFFICIAL=1"
     "BUILD_OFFICIAL=1"
   ]
+  ++ lib.optionals ltoSupport [
+    "AR=${llvmPackages.bintools}/bin/llvm-ar"
+    "LLVM_OBJDUMP=${llvmPackages.bintools}/bin/llvm-objdump"
+    "NM=${llvmPackages.bintools}/bin/llvm-nm"
+    "RANLIB=${llvmPackages.bintools}/bin/llvm-ranlib"
+    "STRIP=${llvmPackages.bintools}/bin/llvm-strip"
+  ]
   ++ extraMakeFlags;
 
   enableParallelBuilding = true;

pkgs/applications/networking/browsers/firefox/lto-dependentlibs-generation-ffx83.patch

@@ -0,0 +1,45 @@
+--- a/toolkit/library/build/dependentlibs.py
++++ b/toolkit/library/build/dependentlibs.py
+@@ -36,26 +36,17 @@ def dependentlibs_win32_objdump(lib):
+     proc.wait()
+     return deps
+ 
+-def dependentlibs_readelf(lib):
++def dependentlibs_elf_objdump(lib):
+     '''Returns the list of dependencies declared in the given ELF .so'''
+-    proc = subprocess.Popen([substs.get('TOOLCHAIN_PREFIX', '') + 'readelf', '-d', lib], stdout = subprocess.PIPE,
++    proc = subprocess.Popen([substs['LLVM_OBJDUMP'], '--private-headers', lib], stdout = subprocess.PIPE,
+                             universal_newlines=True)
+     deps = []
+     for line in proc.stdout:
+-        # Each line has the following format:
+-        #  tag (TYPE)          value
+-        # or with BSD readelf:
+-        #  tag TYPE            value
+-        # Looking for NEEDED type entries
+-        tmp = line.split(' ', 3)
+-        if len(tmp) > 3 and 'NEEDED' in tmp[2]:
+-            # NEEDED lines look like:
+-            # 0x00000001 (NEEDED)             Shared library: [libname]
+-            # or with BSD readelf:
+-            # 0x00000001 NEEDED               Shared library: [libname]
+-            match = re.search('\[(.*)\]', tmp[3])
+-            if match:
+-                deps.append(match.group(1))
++        # We are looking for lines with the format:
++        #   NEEDED             libname
++        tmp = line.split()
++        if len(tmp) == 2 and tmp[0] == 'NEEDED':
++            deps.append(tmp[1])
+     proc.wait()
+     return deps
+ 
+@@ -110,7 +101,7 @@ def gen_list(output, lib):
+     libpaths = [os.path.join(substs['DIST'], 'bin')]
+     binary_type = get_type(lib)
+     if binary_type == ELF:
+-        func = dependentlibs_readelf
++        func = dependentlibs_elf_objdump
+     elif binary_type == MACHO:
+         func = dependentlibs_mac_objdump
+     else: