From df4761d45082d94f6a469bbcab71cee1e31719da Mon Sep 17 00:00:00 2001 From: Benedikt Morbach Date: Sat, 24 Oct 2020 17:23:37 +0200 Subject: [PATCH 01/10] fhs-userenv-bubblewrap: Preserve symlinks Preserve top-level symlinks such as /lib -> /usr/lib. This allows nested containers such as Steam's new runtime to remount /usr if they need to and then run unmodified binaries that reference e.g. /lib/ld-linux-x86-64.so.2 Before, we would mount the fully resolved host directory at /lib and thus the dynamic loader would always be the one from the host filesystem. --- .../build-fhs-userenv-bubblewrap/default.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix b/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix index 6592621570c..dd945678e6f 100644 --- a/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix +++ b/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix @@ -68,13 +68,18 @@ let bwrapCmd = { initArgs ? "" }: '' blacklist=(/nix /dev /proc /etc) ro_mounts=() + symlinks=() for i in ${env}/*; do path="/''${i##*/}" if [[ $path == '/etc' ]]; then - continue + : + elif [[ -L $i ]]; then + symlinks+=(--symlink "$(readlink "$i")" "$path") + blacklist+=("$path") + else + ro_mounts+=(--ro-bind "$i" "$path") + blacklist+=("$path") fi - ro_mounts+=(--ro-bind "$i" "$path") - blacklist+=("$path") done if [[ -d ${env}/etc ]]; then @@ -114,6 +119,7 @@ let --ro-bind /nix /nix ${etcBindFlags} "''${ro_mounts[@]}" + "''${symlinks[@]}" "''${auto_mounts[@]}" ${init runScript}/bin/${name}-init ${initArgs} ) From d5cbb650e18728eaf6fe5004ae8d591feac13435 Mon Sep 17 00:00:00 2001 From: Benedikt Morbach Date: Fri, 30 Oct 2020 19:22:04 +0100 Subject: [PATCH 02/10] fhs-userenv-bubblewrap: add ld.so.conf/cache to fhs --- .../build-fhs-userenv-bubblewrap/default.nix | 28 ++++++++++++++++++- .../build-fhs-userenv-bubblewrap/env.nix | 10 +++++-- 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix b/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix index dd945678e6f..2f99f9f761c 100644 --- a/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix +++ b/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix @@ -1,4 +1,6 @@ -{ lib, callPackage, runCommandLocal, writeShellScriptBin, coreutils, bubblewrap }: +{ lib, callPackage, runCommandLocal, writeShellScriptBin, glibc, coreutils, bubblewrap }: + +let buildFHSEnv = callPackage ./env.nix { }; in args @ { name @@ -60,8 +62,27 @@ let in concatStringsSep "\n " (map (file: "--ro-bind-try /etc/${file} /etc/${file}") files); + # Create this on the fly instead of linking from /nix + # The container might have to modify it and re-run ldconfig if there are + # issues running some binary with LD_LIBRARY_PATH + createLdConfCache = '' + cat > /etc/ld.so.conf < /dev/null + ''; init = run: writeShellScriptBin "${name}-init" '' source /etc/profile + ${createLdConfCache} exec ${run} "$@" ''; @@ -117,6 +138,11 @@ let ${lib.optionalString unshareCgroup "--unshare-cgroup"} --die-with-parent --ro-bind /nix /nix + --tmpfs ${glibc}/etc \ + --symlink /etc/ld.so.conf ${glibc}/etc/ld.so.conf \ + --symlink /etc/ld.so.cache ${glibc}/etc/ld.so.cache \ + --ro-bind ${glibc}/etc/rpc ${glibc}/etc/rpc \ + --remount-ro ${glibc}/etc \ ${etcBindFlags} "''${ro_mounts[@]}" "''${symlinks[@]}" diff --git a/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix b/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix index 8b2d46c4ae9..bcb9a8a0767 100644 --- a/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix +++ b/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix @@ -1,4 +1,4 @@ -{ stdenv, buildEnv, writeText, pkgs, pkgsi686Linux }: +{ stdenv, buildEnv, writeText, writeScriptBin, pkgs, pkgsi686Linux }: { name, profile ? "" , targetPkgs ? pkgs: [], multiPkgs ? pkgs: [] @@ -49,6 +49,11 @@ let [ (toString gcc.cc.lib) ]; + ldconfig = writeScriptBin "ldconfig" '' + #!${pkgs.stdenv.shell} + + exec ${pkgs.glibc.bin}/bin/ldconfig -f /etc/ld.so.conf -C /etc/ld.so.cache "$@" + ''; etcProfile = writeText "profile" '' export PS1='${name}-chrootenv:\u@\h:\w\$ ' export LOCALE_ARCHIVE='/usr/lib/locale/locale-archive' @@ -86,7 +91,8 @@ let # Composes a /usr-like directory structure staticUsrProfileTarget = buildEnv { name = "${name}-usr-target"; - paths = [ etcPkg ] ++ basePkgs ++ targetPaths; + # ldconfig wrapper must come first so it overrides the original ldconfig + paths = [ etcPkg ldconfig ] ++ basePkgs ++ targetPaths; extraOutputsToInstall = [ "out" "lib" "bin" ] ++ extraOutputsToInstall; ignoreCollisions = true; }; From cd29b6ff97e6bf95e537ed425fce8f0b2a92a493 Mon Sep 17 00:00:00 2001 From: ash lea Date: Wed, 3 Feb 2021 20:17:56 +0000 Subject: [PATCH 03/10] steam: add mesa dependencies for pressure-vessel --- pkgs/games/steam/fhsenv.nix | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkgs/games/steam/fhsenv.nix b/pkgs/games/steam/fhsenv.nix index 924714d802a..04afd486c60 100644 --- a/pkgs/games/steam/fhsenv.nix +++ b/pkgs/games/steam/fhsenv.nix @@ -134,6 +134,19 @@ in buildFHSUserEnv rec { libuuid libbsd alsaLib + + # needed by getcap for vr startup + libcap + + # dependencies for mesa drivers, needed inside pressure-vessel + expat + wayland + xlibs.libxcb + xlibs.libXdamage + xlibs.libxshmfence + xlibs.libXxf86vm + llvm_11.lib + libelf ] ++ (if (!nativeOnly) then [ (steamPackages.steam-runtime-wrapped.override { inherit runtimeOnly; From e358a6f4fdc1425015103ec8462fb202d545ca01 Mon Sep 17 00:00:00 2001 From: Luigi Sartor Piucco Date: Wed, 3 Feb 2021 21:24:23 -0300 Subject: [PATCH 04/10] steam: add drivers to bwrap --- nixos/modules/programs/steam.nix | 9 ++++++++- pkgs/games/steam/fhsenv.nix | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/nixos/modules/programs/steam.nix b/nixos/modules/programs/steam.nix index 3c919c47a0c..eecb9de43cc 100644 --- a/nixos/modules/programs/steam.nix +++ b/nixos/modules/programs/steam.nix @@ -4,6 +4,13 @@ with lib; let cfg = config.programs.steam; + + steam = pkgs.steam.override { + extraLibraries = pkgs: with config.hardware.opengl; + if pkgs.hostPlatform.is64bit + then [ package ] ++ extraPackages + else [ package32 ] ++ extraPackages32; + }; in { options.programs.steam.enable = mkEnableOption "steam"; @@ -18,7 +25,7 @@ in { hardware.steam-hardware.enable = true; - environment.systemPackages = [ pkgs.steam ]; + environment.systemPackages = [ steam ]; }; meta.maintainers = with maintainers; [ mkg20001 ]; diff --git a/pkgs/games/steam/fhsenv.nix b/pkgs/games/steam/fhsenv.nix index 04afd486c60..60f8e55ed21 100644 --- a/pkgs/games/steam/fhsenv.nix +++ b/pkgs/games/steam/fhsenv.nix @@ -139,6 +139,7 @@ in buildFHSUserEnv rec { libcap # dependencies for mesa drivers, needed inside pressure-vessel + mesa.drivers expat wayland xlibs.libxcb From 28c96887146014f2b417623552161eaa3d1db96e Mon Sep 17 00:00:00 2001 From: Luigi Sartor Piucco Date: Wed, 3 Feb 2021 21:24:59 -0300 Subject: [PATCH 05/10] steam: help pressure-vessel find Vulkan ICDs --- pkgs/games/steam/fhsenv.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/games/steam/fhsenv.nix b/pkgs/games/steam/fhsenv.nix index 60f8e55ed21..42faaf287d7 100644 --- a/pkgs/games/steam/fhsenv.nix +++ b/pkgs/games/steam/fhsenv.nix @@ -265,6 +265,8 @@ in buildFHSUserEnv rec { fi export STEAM_RUNTIME=${if nativeOnly then "0" else "/steamrt"} + + export VK_ICD_FILENAMES=/usr/share/vulkan/icd.d/intel_icd.x86_64.json:/usr/share/vulkan/icd.d/intel_icd.i686.json:/usr/share/vulkan/icd.d/lvp_icd.x86_64.json:/usr/share/vulkan/icd.d/lvp_icd.i686.json:/usr/share/vulkan/icd.d/nvidia_icd.json:/usr/share/vulkan/icd.d/nvidia_icd32.json:/usr/share/vulkan/icd.d/radeon_icd.x86_64.json:/usr/share/vulkan/icd.d/radeon_icd.i686.json '' + extraProfile; runScript = writeScript "steam-wrapper.sh" '' From bdd902776093bca7e004c5eade5a19557bbd4bdd Mon Sep 17 00:00:00 2001 From: Luigi Sartor Piucco Date: Wed, 3 Feb 2021 21:25:55 -0300 Subject: [PATCH 06/10] fhs-bubblewrap: merge /usr/share from both archs --- .../build-fhs-userenv-bubblewrap/env.nix | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix b/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix index bcb9a8a0767..73d705dbaa6 100644 --- a/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix +++ b/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix @@ -1,4 +1,4 @@ -{ stdenv, buildEnv, writeText, writeScriptBin, pkgs, pkgsi686Linux }: +{ stdenv, lib, buildEnv, writeText, writeScriptBin, pkgs, pkgsi686Linux }: { name, profile ? "" , targetPkgs ? pkgs: [], multiPkgs ? pkgs: [] @@ -138,7 +138,20 @@ let mkdir -m0755 usr cd usr ${setupLibDirs} - for i in bin sbin share include; do + ${lib.optionalString isMultiBuild '' + if [ -d "${staticUsrProfileMulti}/share" ]; then + cp -rLf ${staticUsrProfileMulti}/share share + fi + ''} + if [ -d "${staticUsrProfileTarget}/share" ]; then + if [ -d share ]; then + chmod -R 755 share + cp -rLTf ${staticUsrProfileTarget}/share share + else + cp -rLf ${staticUsrProfileTarget}/share share + fi + fi + for i in bin sbin include; do if [ -d "${staticUsrProfileTarget}/$i" ]; then cp -rsHf "${staticUsrProfileTarget}/$i" "$i" fi From baaec2953107ac991b21aca9ea3892d9ab8cd5e5 Mon Sep 17 00:00:00 2001 From: Luigi Sartor Piucco Date: Wed, 3 Feb 2021 22:16:20 -0300 Subject: [PATCH 07/10] fhs-bubblewrap: mount cache on 32 bit glibc too --- .../build-support/build-fhs-userenv-bubblewrap/default.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix b/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix index 2f99f9f761c..c39b1131f42 100644 --- a/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix +++ b/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix @@ -1,4 +1,4 @@ -{ lib, callPackage, runCommandLocal, writeShellScriptBin, glibc, coreutils, bubblewrap }: +{ lib, callPackage, runCommandLocal, writeShellScriptBin, glibc, pkgsi686Linux, coreutils, bubblewrap }: let buildFHSEnv = callPackage ./env.nix { }; in @@ -143,6 +143,11 @@ let --symlink /etc/ld.so.cache ${glibc}/etc/ld.so.cache \ --ro-bind ${glibc}/etc/rpc ${glibc}/etc/rpc \ --remount-ro ${glibc}/etc \ + --tmpfs ${pkgsi686Linux.glibc}/etc \ + --symlink /etc/ld.so.conf ${pkgsi686Linux.glibc}/etc/ld.so.conf \ + --symlink /etc/ld.so.cache ${pkgsi686Linux.glibc}/etc/ld.so.cache \ + --ro-bind ${pkgsi686Linux.glibc}/etc/rpc ${pkgsi686Linux.glibc}/etc/rpc \ + --remount-ro ${pkgsi686Linux.glibc}/etc \ ${etcBindFlags} "''${ro_mounts[@]}" "''${symlinks[@]}" From f9d9740e68e5f9f5901edf284fdda76e366db9c9 Mon Sep 17 00:00:00 2001 From: Luigi Sartor Piucco Date: Mon, 22 Feb 2021 14:34:08 -0300 Subject: [PATCH 08/10] steam module: add proper steam.run as well --- nixos/modules/programs/steam.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/programs/steam.nix b/nixos/modules/programs/steam.nix index eecb9de43cc..6e9b7729ad6 100644 --- a/nixos/modules/programs/steam.nix +++ b/nixos/modules/programs/steam.nix @@ -25,7 +25,7 @@ in { hardware.steam-hardware.enable = true; - environment.systemPackages = [ steam ]; + environment.systemPackages = [ steam steam.run ]; }; meta.maintainers = with maintainers; [ mkg20001 ]; From 12c2eae2c50f81ab6207c224dbe7d71a529db4ec Mon Sep 17 00:00:00 2001 From: Luigi Sartor Piucco Date: Tue, 23 Feb 2021 11:44:16 -0300 Subject: [PATCH 09/10] build-fhs-userenv-bubblewrap: add folders comment --- .../build-support/build-fhs-userenv-bubblewrap/default.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix b/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix index c39b1131f42..04f89a0b64c 100644 --- a/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix +++ b/pkgs/build-support/build-fhs-userenv-bubblewrap/default.nix @@ -138,6 +138,13 @@ let ${lib.optionalString unshareCgroup "--unshare-cgroup"} --die-with-parent --ro-bind /nix /nix + # Our glibc will look for the cache in its own path in `/nix/store`. + # As such, we need a cache to exist there, because pressure-vessel + # depends on the existence of an ld cache. However, adding one + # globally proved to be a bad idea (see #100655), the solution we + # settled on being mounting one via bwrap. + # Also, the cache needs to go to both 32 and 64 bit glibcs, for games + # of both architectures to work. --tmpfs ${glibc}/etc \ --symlink /etc/ld.so.conf ${glibc}/etc/ld.so.conf \ --symlink /etc/ld.so.cache ${glibc}/etc/ld.so.cache \ From 548d50d6952de1128b5cbd535bb5f8be9b7c843c Mon Sep 17 00:00:00 2001 From: Luigi Sartor Piucco Date: Tue, 23 Feb 2021 11:47:40 -0300 Subject: [PATCH 10/10] build-fhs-userenv-bubblewrap:->writeShellScriptBin --- pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix b/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix index 73d705dbaa6..b9c719a4c78 100644 --- a/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix +++ b/pkgs/build-support/build-fhs-userenv-bubblewrap/env.nix @@ -1,4 +1,4 @@ -{ stdenv, lib, buildEnv, writeText, writeScriptBin, pkgs, pkgsi686Linux }: +{ stdenv, lib, buildEnv, writeText, writeShellScriptBin, pkgs, pkgsi686Linux }: { name, profile ? "" , targetPkgs ? pkgs: [], multiPkgs ? pkgs: [] @@ -49,9 +49,7 @@ let [ (toString gcc.cc.lib) ]; - ldconfig = writeScriptBin "ldconfig" '' - #!${pkgs.stdenv.shell} - + ldconfig = writeShellScriptBin "ldconfig" '' exec ${pkgs.glibc.bin}/bin/ldconfig -f /etc/ld.so.conf -C /etc/ld.so.cache "$@" ''; etcProfile = writeText "profile" ''