From c9955d06be19034cb25f58eb7d46d87c345d9dde Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Wed, 6 Jan 2021 06:01:09 -0500 Subject: [PATCH 1/4] nixos/podman: add nvidia runtime support --- nixos/modules/virtualisation/podman.nix | 53 ++++++++++++++++++------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/nixos/modules/virtualisation/podman.nix b/nixos/modules/virtualisation/podman.nix index f554aeffb45..02709176ce6 100644 --- a/nixos/modules/virtualisation/podman.nix +++ b/nixos/modules/virtualisation/podman.nix @@ -1,6 +1,7 @@ { config, lib, pkgs, utils, ... }: let cfg = config.virtualisation.podman; + toml = pkgs.formats.toml { }; inherit (lib) mkOption types; @@ -53,6 +54,14 @@ in ''; }; + enableNvidia = mkOption { + type = types.bool; + default = false; + description = '' + Enable use of NVidia GPUs from within podman containers. + ''; + }; + extraPackages = mkOption { type = with types; listOf package; default = [ ]; @@ -78,21 +87,37 @@ in }; - config = lib.mkIf cfg.enable { + config = lib.mkIf cfg.enable (lib.mkMerge [ + { + environment.systemPackages = [ cfg.package ] + ++ lib.optional cfg.dockerCompat dockerCompat; - environment.systemPackages = [ cfg.package ] - ++ lib.optional cfg.dockerCompat dockerCompat; + environment.etc."cni/net.d/87-podman-bridge.conflist".source = utils.copyFile "${pkgs.podman-unwrapped.src}/cni/87-podman-bridge.conflist"; - environment.etc."cni/net.d/87-podman-bridge.conflist".source = utils.copyFile "${pkgs.podman-unwrapped.src}/cni/87-podman-bridge.conflist"; - - # Enable common /etc/containers configuration - virtualisation.containers.enable = true; - - assertions = [{ - assertion = cfg.dockerCompat -> !config.virtualisation.docker.enable; - message = "Option dockerCompat conflicts with docker"; - }]; - - }; + virtualisation.containers = { + enable = true; # Enable common /etc/containers configuration + containersConf.extraConfig = lib.optionalString cfg.enableNvidia + (builtins.readFile (toml.generate "podman.nvidia.containers.conf" { + engine = { + conmon_env_vars = [ "PATH=${lib.makeBinPath [ pkgs.nvidia-docker ]}" ]; + runtimes.nvidia = [ "${pkgs.nvidia-docker}/bin/nvidia-container-runtime" ]; + }; + })); + }; + assertions = [ + { + assertion = cfg.dockerCompat -> !config.virtualisation.docker.enable; + message = "Option dockerCompat conflicts with docker"; + } + { + assertion = cfg.enableNvidia -> !config.virtualisation.docker.enableNvidia; + message = "Option enableNvidia conflicts with docker.enableNvidia"; + } + ]; + } + (lib.mkIf cfg.enableNvidia { + environment.etc."nvidia-container-runtime/config.toml".source = "${pkgs.nvidia-docker}/etc/podman-config.toml"; + }) + ]); } From df43c0f9da1ab1e2cbf55f467f1c47306b6d16ff Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Thu, 7 Jan 2021 05:40:52 +1000 Subject: [PATCH 2/4] nvidia-docker: install podman config --- .../virtualization/nvidia-docker/default.nix | 3 +++ .../virtualization/nvidia-docker/podman-config.toml | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 pkgs/applications/virtualization/nvidia-docker/podman-config.toml diff --git a/pkgs/applications/virtualization/nvidia-docker/default.nix b/pkgs/applications/virtualization/nvidia-docker/default.nix index 3f75e36ccae..f46e2341c83 100644 --- a/pkgs/applications/virtualization/nvidia-docker/default.nix +++ b/pkgs/applications/virtualization/nvidia-docker/default.nix @@ -78,6 +78,9 @@ stdenv.mkDerivation rec { --prefix LD_LIBRARY_PATH : /run/opengl-driver/lib:/run/opengl-driver-32/lib cp ${./config.toml} $out/etc/config.toml substituteInPlace $out/etc/config.toml --subst-var-by glibcbin ${lib.getBin glibc} + + cp ${./podman-config.toml} $out/etc/podman-config.toml + substituteInPlace $out/etc/podman-config.toml --subst-var-by glibcbin ${lib.getBin glibc} ''; meta = { diff --git a/pkgs/applications/virtualization/nvidia-docker/podman-config.toml b/pkgs/applications/virtualization/nvidia-docker/podman-config.toml new file mode 100644 index 00000000000..eb39699b96b --- /dev/null +++ b/pkgs/applications/virtualization/nvidia-docker/podman-config.toml @@ -0,0 +1,13 @@ +disable-require = true +#swarm-resource = "DOCKER_RESOURCE_GPU" + +[nvidia-container-cli] +#root = "/run/nvidia/driver" +#path = "/usr/bin/nvidia-container-cli" +environment = [] +#debug = "/var/log/nvidia-container-runtime-hook.log" +ldcache = "/tmp/ld.so.cache" +load-kmods = true +no-cgroups = true +#user = "root:video" +ldconfig = "@@glibcbin@/bin/ldconfig" From 890a298409c49d6004422a74931a99ca9591897c Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Thu, 7 Jan 2021 18:56:56 -0500 Subject: [PATCH 3/4] nvidia-docker: wrapProgram to pickup needed runc executable --- nixos/modules/virtualisation/podman.nix | 7 ++++--- .../virtualization/nvidia-docker/default.nix | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/nixos/modules/virtualisation/podman.nix b/nixos/modules/virtualisation/podman.nix index 02709176ce6..36c0ca8dfea 100644 --- a/nixos/modules/virtualisation/podman.nix +++ b/nixos/modules/virtualisation/podman.nix @@ -2,6 +2,7 @@ let cfg = config.virtualisation.podman; toml = pkgs.formats.toml { }; + nvidia-docker = pkgs.nvidia-docker.override { containerRuntimePath = "${pkgs.runc}/bin/runc"; }; inherit (lib) mkOption types; @@ -99,8 +100,8 @@ in containersConf.extraConfig = lib.optionalString cfg.enableNvidia (builtins.readFile (toml.generate "podman.nvidia.containers.conf" { engine = { - conmon_env_vars = [ "PATH=${lib.makeBinPath [ pkgs.nvidia-docker ]}" ]; - runtimes.nvidia = [ "${pkgs.nvidia-docker}/bin/nvidia-container-runtime" ]; + conmon_env_vars = [ "PATH=${lib.makeBinPath [ nvidia-docker ]}" ]; + runtimes.nvidia = [ "${nvidia-docker}/bin/nvidia-container-runtime" ]; }; })); }; @@ -117,7 +118,7 @@ in ]; } (lib.mkIf cfg.enableNvidia { - environment.etc."nvidia-container-runtime/config.toml".source = "${pkgs.nvidia-docker}/etc/podman-config.toml"; + environment.etc."nvidia-container-runtime/config.toml".source = "${nvidia-docker}/etc/podman-config.toml"; }) ]); } diff --git a/pkgs/applications/virtualization/nvidia-docker/default.nix b/pkgs/applications/virtualization/nvidia-docker/default.nix index f46e2341c83..f8098097c75 100644 --- a/pkgs/applications/virtualization/nvidia-docker/default.nix +++ b/pkgs/applications/virtualization/nvidia-docker/default.nix @@ -6,12 +6,20 @@ , makeWrapper , buildGoModule , buildGoPackage -, git , glibc +, docker +, linkFarm +, containerRuntimePath ? "${docker}/libexec/docker/runc" }: with lib; let libnvidia-container = callPackage ./libnvc.nix { }; + isolatedContainerRuntimePath = linkFarm "isolated_container_runtime_path" [ + { + name = "runc"; + path = containerRuntimePath; + } + ]; nvidia-container-runtime = buildGoPackage rec { pname = "nvidia-container-toolkit"; @@ -74,8 +82,13 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{bin,etc} cp -r bin $out + wrapProgram $out/bin/nvidia-container-cli \ --prefix LD_LIBRARY_PATH : /run/opengl-driver/lib:/run/opengl-driver-32/lib + + # nvidia-container-runtime invokes docker-runc or runc if that isn't available on PATH + wrapProgram $out/bin/nvidia-container-runtime --prefix PATH : ${isolatedContainerRuntimePath} + cp ${./config.toml} $out/etc/config.toml substituteInPlace $out/etc/config.toml --subst-var-by glibcbin ${lib.getBin glibc} From 7115e5ac8d7e7743989207e8947827dce02af8af Mon Sep 17 00:00:00 2001 From: Phillip Cloud Date: Fri, 8 Jan 2021 07:17:05 -0500 Subject: [PATCH 4/4] nvidia-docker: fix pname of nvidia-container-runtime --- pkgs/applications/virtualization/nvidia-docker/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/applications/virtualization/nvidia-docker/default.nix b/pkgs/applications/virtualization/nvidia-docker/default.nix index f8098097c75..0df2efebd82 100644 --- a/pkgs/applications/virtualization/nvidia-docker/default.nix +++ b/pkgs/applications/virtualization/nvidia-docker/default.nix @@ -22,7 +22,7 @@ with lib; let ]; nvidia-container-runtime = buildGoPackage rec { - pname = "nvidia-container-toolkit"; + pname = "nvidia-container-runtime"; version = "3.4.0"; src = fetchFromGitHub { owner = "NVIDIA";