diff --git a/nixos/modules/hardware/video/nvidia.nix b/nixos/modules/hardware/video/nvidia.nix index 6ba8130af71..80ea7bc5d5c 100644 --- a/nixos/modules/hardware/video/nvidia.nix +++ b/nixos/modules/hardware/video/nvidia.nix @@ -172,6 +172,11 @@ in environment.systemPackages = [ nvidia_x11.bin nvidia_x11.settings ] ++ lib.filter (p: p != null) [ nvidia_x11.persistenced ]; + systemd.tmpfiles.rules = optional config.virtualisation.docker.enableNvidia + "L+ /run/nvidia-docker/bin - - - - ${nvidia_x11.bin}/origBin" + ++ optional (nvidia_x11.persistenced != null && config.virtualisation.docker.enableNvidia) + "L+ /run/nvidia-docker/extras/bin/nvidia-persistenced - - - - ${nvidia_x11.persistenced}/origBin/nvidia-persistenced"; + boot.extraModulePackages = [ nvidia_x11.bin ]; # nvidia-uvm is required by CUDA applications. diff --git a/nixos/modules/virtualisation/docker.nix b/nixos/modules/virtualisation/docker.nix index a1a32c1c59a..4ee84c5268e 100644 --- a/nixos/modules/virtualisation/docker.nix +++ b/nixos/modules/virtualisation/docker.nix @@ -52,6 +52,15 @@ in ''; }; + enableNvidia = + mkOption { + type = types.bool; + default = false; + description = '' + Enable nvidia-docker wrapper, supporting NVIDIA GPUs inside docker containers. + ''; + }; + liveRestore = mkOption { type = types.bool; @@ -140,7 +149,8 @@ in ###### implementation config = mkIf cfg.enable (mkMerge [{ - environment.systemPackages = [ cfg.package ]; + environment.systemPackages = [ cfg.package ] + ++ optional cfg.enableNvidia pkgs.nvidia-docker; users.groups.docker.gid = config.ids.gids.docker; systemd.packages = [ cfg.package ]; @@ -157,6 +167,7 @@ in --log-driver=${cfg.logDriver} \ ${optionalString (cfg.storageDriver != null) "--storage-driver=${cfg.storageDriver}"} \ ${optionalString cfg.liveRestore "--live-restore" } \ + ${optionalString cfg.enableNvidia "--add-runtime nvidia=${pkgs.nvidia-docker}/bin/nvidia-container-runtime" } \ ${cfg.extraOptions} '']; ExecReload=[ @@ -165,7 +176,8 @@ in ]; }; - path = [ pkgs.kmod ] ++ (optional (cfg.storageDriver == "zfs") pkgs.zfs); + path = [ pkgs.kmod ] ++ optional (cfg.storageDriver == "zfs") pkgs.zfs + ++ optional cfg.enableNvidia pkgs.nvidia-docker; }; systemd.sockets.docker = { @@ -179,7 +191,6 @@ in }; }; - systemd.services.docker-prune = { description = "Prune docker resources"; @@ -194,7 +205,15 @@ in startAt = optional cfg.autoPrune.enable cfg.autoPrune.dates; }; + + assertions = [ + { assertion = cfg.enableNvidia -> config.hardware.opengl.driSupport32Bit or false; + message = "Option enableNvidia requires 32bit support libraries"; + }]; } + (mkIf cfg.enableNvidia { + environment.etc."nvidia-container-runtime/config.toml".source = "${pkgs.nvidia-docker}/etc/config.toml"; + }) ]); imports = [ diff --git a/pkgs/applications/virtualization/nvidia-docker/config.toml b/pkgs/applications/virtualization/nvidia-docker/config.toml new file mode 100644 index 00000000000..bbd166995f3 --- /dev/null +++ b/pkgs/applications/virtualization/nvidia-docker/config.toml @@ -0,0 +1,13 @@ +disable-require = false +#swarm-resource = "DOCKER_RESOURCE_GPU" + +[nvidia-container-cli] +#root = "/run/nvidia/driver" +#path = "/usr/bin/nvidia-container-cli" +environment = [] +#debug = "/var/log/nvidia-container-runtime-hook.log" +ldcache = "/tmp/ld.so.cache" +load-kmods = true +#no-cgroups = false +#user = "root:video" +ldconfig = "@@glibcbin@/bin/ldconfig" diff --git a/pkgs/applications/virtualization/nvidia-docker/default.nix b/pkgs/applications/virtualization/nvidia-docker/default.nix new file mode 100644 index 00000000000..3e79ff84882 --- /dev/null +++ b/pkgs/applications/virtualization/nvidia-docker/default.nix @@ -0,0 +1,84 @@ +{ stdenv, lib, fetchFromGitHub, fetchpatch, callPackage, makeWrapper +, buildGoPackage, runc, libelf, libcap, libseccomp, glibc }: + +with lib; let + + glibc-ldconf = glibc.overrideAttrs (oldAttrs: { + # ldconfig needs help reading libraries that have been patchelf-ed, as the + # .dynstr section is no longer in the first LOAD segment. See also + # https://sourceware.org/bugzilla/show_bug.cgi?id=23964 and + # https://github.com/NixOS/patchelf/issues/44 + patches = oldAttrs.patches ++ [ (fetchpatch { + name = "ldconfig-patchelf.patch"; + url = "https://sourceware.org/bugzilla/attachment.cgi?id=11444"; + sha256 = "0nzzmq7pli37iyjrgcmvcy92piiwjybpw245ds7q43pbgdm7lc3s"; + })]; + }); + + libnvidia-container = callPackage ./libnvc.nix { }; + + nvidia-container-runtime = fetchFromGitHub { + owner = "NVIDIA"; + repo = "nvidia-container-runtime"; + rev = "runtime-v2.0.0"; + sha256 = "0jcj5xxbg7x7gyhbb67h3ds6vly62gx7j02zm6lg102h34jajj7a"; + }; + + nvidia-container-runtime-hook = buildGoPackage rec { + name = "nvidia-container-runtime-hook-${version}"; + version = "1.4.0"; + + goPackagePath = "nvidia-container-runtime-hook"; + + src = "${nvidia-container-runtime}/hook/nvidia-container-runtime-hook"; + }; + + nvidia-runc = runc.overrideAttrs (oldAttrs: rec { + name = "nvidia-runc"; + version = "1.0.0-rc6"; + src = fetchFromGitHub { + owner = "opencontainers"; + repo = "runc"; + rev = "v${version}"; + sha256 = "1jwacb8xnmx5fr86gximhbl9dlbdwj3rpf27hav9q1si86w5pb1j"; + }; + patches = [ "${nvidia-container-runtime}/runtime/runc/3f2f8b84a77f73d38244dd690525642a72156c64/0001-Add-prestart-hook-nvidia-container-runtime-hook-to-t.patch" ]; + }); + +in stdenv.mkDerivation rec { + name = "nvidia-docker-${version}"; + version = "2.0.3"; + + src = fetchFromGitHub { + owner = "NVIDIA"; + repo = "nvidia-docker"; + rev = "v${version}"; + sha256 = "1vx5m591mnvcb9vy0196x5lh3r8swjsk0fnlv5h62m7m4m07v6wx"; + }; + + nativeBuildInputs = [ makeWrapper ]; + + buildPhase = '' + mkdir bin + cp nvidia-docker bin + cp ${libnvidia-container}/bin/nvidia-container-cli bin + cp ${nvidia-container-runtime-hook}/bin/nvidia-container-runtime-hook bin + cp ${nvidia-runc}/bin/runc bin/nvidia-container-runtime + ''; + + installPhase = '' + mkdir -p $out/{bin,etc} + cp -r bin $out + wrapProgram $out/bin/nvidia-container-cli \ + --prefix LD_LIBRARY_PATH : /run/opengl-driver/lib:/run/opengl-driver-32/lib + cp ${./config.toml} $out/etc/config.toml + substituteInPlace $out/etc/config.toml --subst-var-by glibcbin ${lib.getBin glibc-ldconf} + ''; + + meta = { + homepage = https://github.com/NVIDIA/nvidia-docker; + description = "NVIDIA container runtime for Docker"; + license = licenses.bsd3; + platforms = platforms.linux; + }; +} diff --git a/pkgs/applications/virtualization/nvidia-docker/libnvc-ldconfig-and-path-fixes.patch b/pkgs/applications/virtualization/nvidia-docker/libnvc-ldconfig-and-path-fixes.patch new file mode 100644 index 00000000000..043c1efade8 --- /dev/null +++ b/pkgs/applications/virtualization/nvidia-docker/libnvc-ldconfig-and-path-fixes.patch @@ -0,0 +1,130 @@ +diff --git a/src/ldcache.c b/src/ldcache.c +index 38bab05..e1abc89 100644 +--- a/src/ldcache.c ++++ b/src/ldcache.c +@@ -108,40 +108,27 @@ ldcache_close(struct ldcache *ctx) + + int + ldcache_resolve(struct ldcache *ctx, uint32_t arch, const char *root, const char * const libs[], +- char *paths[], size_t size, ldcache_select_fn select, void *select_ctx) ++ char *paths[], size_t size, const char* version) + { + char path[PATH_MAX]; +- struct header_libc6 *h; +- int override; ++ char dir[PATH_MAX]; ++ char lib[PATH_MAX]; + +- h = (struct header_libc6 *)ctx->ptr; + memset(paths, 0, size * sizeof(*paths)); + +- for (uint32_t i = 0; i < h->nlibs; ++i) { +- int32_t flags = h->libs[i].flags; +- char *key = (char *)ctx->ptr + h->libs[i].key; +- char *value = (char *)ctx->ptr + h->libs[i].value; +- +- if (!(flags & LD_ELF) || (flags & LD_ARCH_MASK) != arch) ++ for (size_t j = 0; j < size; ++j) { ++ snprintf(dir, 100, "/run/opengl-driver%s/lib", ++ arch == LD_I386_LIB32 ? "-32" : ""); ++ if (!strncmp(libs[j], "libvdpau_nvidia.so", 100)) ++ strcat(dir, "/vdpau"); ++ snprintf(lib, 100, "%s/%s.%s", dir, libs[j], version); ++ if (path_resolve_full(ctx->err, path, "/", lib) < 0) ++ return (-1); ++ if (!file_exists(ctx->err, path)) + continue; +- +- for (size_t j = 0; j < size; ++j) { +- if (!str_has_prefix(key, libs[j])) +- continue; +- if (path_resolve(ctx->err, path, root, value) < 0) +- return (-1); +- if (paths[j] != NULL && str_equal(paths[j], path)) +- continue; +- if ((override = select(ctx->err, select_ctx, root, paths[j], path)) < 0) +- return (-1); +- if (override) { +- free(paths[j]); +- paths[j] = xstrdup(ctx->err, path); +- if (paths[j] == NULL) +- return (-1); +- } +- break; +- } ++ paths[j] = xstrdup(ctx->err, path); ++ if (paths[j] == NULL) ++ return (-1); + } + return (0); + } +diff --git a/src/ldcache.h b/src/ldcache.h +index 33d78dd..2b087db 100644 +--- a/src/ldcache.h ++++ b/src/ldcache.h +@@ -50,6 +50,6 @@ void ldcache_init(struct ldcache *, struct error *, const char *); + int ldcache_open(struct ldcache *); + int ldcache_close(struct ldcache *); + int ldcache_resolve(struct ldcache *, uint32_t, const char *, const char * const [], +- char *[], size_t, ldcache_select_fn, void *); ++ char *[], size_t, const char*); + + #endif /* HEADER_LDCACHE_H */ +diff --git a/src/nvc_info.c b/src/nvc_info.c +index cc96542..3fe7612 100644 +--- a/src/nvc_info.c ++++ b/src/nvc_info.c +@@ -163,15 +163,13 @@ find_library_paths(struct error *err, struct nvc_driver_info *info, const char * + if (path_resolve_full(err, path, root, ldcache) < 0) + return (-1); + ldcache_init(&ld, err, path); +- if (ldcache_open(&ld) < 0) +- return (-1); + + info->nlibs = size; + info->libs = array_new(err, size); + if (info->libs == NULL) + goto fail; + if (ldcache_resolve(&ld, LIB_ARCH, root, libs, +- info->libs, info->nlibs, select_libraries, info) < 0) ++ info->libs, info->nlibs, info->nvrm_version) < 0) + goto fail; + + info->nlibs32 = size; +@@ -179,13 +177,11 @@ find_library_paths(struct error *err, struct nvc_driver_info *info, const char * + if (info->libs32 == NULL) + goto fail; + if (ldcache_resolve(&ld, LIB32_ARCH, root, libs, +- info->libs32, info->nlibs32, select_libraries, info) < 0) ++ info->libs32, info->nlibs32, info->nvrm_version) < 0) + goto fail; + rv = 0; + + fail: +- if (ldcache_close(&ld) < 0) +- return (-1); + return (rv); + } + +@@ -199,7 +195,7 @@ find_binary_paths(struct error *err, struct nvc_driver_info *info, const char *r + char path[PATH_MAX]; + int rv = -1; + +- if ((env = secure_getenv("PATH")) == NULL) { ++ if ((env = "/run/nvidia-docker/bin:/run/nvidia-docker/extras/bin") == NULL) { + error_setx(err, "environment variable PATH not found"); + return (-1); + } +diff --git a/src/nvc_ldcache.c b/src/nvc_ldcache.c +index d41a24d..65b7878 100644 +--- a/src/nvc_ldcache.c ++++ b/src/nvc_ldcache.c +@@ -331,7 +331,7 @@ nvc_ldcache_update(struct nvc_context *ctx, const struct nvc_container *cnt) + if (validate_args(ctx, cnt != NULL) < 0) + return (-1); + +- argv = (char * []){cnt->cfg.ldconfig, cnt->cfg.libs_dir, cnt->cfg.libs32_dir, NULL}; ++ argv = (char * []){cnt->cfg.ldconfig, "-f", "/tmp/ld.so.conf.nvidia-host", "-C", "/tmp/ld.so.cache.nvidia-host", cnt->cfg.libs_dir, cnt->cfg.libs32_dir, NULL}; + if (*argv[0] == '@') { + /* + * We treat this path specially to be relative to the host filesystem. diff --git a/pkgs/applications/virtualization/nvidia-docker/libnvc.nix b/pkgs/applications/virtualization/nvidia-docker/libnvc.nix new file mode 100644 index 00000000000..46b8e3ba43c --- /dev/null +++ b/pkgs/applications/virtualization/nvidia-docker/libnvc.nix @@ -0,0 +1,53 @@ +{ stdenv, lib, fetchFromGitHub, libelf, libcap, libseccomp }: + +with lib; let + + modp-ver = "396.51"; + + nvidia-modprobe = fetchFromGitHub { + owner = "NVIDIA"; + repo = "nvidia-modprobe"; + rev = modp-ver; + sha256 = "1fw2qwc84k64agw6fx2v0mjf88aggph9c6qhs4cv7l3gmflv8qbk"; + }; + +in stdenv.mkDerivation rec { + name = "libnvidia-container-${version}"; + version = "1.0.0"; + + src = fetchFromGitHub { + owner = "NVIDIA"; + repo = "libnvidia-container"; + rev = "v${version}"; + sha256 = "1ws6mfsbgxhzlb5w1r8qqg2arvxkr21n59i4cqsyz3h5jsqsflbw"; + }; + + # locations of nvidia-driver libraries are not resolved via ldconfig which + # doesn't get used on NixOS. Additional support binaries like nvidia-smi are + # not resolved via the environment PATH but via the derivation output path. + patches = [ ./libnvc-ldconfig-and-path-fixes.patch ]; + + makeFlags = [ + "WITH_LIBELF=yes" + "prefix=$(out)" + ]; + + postPatch = '' + sed -i 's/^REVISION :=.*/REVISION = ${src.rev}/' mk/common.mk + sed -i 's/^COMPILER :=.*/COMPILER = $(CC)/' mk/common.mk + + mkdir -p deps/src/nvidia-modprobe-${modp-ver} + cp -r ${nvidia-modprobe}/* deps/src/nvidia-modprobe-${modp-ver} + chmod -R u+w deps/src + touch deps/src/nvidia-modprobe-${modp-ver}/.download_stamp + ''; + + buildInputs = [ libelf libcap libseccomp ]; + + meta = { + homepage = https://github.com/NVIDIA/libnvidia-container; + description = "NVIDIA container runtime library"; + license = licenses.bsd3; + platforms = platforms.linux; + }; +} diff --git a/pkgs/os-specific/linux/nvidia-x11/builder.sh b/pkgs/os-specific/linux/nvidia-x11/builder.sh index 46ea5a55aa5..c0ebb768eea 100755 --- a/pkgs/os-specific/linux/nvidia-x11/builder.sh +++ b/pkgs/os-specific/linux/nvidia-x11/builder.sh @@ -129,6 +129,8 @@ installPhase() { for i in nvidia-cuda-mps-control nvidia-cuda-mps-server nvidia-smi nvidia-debugdump; do if [ -e "$i" ]; then install -Dm755 $i $bin/bin/$i + # unmodified binary backup for mounting in containers + install -Dm755 $i $bin/origBin/$i patchelf --interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" \ --set-rpath $out/lib:$libPath $bin/bin/$i fi diff --git a/pkgs/os-specific/linux/nvidia-x11/persistenced.nix b/pkgs/os-specific/linux/nvidia-x11/persistenced.nix index 474d3a69119..c6993d146f1 100644 --- a/pkgs/os-specific/linux/nvidia-x11/persistenced.nix +++ b/pkgs/os-specific/linux/nvidia-x11/persistenced.nix @@ -16,6 +16,11 @@ stdenv.mkDerivation rec { installFlags = [ "PREFIX=$(out)" ]; postFixup = '' + # Save a copy of persistenced for mounting in containers + mkdir $out/origBin + cp $out/{bin,origBin}/nvidia-persistenced + patchelf --set-interpreter /lib64/ld-linux-x86-64.so.2 $out/origBin/nvidia-persistenced + patchelf --set-rpath "$(patchelf --print-rpath $out/bin/nvidia-persistenced):${nvidia_x11}/lib" \ $out/bin/nvidia-persistenced ''; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 5dc6fe17bcc..50a93da4aa1 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -11839,6 +11839,8 @@ in nv-codec-headers = callPackage ../development/libraries/nv-codec-headers { }; + nvidia-docker = callPackage ../applications/virtualization/nvidia-docker { }; + nvidia-texture-tools = callPackage ../development/libraries/nvidia-texture-tools { }; nvidia-video-sdk = callPackage ../development/libraries/nvidia-video-sdk { };