k3s: init at 1.17.3+k3s1

This packages k3s as a single self-contained binary (as upstream k3s
does), but without having to download any impure already-built binaries.

The upstream packaging scripts are used. Due to k3s's rather complicated
packaging arrangement, this ends up being a rather long derivation.

maintainers/maintainer-list.nix

@@ -2404,6 +2404,12 @@
       fingerprint = "67FE 98F2 8C44 CF22 1828  E12F D57E FA62 5C9A 925F";
     }];
   };
+  euank = {
+    email = "euank-nixpkg@euank.com";
+    github = "euank";
+    githubId = 2147649;
+    name = "Euan Kemp";
+  };
   evanjs = {
     email = "evanjsx@gmail.com";
     github = "evanjs";

pkgs/applications/networking/cluster/k3s/default.nix

@@ -0,0 +1,233 @@
+with import <nixpkgs> {};
+
+{ stdenv, lib, makeWrapper, fetchFromGitHub, fetchurl, fetchzip }:
+
+with lib;
+
+# k3s is a kinda weird derivation. One of the main points of k3s is the
+# simplicity of it being one binary that can perform several tasks.
+# However, when you have a good package manager (like nix), that doesn't
+# actually make much of a difference; you don't really care if it's one binary
+# or 10 since with a good package manager, installing and running it is
+# identical.
+# Since upstream k3s packages itself as one large binary with several
+# "personalities" (in the form of subcommands like 'k3s agent' and 'k3s
+# kubectl'), it ends up being easiest to mostly mimic upstream packaging, with
+# some exceptions.
+# K3s also carries patches to some packages (such as containerd and cni
+# plugins), so we intentionally use the k3s versions of those binaries for k3s,
+# even if the upstream version of those binaries exist in nixpkgs already. In
+# the end, that means we have a thick k3s binary that behaves like the upstream
+# one for the most part.
+# However, k3s also bundles several pieces of unpatched software, from the
+# strongswan vpn software, to iptables, to socat, conntrack, busybox, etc.
+# Those pieces of software we entirely ignore upstream's handling of, and just
+# make sure they're in the path if desired.
+let
+  k3sVersion = "1.17.3+k3s1";     # k3s git tag
+  traefikChartVersion = "1.81.0"; # taken from ./scripts/version.sh at the above k3s tag
+  k3sRootVersion = "0.3.0";       # taken from .s/cripts/version.sh at the above k3s tag
+  # bundled into the k3s binary
+  traefikChart = fetchurl {
+    url = "https://kubernetes-charts.storage.googleapis.com/traefik-${traefikChartVersion}.tgz";
+    sha256 = "1aqpzgjlvqhil0g3angz94zd4xbl4iq0qmpjcy5aq1xv9qciwdi9";
+  };
+  # so, k3s is a complicated thing to package
+  # This derivation attempts to avoid including any random binaries from the
+  # internet. k3s-root is _mostly_ binaries built to be bundled in k3s (which
+  # we don't care about doing, we can add those as build or runtime
+  # dependencies using a real package manager).
+  # In addition to those binaries, it's also configuration though (right now
+  # mostly strongswan configuration), and k3s does use those files.
+  # As such, we download it in order to grab 'etc' and bundle it into the final
+  # k3s binary.
+  k3sRoot = fetchzip {
+    # Note: marked as apache 2.0 license
+    url = "https://github.com/rancher/k3s-root/releases/download/v${k3sRootVersion}/k3s-root-amd64.tar";
+    sha256 = "12xafn5jivl8lqdcs25b28xrc4mf7yf1xif5np169nvvxgvmpdxp";
+    stripRoot=false;
+  };
+  k3sPlugins = buildGoPackage rec {
+    name = "k3s-cni-plugins";
+    version = "0.7.6-k3s1"; # from ./scripts/version.sh 'VERSION_CNIPLUGINS'; update when k3s's repo is updated.
+
+    goPackagePath = "github.com/containernetworking/plugins";
+    subPackages = [ "." ];
+
+    src = fetchFromGitHub {
+      owner = "rancher";
+      repo = "plugins";
+      rev = "v${version}";
+      sha256 = "0ax72z1ziann352bp6khfds8vlf3bbkqckrkpx4l4jxgqks45izs";
+    };
+
+    meta = {
+      description = "k3s-cni-plugins";
+      license = licenses.asl20;
+      homepage = https://k3s.io;
+      maintainers = [];
+      platforms = platforms.linux;
+    };
+  };
+  # Grab this separately from a build because it's used by both stages of the
+  # k3s build.
+  k3sRepo = fetchgit {
+    url = "https://github.com/rancher/k3s";
+    rev = "v${k3sVersion}";
+    leaveDotGit = true; # for version / build date below
+    sha256 = "0qahyc0mf9glxj49va6d20mcncqg4svfic2iz8b1lqid5c4g68mm";
+  };
+  # Stage 1 of the k3s build:
+  # Let's talk about how k3s is structured.
+  # One of the ideas of k3s is that there's the single "k3s" binary which can
+  # do everything you need, from running a k3s server, to being a worker node,
+  # to running kubectl.
+  # The way that actually works is that k3s is a single go binary that contains
+  # a bunch of bindata that it unpacks at runtime into directories (either the
+  # user's home directory or /var/lib/rancher if run as root).
+  # This bindata includes both binaries and configuration.
+  # In order to let nixpkgs do all its autostripping/patching/etc, we split this into two derivations.
+  # First, we build all the binaries that get packed into the thick k3s binary
+  # (and output them from one derivation so they'll all be suitably patched up).
+  # Then, we bundle those binaries into our thick k3s binary and use that as
+  # the final single output.
+  # This approach was chosen because it ensures the bundled binaries all are
+  # correctly built to run with nix (we can lean on the existing buildGoPackage
+  # stuff), and we can again lean on that tooling for the final k3s binary too.
+  # Other alternatives would be to manually run the
+  # strip/patchelf/remove-references step ourselves in the installPhase of the
+  # derivation when we've built all the binaries, but haven't bundled them in
+  # with generated bindata yet.
+  k3sBuildStage1 = buildGoPackage rec {
+    name = "k3s-build-1";
+    version = "${k3sVersion}";
+
+    goPackagePath = "github.com/rancher/k3s";
+
+    src = k3sRepo;
+
+    patches = [ ./patches/00-k3s.patch ];
+
+    nativeBuildInputs = [ pkgconfig autoPatchelfHook breakpointHook ];
+    buildInputs = [ git runc libseccomp ];
+
+    buildPhase = ''
+      pushd go/src/${goPackagePath}
+
+      patchShebangs ./scripts/build ./scripts/version.sh
+      mkdir -p bin
+      ./scripts/build
+
+      popd
+    '';
+
+    installPhase = ''
+      pushd go/src/${goPackagePath}
+
+      mkdir -p "$bin/bin"
+      install -m 0755 -t "$bin/bin" ./bin/*
+
+      popd
+    '';
+
+    meta = {
+      description = "The various binaries that get packaged into the final k3s binary.";
+      license = licenses.asl20;
+      homepage = https://k3s.io;
+      maintainers = [];
+      platforms = platforms.linux;
+    };
+  };
+  k3sBuild = buildGoPackage rec {
+    name = "k3s-build";
+    version = "${k3sVersion}";
+
+    goPackagePath = "github.com/rancher/k3s";
+
+    src = k3sRepo;
+
+    patches = [ ./patches/00-k3s.patch ];
+
+    nativeBuildInputs = [ pkgconfig autoPatchelfHook breakpointHook ];
+    buildInputs = [ git k3sBuildStage1 ];
+
+    # In order to build the thick k3s binary (which is what
+    # ./scripts/package-cli does), we need to get all the binaries that script
+    # expects in place.
+    buildPhase = ''
+      pushd go/src/${goPackagePath}
+
+      patchShebangs ./scripts/build ./scripts/version.sh ./scripts/package-cli
+
+      mkdir -p bin
+
+      install -m 0755 -t ./bin ${k3sBuildStage1}/bin/*
+      install -m 0755 -T "${k3sPlugins}/bin/plugins" ./bin/cni
+      # Note: use the already-nixpkgs-bundled k3s rather than the one bundled
+      # in k3s because the k3s one is completely unmodified from upstream
+      # (unlike containerd, cni, etc)
+      install -m 0755 -T "${runc}/bin/runc" ./bin/runc
+      cp -R "${k3sRoot}/etc" ./etc
+      mkdir -p "build/static/charts"
+      cp "${traefikChart}" "build/static/charts/traefik-${traefikChartVersion}.tgz"
+
+      ./scripts/package-cli
+
+      popd
+    '';
+
+    installPhase = ''
+      pushd go/src/${goPackagePath}
+
+      mkdir -p "$bin/bin"
+      install -m 0755 -t "$bin/bin" ./dist/artifacts/k3s
+
+      popd
+    '';
+
+    meta = {
+      description = "The k3s go binary which is used by the final wrapped output below.";
+      license = licenses.asl20;
+      homepage = https://k3s.io;
+      maintainers = [];
+      platforms = platforms.linux;
+    };
+  };
+in
+stdenv.mkDerivation rec {
+  name = "k3s";
+
+  # Important utilities used by  the kubelet, see
+  # https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494
+  # Note the list in that issue is stale and some aren't relevant for k3s.
+  k3sRuntimeDeps = [ 
+    socat iptables iproute bridge-utils ethtool utillinux ipset conntrack-tools
+  ];
+
+  buildInputs = [ 
+    k3sBuild makeWrapper
+  ] ++ k3sRuntimeDeps;
+
+  unpackPhase = "true";
+
+  # And, one final derivation (you thought the last one was it, right?)
+  # We got the binary we wanted above, but it doesn't have all the runtime
+  # dependencies k8s wants, including mount utilities for kubelet, networking
+  # tools for cni/kubelet stuff, etc
+  # Use a wrapper script to reference all the binaries that k3s tries to
+  # execute, but that we didn't bundle with it.
+  installPhase = ''
+    mkdir -p "$out/bin"
+    makeWrapper ${k3sBuild}/bin/k3s "$out/bin/k3s" \
+      --prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \
+      --prefix PATH : "$out/bin"
+  '';
+
+    meta = {
+      description = "A lightweight Kubernetes distribution.";
+      license = licenses.asl20;
+      homepage = https://k3s.io;
+      maintainers = with maintainers; [ euank ];
+      platforms = platforms.linux;
+    };
+}

pkgs/applications/networking/cluster/k3s/patches/00-k3s.patch

@@ -0,0 +1,74 @@
+diff --git a/main.go b/main.go
+index 62908bb7bb..0527222887 100644
+--- a/main.go
++++ b/main.go
+@@ -1,5 +1,5 @@
+ //go:generate go run pkg/codegen/cleanup/main.go
+-//go:generate /bin/rm -rf pkg/generated
++//go:generate rm -rf pkg/generated
+ //go:generate go run pkg/codegen/main.go
+ //go:generate go fmt pkg/deploy/zz_generated_bindata.go
+ //go:generate go fmt pkg/static/zz_generated_bindata.go
+diff --git a/scripts/build b/scripts/build
+index 72d3c07ece..3e5455b262 100755
+--- a/scripts/build
++++ b/scripts/build
+@@ -10,7 +10,8 @@ PKG_CONTAINERD="github.com/containerd/containerd"
+ PKG_RANCHER_CONTAINERD="github.com/rancher/containerd"
+ PKG_CRICTL="github.com/kubernetes-sigs/cri-tools"
+ 
+-buildDate=$(date -u '+%Y-%m-%dT%H:%M:%SZ')
++# Deterministic build date
++buildDate="$(date -d "$(git log -1 --format=%ai)" -u "+%Y-%m-%dT%H:%M:%SZ")"
+ 
+ VENDOR_PREFIX="${PKG}/vendor/"
+ VERSIONFLAGS="
+@@ -82,17 +83,7 @@ cleanup() {
+ }
+ 
+ INSTALLBIN=$(pwd)/bin
+-if [ ! -x ${INSTALLBIN}/cni ]; then
+-(
+-    echo Building cni
+-    TMPDIR=$(mktemp -d)
+-    trap cleanup EXIT
+-    WORKDIR=$TMPDIR/src/github.com/containernetworking/plugins
+-    git clone -b $VERSION_CNIPLUGINS https://github.com/rancher/plugins.git $WORKDIR
+-    cd $WORKDIR
+-    GOPATH=$TMPDIR CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$LDFLAGS $STATIC" -o $INSTALLBIN/cni
+-)
+-fi
++# skip building cni, use our separately built one
+ # echo Building agent
+ # CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/k3s-agent ./cmd/agent/main.go
+ echo Building server
+@@ -108,9 +99,8 @@ ln -s containerd ./bin/ctr
+ #CGO_ENABLED=1 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC_SQLITE" -o bin/ctr ./cmd/ctr/main.go
+ # echo Building containerd
+ # CGO_ENABLED=0 go build -tags "$TAGS" -ldflags "$VERSIONFLAGS $LDFLAGS $STATIC" -o bin/containerd ./cmd/containerd/
+-echo Building runc
+-make EXTRA_LDFLAGS="-w -s" BUILDTAGS="apparmor seccomp" -C ./vendor/github.com/opencontainers/runc static
+-cp -f ./vendor/github.com/opencontainers/runc/runc ./bin/runc
++
++# skip building runc; use our packaged one
+ 
+ echo Building containerd-shim
+ make -C ./vendor/github.com/containerd/containerd bin/containerd-shim
+diff --git a/scripts/package-cli b/scripts/package-cli
+index 4c66ce32df..6d1e0c03cb 100755
+--- a/scripts/package-cli
++++ b/scripts/package-cli
+@@ -55,10 +55,10 @@ LDFLAGS="
+     -X github.com/rancher/k3s/pkg/version.GitCommit=${COMMIT:0:8}
+     -w -s
+ "
+-STATIC="-extldflags '-static'"
+ if [ "$DQLITE" = "true" ]; then
+     DQLITE_TAGS="dqlite"
+ fi
+-CGO_ENABLED=0 go build -tags "$DQLITE_TAGS" -ldflags "$LDFLAGS $STATIC" -o ${CMD_NAME} ./cmd/k3s/main.go
++go build -tags "$DQLITE_TAGS" -ldflags "$LDFLAGS" -o ${CMD_NAME} ./cmd/k3s/main.go
+ 
+-./scripts/build-upload ${CMD_NAME} ${COMMIT}
++# for nixos, don't upload it
++# ./scripts/build-upload ${CMD_NAME} ${COMMIT}

pkgs/top-level/all-packages.nix

@@ -20159,6 +20159,8 @@ in
     boost = boost155.override { enablePython = true; };
   };
 
+  k3s = callPackage ../applications/networking/cluster/k3s {};
+
   k9copy = libsForQt5.callPackage ../applications/video/k9copy {};
 
   kail = callPackage ../tools/networking/kail {  };