diff options
Diffstat (limited to 'nixpkgs/pkgs/applications/networking/cluster/k3s')
16 files changed, 1412 insertions, 0 deletions
diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/1_24/chart-versions.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_24/chart-versions.nix new file mode 100644 index 000000000000..8c40604d0f1c --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_24/chart-versions.nix @@ -0,0 +1,10 @@ +{ + traefik-crd = { + url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-20.3.1+up20.3.0.tgz"; + sha256 = "1775vjldvqvhzdbzanxhbaqbmkih09yb91im651q8bc7z5sb9ckn"; + }; + traefik = { + url = "https://k3s.io/k3s-charts/assets/traefik/traefik-20.3.1+up20.3.0.tgz"; + sha256 = "1rj0f0n0vgjcbzfwzhqmsd501i2f6vw145w9plbp8gwdyzmg2nc6"; + }; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/1_24/default.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_24/default.nix new file mode 100644 index 000000000000..25b615d6718f --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_24/default.nix @@ -0,0 +1,332 @@ +{ stdenv +, lib +, makeWrapper +, socat +, iptables +, iproute2 +, ipset +, bridge-utils +, btrfs-progs +, conntrack-tools +, buildGoModule +, runc +, rsync +, kmod +, libseccomp +, pkg-config +, ethtool +, util-linux +, fetchFromGitHub +, fetchurl +, fetchzip +, fetchgit +, zstd +, yq-go +, sqlite +, nixosTests +, k3s +, pkgsBuildBuild +}: + +# k3s is a kinda weird derivation. One of the main points of k3s is the +# simplicity of it being one binary that can perform several tasks. +# However, when you have a good package manager (like nix), that doesn't +# actually make much of a difference; you don't really care if it's one binary +# or 10 since with a good package manager, installing and running it is +# identical. +# Since upstream k3s packages itself as one large binary with several +# "personalities" (in the form of subcommands like 'k3s agent' and 'k3s +# kubectl'), it ends up being easiest to mostly mimic upstream packaging, with +# some exceptions. +# K3s also carries patches to some packages (such as containerd and cni +# plugins), so we intentionally use the k3s versions of those binaries for k3s, +# even if the upstream version of those binaries exist in nixpkgs already. In +# the end, that means we have a thick k3s binary that behaves like the upstream +# one for the most part. +# However, k3s also bundles several pieces of unpatched software, from the +# strongswan vpn software, to iptables, to socat, conntrack, busybox, etc. +# Those pieces of software we entirely ignore upstream's handling of, and just +# make sure they're in the path if desired. +let + k3sVersion = "1.24.10+k3s1"; # k3s git tag + k3sCommit = "546a94e9ae1c3be6f9c0dcde32a6e6672b035bc8"; # k3s git commit at the above version + k3sRepoSha256 = "sha256-HfkGb3GtR2wQkVIze26aFh6A6W0fegr8ovpSel7oujQ="; + k3sVendorHash = "sha256-YAerisDr/knlKPaO2fVMZA4FUpwshFmkpi3mJAmLqKM="; + + # Based on the traefik charts here: https://github.com/k3s-io/k3s/blob/v1.24.10%2Bk3s1/scripts/download#L29-L32 + # see also https://github.com/k3s-io/k3s/blob/v1.24.10%2Bk3s1/manifests/traefik.yaml#L8-L16 + # At the time of writing, there are two traefik charts, and that's it + charts = import ./chart-versions.nix; + + # taken from ./scripts/version.sh VERSION_ROOT https://github.com/k3s-io/k3s/blob/v1.24.10%2Bk3s1/scripts/version.sh#L56 + k3sRootVersion = "0.12.1"; + k3sRootSha256 = "sha256-xCXbarWztnvW2xn3cGa84hie3OevVZeGEDWh+Uf3RBw="; + + # taken from ./scripts/version.sh VERSION_CNIPLUGINS https://github.com/k3s-io/k3s/blob/v1.24.10%2Bk3s1/scripts/version.sh#L49 + k3sCNIVersion = "1.1.1-k3s1"; + k3sCNISha256 = "14mb3zsqibj1sn338gjmsyksbm0mxv9p016dij7zidccx2rzn6nl"; + + # taken from go.mod, the 'github.com/containerd/containerd' line + # run `grep github.com/containerd/containerd go.mod | head -n1 | awk '{print $4}'` + # https://github.com/k3s-io/k3s/blob/v1.24.10%2Bk3s1/go.mod#L10 + containerdVersion = "1.5.16-k3s1"; + containerdSha256 = "sha256-dxC44qE1A20Hd2j77Ir9Sla8xncttswWIuGGM/5FWi8="; + + # run `grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}'` in the k3s repo at the tag + # https://github.com/k3s-io/k3s/blob/v1.24.10%2Bk3s1/go.mod#L18 + criCtlVersion = "1.24.0-k3s1"; + + baseMeta = k3s.meta; + + # https://github.com/k3s-io/k3s/blob/5fb370e53e0014dc96183b8ecb2c25a61e891e76/scripts/build#L19-L40 + versionldflags = [ + "-X github.com/rancher/k3s/pkg/version.Version=v${k3sVersion}" + "-X github.com/rancher/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitVersion=v${k3sVersion}" + "-X k8s.io/client-go/pkg/version.gitCommit=${k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitTreeState=clean" + "-X k8s.io/client-go/pkg/version.buildDate=1970-01-01T01:01:01Z" + "-X k8s.io/component-base/version.gitVersion=v${k3sVersion}" + "-X k8s.io/component-base/version.gitCommit=${k3sCommit}" + "-X k8s.io/component-base/version.gitTreeState=clean" + "-X k8s.io/component-base/version.buildDate=1970-01-01T01:01:01Z" + "-X github.com/kubernetes-sigs/cri-tools/pkg/version.Version=v${criCtlVersion}" + "-X github.com/containerd/containerd/version.Version=v${containerdVersion}" + "-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd" + ]; + + # bundled into the k3s binary + traefikChart = fetchurl charts.traefik; + traefik-crdChart = fetchurl charts.traefik-crd; + + # so, k3s is a complicated thing to package + # This derivation attempts to avoid including any random binaries from the + # internet. k3s-root is _mostly_ binaries built to be bundled in k3s (which + # we don't care about doing, we can add those as build or runtime + # dependencies using a real package manager). + # In addition to those binaries, it's also configuration though (right now + # mostly strongswan configuration), and k3s does use those files. + # As such, we download it in order to grab 'etc' and bundle it into the final + # k3s binary. + k3sRoot = fetchzip { + # Note: marked as apache 2.0 license + url = "https://github.com/k3s-io/k3s-root/releases/download/v${k3sRootVersion}/k3s-root-amd64.tar"; + sha256 = k3sRootSha256; + stripRoot = false; + }; + k3sCNIPlugins = buildGoModule rec { + pname = "k3s-cni-plugins"; + version = k3sCNIVersion; + vendorHash = null; + + subPackages = [ "." ]; + + src = fetchFromGitHub { + owner = "rancher"; + repo = "plugins"; + rev = "v${version}"; + sha256 = k3sCNISha256; + }; + + postInstall = '' + mv $out/bin/plugins $out/bin/cni + ''; + + meta = baseMeta // { + description = "CNI plugins, as patched by rancher for k3s"; + }; + }; + # Grab this separately from a build because it's used by both stages of the + # k3s build. + k3sRepo = fetchgit { + url = "https://github.com/k3s-io/k3s"; + rev = "v${k3sVersion}"; + sha256 = k3sRepoSha256; + }; + # Stage 1 of the k3s build: + # Let's talk about how k3s is structured. + # One of the ideas of k3s is that there's the single "k3s" binary which can + # do everything you need, from running a k3s server, to being a worker node, + # to running kubectl. + # The way that actually works is that k3s is a single go binary that contains + # a bunch of bindata that it unpacks at runtime into directories (either the + # user's home directory or /var/lib/rancher if run as root). + # This bindata includes both binaries and configuration. + # In order to let nixpkgs do all its autostripping/patching/etc, we split this into two derivations. + # First, we build all the binaries that get packed into the thick k3s binary + # (and output them from one derivation so they'll all be suitably patched up). + # Then, we bundle those binaries into our thick k3s binary and use that as + # the final single output. + # This approach was chosen because it ensures the bundled binaries all are + # correctly built to run with nix (we can lean on the existing buildGoModule + # stuff), and we can again lean on that tooling for the final k3s binary too. + # Other alternatives would be to manually run the + # strip/patchelf/remove-references step ourselves in the installPhase of the + # derivation when we've built all the binaries, but haven't bundled them in + # with generated bindata yet. + + k3sServer = buildGoModule rec { + pname = "k3s-server"; + version = k3sVersion; + + src = k3sRepo; + vendorHash = k3sVendorHash; + + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ libseccomp sqlite.dev ]; + + subPackages = [ "cmd/server" ]; + ldflags = versionldflags; + + tags = [ "libsqlite3" "linux" ]; + + # create the multicall symlinks for k3s + postInstall = '' + mv $out/bin/server $out/bin/k3s + pushd $out + # taken verbatim from https://github.com/k3s-io/k3s/blob/v1.24.10%2Bk3s1/scripts/build#L123-L131 + ln -s k3s ./bin/k3s-agent + ln -s k3s ./bin/k3s-server + ln -s k3s ./bin/k3s-etcd-snapshot + ln -s k3s ./bin/k3s-secrets-encrypt + ln -s k3s ./bin/k3s-certificate + ln -s k3s ./bin/k3s-completion + ln -s k3s ./bin/kubectl + ln -s k3s ./bin/crictl + ln -s k3s ./bin/ctr + popd + ''; + + meta = baseMeta // { + description = "The various binaries that get packaged into the final k3s binary"; + }; + }; + k3sContainerd = buildGoModule { + pname = "k3s-containerd"; + version = containerdVersion; + src = fetchFromGitHub { + owner = "k3s-io"; + repo = "containerd"; + rev = "v${containerdVersion}"; + sha256 = containerdSha256; + }; + vendorHash = null; + buildInputs = [ btrfs-progs ]; + subPackages = [ "cmd/containerd" "cmd/containerd-shim-runc-v2" ]; + ldflags = versionldflags; + }; +in +buildGoModule rec { + pname = "k3s"; + version = k3sVersion; + + src = k3sRepo; + vendorHash = k3sVendorHash; + + postPatch = '' + # Nix prefers dynamically linked binaries over static binary. + + substituteInPlace scripts/package-cli \ + --replace '"$LDFLAGS $STATIC" -o' \ + '"$LDFLAGS" -o' \ + --replace "STATIC=\"-extldflags \'-static\'\"" \ + "" + + # Upstream codegen fails with trimpath set. Removes "trimpath" for 'go generate': + + substituteInPlace scripts/package-cli \ + --replace '"''${GO}" generate' \ + 'GOFLAGS="" \ + GOOS="${pkgsBuildBuild.go.GOOS}" \ + GOARCH="${pkgsBuildBuild.go.GOARCH}" \ + CC="${pkgsBuildBuild.stdenv.cc}/bin/cc" \ + "''${GO}" generate' + ''; + + # Important utilities used by the kubelet, see + # https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494 + # Note the list in that issue is stale and some aren't relevant for k3s. + k3sRuntimeDeps = [ + kmod + socat + iptables + iproute2 + ipset + bridge-utils + ethtool + util-linux # kubelet wants 'nsenter' from util-linux: https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-705994388 + conntrack-tools + ]; + + buildInputs = k3sRuntimeDeps; + + nativeBuildInputs = [ + makeWrapper + rsync + yq-go + zstd + ]; + + # embedded in the final k3s cli + propagatedBuildInputs = [ + k3sCNIPlugins + k3sContainerd + k3sServer + runc + ]; + + # We override most of buildPhase due to peculiarities in k3s's build. + # Specifically, it has a 'go generate' which runs part of the package. See + # this comment: + # https://github.com/NixOS/nixpkgs/pull/158089#discussion_r799965694 + # So, why do we use buildGoModule at all? For the `vendorHash` / `go mod download` stuff primarily. + buildPhase = '' + patchShebangs ./scripts/package-cli ./scripts/download ./scripts/build-upload + + # copy needed 'go generate' inputs into place + mkdir -p ./bin/aux + rsync -a --no-perms ${k3sServer}/bin/ ./bin/ + ln -vsf ${runc}/bin/runc ./bin/runc + ln -vsf ${k3sCNIPlugins}/bin/cni ./bin/cni + ln -vsf ${k3sContainerd}/bin/* ./bin/ + rsync -a --no-perms --chmod u=rwX ${k3sRoot}/etc/ ./etc/ + mkdir -p ./build/static/charts + + cp ${traefikChart} ./build/static/charts + cp ${traefik-crdChart} ./build/static/charts + + export ARCH=$GOARCH + export DRONE_TAG="v${k3sVersion}" + export DRONE_COMMIT="${k3sCommit}" + # use ./scripts/package-cli to run 'go generate' + 'go build' + + ./scripts/package-cli + mkdir -p $out/bin + ''; + + # Otherwise it depends on 'getGoDirs', which is normally set in buildPhase + doCheck = false; + + installPhase = '' + # wildcard to match the arm64 build too + install -m 0755 dist/artifacts/k3s* -D $out/bin/k3s + wrapProgram $out/bin/k3s \ + --prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \ + --prefix PATH : "$out/bin" + ln -s $out/bin/k3s $out/bin/kubectl + ln -s $out/bin/k3s $out/bin/crictl + ln -s $out/bin/k3s $out/bin/ctr + ''; + + doInstallCheck = true; + installCheckPhase = '' + $out/bin/k3s --version | grep -F "v${k3sVersion}" >/dev/null + ''; + + # Fix-Me: Needs to be adapted specifically for 1.24 + # passthru.updateScript = ./update.sh; + + passthru.tests = k3s.passthru.mkTests k3sVersion; + + meta = baseMeta; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/1_25/0001-script-download-strip-downloading-just-package-CRD.patch b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_25/0001-script-download-strip-downloading-just-package-CRD.patch new file mode 100644 index 000000000000..115fd6824772 --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_25/0001-script-download-strip-downloading-just-package-CRD.patch @@ -0,0 +1,41 @@ +From 6f53bd36a40da4c71486e3b79f6e32d53d6eea5d Mon Sep 17 00:00:00 2001 +From: Euan Kemp <euank@euank.com> +Date: Thu, 3 Feb 2022 23:50:40 -0800 +Subject: [PATCH 2/2] scrips/download: strip downloading, just package CRD + +The CRD packaging is a complicated set of commands, so let's reuse it. +--- + scripts/download | 10 ++-------- + 1 file changed, 2 insertions(+), 8 deletions(-) + +diff --git a/scripts/download b/scripts/download +index 5effc0562a..82361803ee 100755 +--- a/scripts/download ++++ b/scripts/download +@@ -24,12 +24,6 @@ rm -rf ${CONTAINERD_DIR} + mkdir -p ${CHARTS_DIR} + mkdir -p ${DATA_DIR} + +-curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${VERSION_ROOT}/k3s-root-${ARCH}.tar | tar xf - --exclude=bin/socat +- +-git clone --single-branch --branch=${VERSION_RUNC} --depth=1 https://github.com/opencontainers/runc ${RUNC_DIR} +- +-git clone --single-branch --branch=${VERSION_CONTAINERD} --depth=1 https://github.com/k3s-io/containerd ${CONTAINERD_DIR} +- + setup_tmp() { + TMP_DIR=$(mktemp -d --tmpdir=${CHARTS_DIR}) + cleanup() { +@@ -44,8 +38,8 @@ setup_tmp() { + + download_and_package_traefik () { + echo "Downloading Traefik Helm chart from ${TRAEFIK_URL}" +- curl -sfL ${TRAEFIK_URL} -o ${TMP_DIR}/${TRAEFIK_FILE} +- code=$? ++ # nixpkgs: copy in our known traefik chart instead ++ cp $TRAEFIK_CHART_FILE ${TMP_DIR}/${TRAEFIK_FILE} + + if [ $code -ne 0 ]; then + echo "Error: Failed to download Traefik Helm chart!" +-- +2.34.1 + diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/1_25/default.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_25/default.nix new file mode 100644 index 000000000000..7ea3ff7867e6 --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_25/default.nix @@ -0,0 +1,336 @@ +{ stdenv +, lib +, makeWrapper +, socat +, iptables +, iproute2 +, ipset +, bridge-utils +, btrfs-progs +, conntrack-tools +, buildGoModule +, runc +, rsync +, kmod +, libseccomp +, pkg-config +, ethtool +, util-linux +, fetchFromGitHub +, fetchurl +, fetchzip +, fetchgit +, zstd +, yq-go +, sqlite +, nixosTests +, pkgsBuildBuild +, k3s +}: + +# k3s is a kinda weird derivation. One of the main points of k3s is the +# simplicity of it being one binary that can perform several tasks. +# However, when you have a good package manager (like nix), that doesn't +# actually make much of a difference; you don't really care if it's one binary +# or 10 since with a good package manager, installing and running it is +# identical. +# Since upstream k3s packages itself as one large binary with several +# "personalities" (in the form of subcommands like 'k3s agent' and 'k3s +# kubectl'), it ends up being easiest to mostly mimic upstream packaging, with +# some exceptions. +# K3s also carries patches to some packages (such as containerd and cni +# plugins), so we intentionally use the k3s versions of those binaries for k3s, +# even if the upstream version of those binaries exist in nixpkgs already. In +# the end, that means we have a thick k3s binary that behaves like the upstream +# one for the most part. +# However, k3s also bundles several pieces of unpatched software, from the +# strongswan vpn software, to iptables, to socat, conntrack, busybox, etc. +# Those pieces of software we entirely ignore upstream's handling of, and just +# make sure they're in the path if desired. +let + k3sVersion = "1.25.3+k3s1"; # k3s git tag + k3sCommit = "f2585c1671b31b4b34bddbb3bf4e7d69662b0821"; # k3s git commit at the above version + k3sRepoSha256 = "0zwf3iwjcidx14zw36s1hr0q8wmmbfc0rfqwd7fmpjq597h8zkms"; + k3sVendorHash = "sha256-U67tJRGqPFk5AfRe7I50zKGC9HJ2oh+iI/C7qF/76BQ="; + + # taken from ./manifests/traefik.yaml, extracted from '.spec.chart' https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/download#L9 + # The 'patch' and 'minor' versions are currently hardcoded as single digits only, so ignore the trailing two digits. Weird, I know. + traefikChartVersion = "12.0.0"; + traefikChartSha256 = "1sqmi71fi3ad5dh5fmsp9mv80x6pkgqwi4r9fr8l6i9sdnai6f1a"; + + # taken from ./scripts/version.sh VERSION_ROOT https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L47 + k3sRootVersion = "0.11.0"; + k3sRootSha256 = "016n56vi09xkvjph7wgzb2m86mhd5x65fs4d11pmh20hl249r620"; + + # taken from ./scripts/version.sh VERSION_CNIPLUGINS https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L45 + k3sCNIVersion = "1.1.1-k3s1"; + k3sCNISha256 = "14mb3zsqibj1sn338gjmsyksbm0mxv9p016dij7zidccx2rzn6nl"; + + # taken from go.mod, the 'github.com/containerd/containerd' line + # run `grep github.com/containerd/containerd go.mod | head -n1 | awk '{print $4}'` + containerdVersion = "1.5.13-k3s2"; + containerdSha256 = "1pfr2ji4aij9js90gf4a3hqnhyw5hshcjdccm62l700j68gs5z97"; + + # run `grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}'` in the k3s repo at the tag + criCtlVersion = "1.25.0-k3s1"; + + baseMeta = k3s.meta; + + # https://github.com/k3s-io/k3s/blob/5fb370e53e0014dc96183b8ecb2c25a61e891e76/scripts/build#L19-L40 + versionldflags = [ + "-X github.com/rancher/k3s/pkg/version.Version=v${k3sVersion}" + "-X github.com/rancher/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitVersion=v${k3sVersion}" + "-X k8s.io/client-go/pkg/version.gitCommit=${k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitTreeState=clean" + "-X k8s.io/client-go/pkg/version.buildDate=1970-01-01T01:01:01Z" + "-X k8s.io/component-base/version.gitVersion=v${k3sVersion}" + "-X k8s.io/component-base/version.gitCommit=${k3sCommit}" + "-X k8s.io/component-base/version.gitTreeState=clean" + "-X k8s.io/component-base/version.buildDate=1970-01-01T01:01:01Z" + "-X github.com/kubernetes-sigs/cri-tools/pkg/version.Version=v${criCtlVersion}" + "-X github.com/containerd/containerd/version.Version=v${containerdVersion}" + "-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd" + ]; + + # bundled into the k3s binary + traefikChart = fetchurl { + url = "https://helm.traefik.io/traefik/traefik-${traefikChartVersion}.tgz"; + sha256 = traefikChartSha256; + }; + # so, k3s is a complicated thing to package + # This derivation attempts to avoid including any random binaries from the + # internet. k3s-root is _mostly_ binaries built to be bundled in k3s (which + # we don't care about doing, we can add those as build or runtime + # dependencies using a real package manager). + # In addition to those binaries, it's also configuration though (right now + # mostly strongswan configuration), and k3s does use those files. + # As such, we download it in order to grab 'etc' and bundle it into the final + # k3s binary. + k3sRoot = fetchzip { + # Note: marked as apache 2.0 license + url = "https://github.com/k3s-io/k3s-root/releases/download/v${k3sRootVersion}/k3s-root-amd64.tar"; + sha256 = k3sRootSha256; + stripRoot = false; + }; + k3sCNIPlugins = buildGoModule rec { + pname = "k3s-cni-plugins"; + version = k3sCNIVersion; + vendorHash = null; + + subPackages = [ "." ]; + + src = fetchFromGitHub { + owner = "rancher"; + repo = "plugins"; + rev = "v${version}"; + sha256 = k3sCNISha256; + }; + + postInstall = '' + mv $out/bin/plugins $out/bin/cni + ''; + + meta = baseMeta // { + description = "CNI plugins, as patched by rancher for k3s"; + }; + }; + # Grab this separately from a build because it's used by both stages of the + # k3s build. + k3sRepo = fetchgit { + url = "https://github.com/k3s-io/k3s"; + rev = "v${k3sVersion}"; + sha256 = k3sRepoSha256; + }; + # Stage 1 of the k3s build: + # Let's talk about how k3s is structured. + # One of the ideas of k3s is that there's the single "k3s" binary which can + # do everything you need, from running a k3s server, to being a worker node, + # to running kubectl. + # The way that actually works is that k3s is a single go binary that contains + # a bunch of bindata that it unpacks at runtime into directories (either the + # user's home directory or /var/lib/rancher if run as root). + # This bindata includes both binaries and configuration. + # In order to let nixpkgs do all its autostripping/patching/etc, we split this into two derivations. + # First, we build all the binaries that get packed into the thick k3s binary + # (and output them from one derivation so they'll all be suitably patched up). + # Then, we bundle those binaries into our thick k3s binary and use that as + # the final single output. + # This approach was chosen because it ensures the bundled binaries all are + # correctly built to run with nix (we can lean on the existing buildGoModule + # stuff), and we can again lean on that tooling for the final k3s binary too. + # Other alternatives would be to manually run the + # strip/patchelf/remove-references step ourselves in the installPhase of the + # derivation when we've built all the binaries, but haven't bundled them in + # with generated bindata yet. + + k3sServer = buildGoModule rec { + pname = "k3s-server"; + version = k3sVersion; + + src = k3sRepo; + vendorHash = k3sVendorHash; + + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ libseccomp sqlite.dev ]; + + subPackages = [ "cmd/server" ]; + ldflags = versionldflags; + + tags = [ "libsqlite3" "linux" ]; + + # create the multicall symlinks for k3s + postInstall = '' + mv $out/bin/server $out/bin/k3s + pushd $out + # taken verbatim from https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/build#L105-L113 + ln -s k3s ./bin/k3s-agent + ln -s k3s ./bin/k3s-server + ln -s k3s ./bin/k3s-etcd-snapshot + ln -s k3s ./bin/k3s-secrets-encrypt + ln -s k3s ./bin/k3s-certificate + ln -s k3s ./bin/kubectl + ln -s k3s ./bin/crictl + ln -s k3s ./bin/ctr + popd + ''; + + meta = baseMeta // { + description = "The various binaries that get packaged into the final k3s binary"; + }; + }; + k3sContainerd = buildGoModule { + pname = "k3s-containerd"; + version = containerdVersion; + src = fetchFromGitHub { + owner = "k3s-io"; + repo = "containerd"; + rev = "v${containerdVersion}"; + sha256 = containerdSha256; + }; + vendorHash = null; + buildInputs = [ btrfs-progs ]; + subPackages = [ "cmd/containerd" "cmd/containerd-shim-runc-v2" ]; + ldflags = versionldflags; + }; +in +buildGoModule rec { + pname = "k3s"; + version = k3sVersion; + + src = k3sRepo; + vendorHash = k3sVendorHash; + + patches = [ + ./0001-script-download-strip-downloading-just-package-CRD.patch + ]; + + postPatch = '' + # Nix prefers dynamically linked binaries over static binary. + + substituteInPlace scripts/package-cli \ + --replace '"$LDFLAGS $STATIC" -o' \ + '"$LDFLAGS" -o' \ + --replace "STATIC=\"-extldflags \'-static\'\"" \ + "" + + # Upstream codegen fails with trimpath set. Removes "trimpath" for 'go generate': + + substituteInPlace scripts/package-cli \ + --replace '"''${GO}" generate' \ + 'GOFLAGS="" \ + GOOS="${pkgsBuildBuild.go.GOOS}" \ + GOARCH="${pkgsBuildBuild.go.GOARCH}" \ + CC="${pkgsBuildBuild.stdenv.cc}/bin/cc" \ + "''${GO}" generate' + ''; + + # Important utilities used by the kubelet, see + # https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494 + # Note the list in that issue is stale and some aren't relevant for k3s. + k3sRuntimeDeps = [ + kmod + socat + iptables + iproute2 + ipset + bridge-utils + ethtool + util-linux # kubelet wants 'nsenter' from util-linux: https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-705994388 + conntrack-tools + ]; + + buildInputs = k3sRuntimeDeps; + + nativeBuildInputs = [ + makeWrapper + rsync + yq-go + zstd + ]; + + # embedded in the final k3s cli + propagatedBuildInputs = [ + k3sCNIPlugins + k3sContainerd + k3sServer + runc + ]; + + # We override most of buildPhase due to peculiarities in k3s's build. + # Specifically, it has a 'go generate' which runs part of the package. See + # this comment: + # https://github.com/NixOS/nixpkgs/pull/158089#discussion_r799965694 + # So, why do we use buildGoModule at all? For the `vendorHash` / `go mod download` stuff primarily. + buildPhase = '' + patchShebangs ./scripts/package-cli ./scripts/download ./scripts/build-upload + + # copy needed 'go generate' inputs into place + mkdir -p ./bin/aux + rsync -a --no-perms ${k3sServer}/bin/ ./bin/ + ln -vsf ${runc}/bin/runc ./bin/runc + ln -vsf ${k3sCNIPlugins}/bin/cni ./bin/cni + ln -vsf ${k3sContainerd}/bin/* ./bin/ + rsync -a --no-perms --chmod u=rwX ${k3sRoot}/etc/ ./etc/ + mkdir -p ./build/static/charts + # Note, upstream's chart has a 00 suffix. This seems to not matter though, so we're ignoring that naming detail. + export TRAEFIK_CHART_FILE=${traefikChart} + # place the traefik chart using their code since it's complicated + # We trim the actual download, see patches + ./scripts/download + + export ARCH=$GOARCH + export DRONE_TAG="v${k3sVersion}" + export DRONE_COMMIT="${k3sCommit}" + # use ./scripts/package-cli to run 'go generate' + 'go build' + + ./scripts/package-cli + mkdir -p $out/bin + ''; + + # Otherwise it depends on 'getGoDirs', which is normally set in buildPhase + doCheck = false; + + installPhase = '' + # wildcard to match the arm64 build too + install -m 0755 dist/artifacts/k3s* -D $out/bin/k3s + wrapProgram $out/bin/k3s \ + --prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \ + --prefix PATH : "$out/bin" + ln -s $out/bin/k3s $out/bin/kubectl + ln -s $out/bin/k3s $out/bin/crictl + ln -s $out/bin/k3s $out/bin/ctr + ''; + + doInstallCheck = true; + installCheckPhase = '' + $out/bin/k3s --version | grep -F "v${k3sVersion}" >/dev/null + ''; + + # Fix-Me: Needs to be adapted specifically for 1.25 + # passthru.updateScript = ./update.sh; + + passthru.tests = k3s.passthru.mkTests k3sVersion; + + meta = baseMeta; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/1_26/chart-versions.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_26/chart-versions.nix new file mode 100644 index 000000000000..d3ff99358af2 --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_26/chart-versions.nix @@ -0,0 +1,10 @@ +{ + traefik-crd = { + url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-21.2.1+up21.2.0.tgz"; + sha256 = "05j3vyikb7g2z2i07rij9h4ki5lb2hb2rynpiqfd4l1y5qm0qhw9"; + }; + traefik = { + url = "https://k3s.io/k3s-charts/assets/traefik/traefik-21.2.1+up21.2.0.tgz"; + sha256 = "0gvz0yzph2893scd0q10b938yc7f36b3zqs57pkjgqqpl1d0nwhg"; + }; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/1_26/versions.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_26/versions.nix new file mode 100644 index 000000000000..5bbfbcf380a5 --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_26/versions.nix @@ -0,0 +1,14 @@ +{ + k3sVersion = "1.26.9+k3s1"; + k3sCommit = "4e217286a7ea41b82f1b67ab851d444ecf9a0f9b"; + k3sRepoSha256 = "1rf2gzf3ilcd1gc6d4k1w6cficr70x8lwzcq81njpz72dr6883z3"; + k3sVendorHash = "sha256-heCQNRaa0qFNkL69KEiIH2qEg+pukgS+fLOSWcwFddA="; + chartVersions = import ./chart-versions.nix; + k3sRootVersion = "0.12.2"; + k3sRootSha256 = "1gjynvr350qni5mskgm7pcc7alss4gms4jmkiv453vs8mmma9c9k"; + k3sCNIVersion = "1.3.0-k3s1"; + k3sCNISha256 = "0zma9g4wvdnhs9igs03xlx15bk2nq56j73zns9xgqmfiixd9c9av"; + containerdVersion = "1.7.6-k3s1.26"; + containerdSha256 = "1bj7nggfmkrrgm5yk08p665z1mw1y376k4g3vjbkqldfglzpx7sq"; + criCtlVersion = "1.26.0-rc.0-k3s1"; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/1_27/chart-versions.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_27/chart-versions.nix new file mode 100644 index 000000000000..1acca4d0e101 --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_27/chart-versions.nix @@ -0,0 +1,10 @@ +{ + traefik-crd = { + url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-25.0.2+up25.0.0.tgz"; + sha256 = "0jygzsn5pxzf7423x5iqfffgx5xvm7c7hfck46y7vpv1fdkiipcq"; + }; + traefik = { + url = "https://k3s.io/k3s-charts/assets/traefik/traefik-25.0.2+up25.0.0.tgz"; + sha256 = "1g9n19lnqdkmbbr3rnbwc854awha0kqqfwyxanyx1lg5ww8ldp89"; + }; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/1_27/versions.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_27/versions.nix new file mode 100644 index 000000000000..596f21ab55cd --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_27/versions.nix @@ -0,0 +1,14 @@ +{ + k3sVersion = "1.27.9+k3s1"; + k3sCommit = "2c249a39358bd36438ab53aedef5487d950fd558"; + k3sRepoSha256 = "16zcp1ih34zpz6115ivbcs49n5yikgj8mpiv177jvvb2vakmkgv6"; + k3sVendorHash = "sha256-zvoBN1mErSXovv/xVzjntHyZjVyCfPzsOdlcTSIwKus="; + chartVersions = import ./chart-versions.nix; + k3sRootVersion = "0.12.2"; + k3sRootSha256 = "1gjynvr350qni5mskgm7pcc7alss4gms4jmkiv453vs8mmma9c9k"; + k3sCNIVersion = "1.3.0-k3s1"; + k3sCNISha256 = "0zma9g4wvdnhs9igs03xlx15bk2nq56j73zns9xgqmfiixd9c9av"; + containerdVersion = "1.7.11-k3s2.27"; + containerdSha256 = "0xjxc5dgh3drk2glvcabd885damjffp9r4cs0cm1zgnrrbhlipra"; + criCtlVersion = "1.26.0-rc.0-k3s1"; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/1_28/chart-versions.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_28/chart-versions.nix new file mode 100644 index 000000000000..1acca4d0e101 --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_28/chart-versions.nix @@ -0,0 +1,10 @@ +{ + traefik-crd = { + url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-25.0.2+up25.0.0.tgz"; + sha256 = "0jygzsn5pxzf7423x5iqfffgx5xvm7c7hfck46y7vpv1fdkiipcq"; + }; + traefik = { + url = "https://k3s.io/k3s-charts/assets/traefik/traefik-25.0.2+up25.0.0.tgz"; + sha256 = "1g9n19lnqdkmbbr3rnbwc854awha0kqqfwyxanyx1lg5ww8ldp89"; + }; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/1_28/versions.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_28/versions.nix new file mode 100644 index 000000000000..f1d3ea13177a --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_28/versions.nix @@ -0,0 +1,14 @@ +{ + k3sVersion = "1.28.6+k3s1"; + k3sCommit = "39a0001575780fffa6aae0271f4cb4ce7413aac8"; + k3sRepoSha256 = "1bhbpbgs02gh5y7pgn6vmanacrz3p0b2gq3w2kqpb11bijp2alld"; + k3sVendorHash = "sha256-Mo+gZ+NOZqd3CP/Z02LfO4dHyEuRhabZVAU60GofOMo="; + chartVersions = import ./chart-versions.nix; + k3sRootVersion = "0.12.2"; + k3sRootSha256 = "1gjynvr350qni5mskgm7pcc7alss4gms4jmkiv453vs8mmma9c9k"; + k3sCNIVersion = "1.3.0-k3s1"; + k3sCNISha256 = "0zma9g4wvdnhs9igs03xlx15bk2nq56j73zns9xgqmfiixd9c9av"; + containerdVersion = "1.7.11-k3s2"; + containerdSha256 = "0279sil02wz7310xhrgmdbc0r2qibj9lafy0i9k24jdrh74icmib"; + criCtlVersion = "1.26.0-rc.0-k3s1"; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/1_29/chart-versions.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_29/chart-versions.nix new file mode 100644 index 000000000000..1acca4d0e101 --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_29/chart-versions.nix @@ -0,0 +1,10 @@ +{ + traefik-crd = { + url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-25.0.2+up25.0.0.tgz"; + sha256 = "0jygzsn5pxzf7423x5iqfffgx5xvm7c7hfck46y7vpv1fdkiipcq"; + }; + traefik = { + url = "https://k3s.io/k3s-charts/assets/traefik/traefik-25.0.2+up25.0.0.tgz"; + sha256 = "1g9n19lnqdkmbbr3rnbwc854awha0kqqfwyxanyx1lg5ww8ldp89"; + }; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/1_29/versions.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_29/versions.nix new file mode 100644 index 000000000000..00bc1476306d --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/1_29/versions.nix @@ -0,0 +1,14 @@ +{ + k3sVersion = "1.29.0+k3s1"; + k3sCommit = "3190a5faa28d7a0d428c756d67adcab7eb11e6a5"; + k3sRepoSha256 = "1g75a7kz9nnv0vagzhggkw0zqigykimdwsmibgssa8vyjpg7idda"; + k3sVendorHash = "sha256-iHmPVjYR/ZLH9UZ5yNEApyuGQsEwtxVbQw7Pu7WrpaQ="; + chartVersions = import ./chart-versions.nix; + k3sRootVersion = "0.12.2"; + k3sRootSha256 = "1gjynvr350qni5mskgm7pcc7alss4gms4jmkiv453vs8mmma9c9k"; + k3sCNIVersion = "1.3.0-k3s1"; + k3sCNISha256 = "0zma9g4wvdnhs9igs03xlx15bk2nq56j73zns9xgqmfiixd9c9av"; + containerdVersion = "1.7.11-k3s2"; + containerdSha256 = "0279sil02wz7310xhrgmdbc0r2qibj9lafy0i9k24jdrh74icmib"; + criCtlVersion = "1.29.0-k3s1"; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/README.md b/nixpkgs/pkgs/applications/networking/cluster/k3s/README.md new file mode 100644 index 000000000000..df2bead6be53 --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/README.md @@ -0,0 +1,73 @@ +# k3s versions + +K3s, Kubernetes, and other clustered software has the property of not being able to update atomically. Most software in nixpkgs, like for example bash, can be updated as part of a "nixos-rebuild switch" without having to worry about the old and the new bash interacting in some way. + +K3s/Kubernetes, on the other hand, is typically run across several NixOS machines, and each NixOS machine is updated independently. As such, different versions of the package and NixOS module must maintain compatibility with each other through temporary version skew during updates. + +The upstream Kubernetes project [documents this in their version-skew policy](https://kubernetes.io/releases/version-skew-policy/#supported-component-upgrade-order). + +Within nixpkgs, we strive to maintain a valid "upgrade path" that does not run +afoul of the upstream version skew policy. + +## Upstream release cadence and support + +K3s is built on top of K8s, and typically provides a similar release cadence and support window (simply by cherry-picking over k8s patches). As such, we assume k3s's support lifecycle is identical to upstream K8s. + +This is documented upstream [here](https://kubernetes.io/releases/patch-releases/#support-period). + +In short, a new Kubernetes version is released roughly every 4 months, and each release is supported for a little over 1 year. + +Any version that is not supported by upstream should be dropped from nixpkgs. + +## Versions in NixOS releases + +NixOS releases should avoid having deprecated software, or making major version upgrades, wherever possible. + +As such, we would like to have only the newest K3s version in each NixOS +release at the time the release branch is branched off, which will ensure the +K3s version in that release will receive updates for the longest duration +possible. + +However, this conflicts with another desire: we would like people to be able to upgrade between NixOS stable releases without needing to make a large enough k3s version jump that they violate the Kubernetes version skew policy. + +To give an example, we may have the following timeline for k8s releases: + +(Note, the exact versions and dates may be wrong, this is an illustrative example, reality may differ). + +```mermaid +gitGraph + branch k8s + commit + branch "k8s-1.24" + checkout "k8s-1.24" + commit id: "1.24.0" tag: "2022-05-03" + branch "k8s-1.25" + checkout "k8s-1.25" + commit id: "1.25.0" tag: "2022-08-23" + branch "k8s-1.26" + checkout "k8s-1.26" + commit id: "1.26.0" tag: "2022-12-08" + checkout k8s-1.24 + commit id: "1.24-EOL" tag: "2023-07-28" + checkout k8s-1.25 + commit id: "1.25-EOL" tag: "2023-10-27" + checkout k8s-1.26 + commit id: "1.26-EOL" tag: "2024-02-28" +``` + +(Note: the above graph will render if you view this markdown on GitHub, or when using [mermaid](https://mermaid.js.org/)) + +In this scenario even though k3s 1.24 is still technically supported when the NixOS 23.05 +release is cut, since it goes EOL before the NixOS 23.11 release is made, we would +not want to include it. Similarly, k3s 1.25 would go EOL before NixOS 23.11. + +As such, we should only include k3s 1.26 in the 23.05 release. + +We can then make a similar argument when NixOS 23.11 comes around to not +include k3s 1.26 or 1.27. However, that means someone upgrading from the NixOS +22.05 release to the NixOS 23.11 would not have a supported upgrade path. + +In order to resolve this issue, we propose backporting not just new patch releases to older NixOS releases, but also new k3s versions, up to one version before the first version that is included in the next NixOS release. + +In the above example, where NixOS 23.05 included k3s 1.26, and 23.11 included k3s 1.28, that means we would backport 1.27 to the NixOS 23.05 release, and backport all patches for 1.26 and 1.27. +This would allow someone to upgrade between those NixOS releases in a supported configuration. diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/builder.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/builder.nix new file mode 100644 index 000000000000..a914cf87102e --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/builder.nix @@ -0,0 +1,354 @@ +lib: +{ + # git tag + k3sVersion, + # commit hash + k3sCommit, + k3sRepoSha256 ? lib.fakeHash, + k3sVendorHash ? lib.fakeHash, + # taken from ./scripts/version.sh VERSION_ROOT https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L47 + k3sRootVersion, + k3sRootSha256 ? lib.fakeHash, + # Based on the traefik charts here: https://github.com/k3s-io/k3s/blob/d71ab6317e22dd34673faa307a412a37a16767f6/scripts/download#L29-L32 + # see also https://github.com/k3s-io/k3s/blob/d71ab6317e22dd34673faa307a412a37a16767f6/manifests/traefik.yaml#L8 + chartVersions, + # taken from ./scripts/version.sh VERSION_CNIPLUGINS https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L45 + k3sCNIVersion, + k3sCNISha256 ? lib.fakeHash, + # taken from ./scripts/version.sh VERSION_CONTAINERD + containerdVersion, + containerdSha256 ? lib.fakeHash, + # run `grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}'` in the k3s repo at the tag + criCtlVersion, + updateScript ? null, +}: + +# builder.nix contains a "builder" expression that, given k3s version and hash +# variables, creates a package for that version. +# Due to variance in k3s's build process, this builder only works for k3s 1.26+ +# currently. +# It is likely we will have to split out additional builders for additional +# versions in the future, or customize this one further. +{ lib +, makeWrapper +, socat +, iptables +, iproute2 +, ipset +, bridge-utils +, btrfs-progs +, conntrack-tools +, buildGoModule +, runc +, rsync +, kmod +, libseccomp +, pkg-config +, ethtool +, util-linux +, fetchFromGitHub +, fetchurl +, fetchzip +, fetchgit +, zstd +, yq-go +, sqlite +, nixosTests +, pkgsBuildBuild +}: + +# k3s is a kinda weird derivation. One of the main points of k3s is the +# simplicity of it being one binary that can perform several tasks. +# However, when you have a good package manager (like nix), that doesn't +# actually make much of a difference; you don't really care if it's one binary +# or 10 since with a good package manager, installing and running it is +# identical. +# Since upstream k3s packages itself as one large binary with several +# "personalities" (in the form of subcommands like 'k3s agent' and 'k3s +# kubectl'), it ends up being easiest to mostly mimic upstream packaging, with +# some exceptions. +# K3s also carries patches to some packages (such as containerd and cni +# plugins), so we intentionally use the k3s versions of those binaries for k3s, +# even if the upstream version of those binaries exist in nixpkgs already. In +# the end, that means we have a thick k3s binary that behaves like the upstream +# one for the most part. +# However, k3s also bundles several pieces of unpatched software, from the +# strongswan vpn software, to iptables, to socat, conntrack, busybox, etc. +# Those pieces of software we entirely ignore upstream's handling of, and just +# make sure they're in the path if desired. +let + + baseMeta = with lib; { + description = "A lightweight Kubernetes distribution"; + license = licenses.asl20; + homepage = "https://k3s.io"; + maintainers = with maintainers; [ euank mic92 yajo ]; + platforms = platforms.linux; + + # resolves collisions with other installations of kubectl, crictl, ctr + # prefer non-k3s versions + priority = 5; + }; + + # https://github.com/k3s-io/k3s/blob/5fb370e53e0014dc96183b8ecb2c25a61e891e76/scripts/build#L19-L40 + versionldflags = [ + "-X github.com/rancher/k3s/pkg/version.Version=v${k3sVersion}" + "-X github.com/rancher/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitVersion=v${k3sVersion}" + "-X k8s.io/client-go/pkg/version.gitCommit=${k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitTreeState=clean" + "-X k8s.io/client-go/pkg/version.buildDate=1970-01-01T01:01:01Z" + "-X k8s.io/component-base/version.gitVersion=v${k3sVersion}" + "-X k8s.io/component-base/version.gitCommit=${k3sCommit}" + "-X k8s.io/component-base/version.gitTreeState=clean" + "-X k8s.io/component-base/version.buildDate=1970-01-01T01:01:01Z" + "-X github.com/kubernetes-sigs/cri-tools/pkg/version.Version=v${criCtlVersion}" + "-X github.com/containerd/containerd/version.Version=v${containerdVersion}" + "-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd" + ]; + + # bundled into the k3s binary + traefikChart = fetchurl chartVersions.traefik; + traefik-crdChart = fetchurl chartVersions.traefik-crd; + + # so, k3s is a complicated thing to package + # This derivation attempts to avoid including any random binaries from the + # internet. k3s-root is _mostly_ binaries built to be bundled in k3s (which + # we don't care about doing, we can add those as build or runtime + # dependencies using a real package manager). + # In addition to those binaries, it's also configuration though (right now + # mostly strongswan configuration), and k3s does use those files. + # As such, we download it in order to grab 'etc' and bundle it into the final + # k3s binary. + k3sRoot = fetchzip { + # Note: marked as apache 2.0 license + url = "https://github.com/k3s-io/k3s-root/releases/download/v${k3sRootVersion}/k3s-root-amd64.tar"; + sha256 = k3sRootSha256; + stripRoot = false; + }; + k3sCNIPlugins = buildGoModule rec { + pname = "k3s-cni-plugins"; + version = k3sCNIVersion; + vendorHash = null; + + subPackages = [ "." ]; + + src = fetchFromGitHub { + owner = "rancher"; + repo = "plugins"; + rev = "v${version}"; + sha256 = k3sCNISha256; + }; + + postInstall = '' + mv $out/bin/plugins $out/bin/cni + ''; + + meta = baseMeta // { + description = "CNI plugins, as patched by rancher for k3s"; + }; + }; + # Grab this separately from a build because it's used by both stages of the + # k3s build. + k3sRepo = fetchgit { + url = "https://github.com/k3s-io/k3s"; + rev = "v${k3sVersion}"; + sha256 = k3sRepoSha256; + }; + # Stage 1 of the k3s build: + # Let's talk about how k3s is structured. + # One of the ideas of k3s is that there's the single "k3s" binary which can + # do everything you need, from running a k3s server, to being a worker node, + # to running kubectl. + # The way that actually works is that k3s is a single go binary that contains + # a bunch of bindata that it unpacks at runtime into directories (either the + # user's home directory or /var/lib/rancher if run as root). + # This bindata includes both binaries and configuration. + # In order to let nixpkgs do all its autostripping/patching/etc, we split this into two derivations. + # First, we build all the binaries that get packed into the thick k3s binary + # (and output them from one derivation so they'll all be suitably patched up). + # Then, we bundle those binaries into our thick k3s binary and use that as + # the final single output. + # This approach was chosen because it ensures the bundled binaries all are + # correctly built to run with nix (we can lean on the existing buildGoModule + # stuff), and we can again lean on that tooling for the final k3s binary too. + # Other alternatives would be to manually run the + # strip/patchelf/remove-references step ourselves in the installPhase of the + # derivation when we've built all the binaries, but haven't bundled them in + # with generated bindata yet. + + k3sServer = buildGoModule { + pname = "k3s-server"; + version = k3sVersion; + + src = k3sRepo; + vendorHash = k3sVendorHash; + + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ libseccomp sqlite.dev ]; + + subPackages = [ "cmd/server" ]; + ldflags = versionldflags; + + tags = [ "ctrd" "libsqlite3" "linux" ]; + + # create the multicall symlinks for k3s + postInstall = '' + mv $out/bin/server $out/bin/k3s + pushd $out + # taken verbatim from https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/build#L105-L113 + ln -s k3s ./bin/containerd + ln -s k3s ./bin/crictl + ln -s k3s ./bin/ctr + ln -s k3s ./bin/k3s-agent + ln -s k3s ./bin/k3s-certificate + ln -s k3s ./bin/k3s-completion + ln -s k3s ./bin/k3s-etcd-snapshot + ln -s k3s ./bin/k3s-secrets-encrypt + ln -s k3s ./bin/k3s-server + ln -s k3s ./bin/k3s-token + ln -s k3s ./bin/kubectl + popd + ''; + + meta = baseMeta // { + description = "The various binaries that get packaged into the final k3s binary"; + }; + }; + # Only used for the shim since + # https://github.com/k3s-io/k3s/blob/v1.27.2%2Bk3s1/scripts/build#L153 + k3sContainerd = buildGoModule { + pname = "k3s-containerd"; + version = containerdVersion; + src = fetchFromGitHub { + owner = "k3s-io"; + repo = "containerd"; + rev = "v${containerdVersion}"; + sha256 = containerdSha256; + }; + vendorHash = null; + buildInputs = [ btrfs-progs ]; + subPackages = [ "cmd/containerd-shim-runc-v2" ]; + ldflags = versionldflags; + }; +in +buildGoModule rec { + pname = "k3s"; + version = k3sVersion; + + tags = [ "libsqlite3" "linux" "ctrd" ]; + src = k3sRepo; + vendorHash = k3sVendorHash; + + postPatch = '' + # Nix prefers dynamically linked binaries over static binary. + + substituteInPlace scripts/package-cli \ + --replace '"$LDFLAGS $STATIC" -o' \ + '"$LDFLAGS" -o' \ + --replace "STATIC=\"-extldflags \'-static\'\"" \ + "" + + # Upstream codegen fails with trimpath set. Removes "trimpath" for 'go generate': + + substituteInPlace scripts/package-cli \ + --replace '"''${GO}" generate' \ + 'GOFLAGS="" \ + GOOS="${pkgsBuildBuild.go.GOOS}" \ + GOARCH="${pkgsBuildBuild.go.GOARCH}" \ + CC="${pkgsBuildBuild.stdenv.cc}/bin/cc" \ + "''${GO}" generate' + ''; + + # Important utilities used by the kubelet, see + # https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494 + # Note the list in that issue is stale and some aren't relevant for k3s. + k3sRuntimeDeps = [ + kmod + socat + iptables + iproute2 + ipset + bridge-utils + ethtool + util-linux # kubelet wants 'nsenter' from util-linux: https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-705994388 + conntrack-tools + runc + ]; + + buildInputs = k3sRuntimeDeps; + + nativeBuildInputs = [ + makeWrapper + rsync + yq-go + zstd + ]; + + # embedded in the final k3s cli + propagatedBuildInputs = [ + k3sCNIPlugins + k3sContainerd + k3sServer + ]; + + # We override most of buildPhase due to peculiarities in k3s's build. + # Specifically, it has a 'go generate' which runs part of the package. See + # this comment: + # https://github.com/NixOS/nixpkgs/pull/158089#discussion_r799965694 + # So, why do we use buildGoModule at all? For the `vendorHash` / `go mod download` stuff primarily. + buildPhase = '' + patchShebangs ./scripts/package-cli ./scripts/download ./scripts/build-upload + + # copy needed 'go generate' inputs into place + mkdir -p ./bin/aux + rsync -a --no-perms ${k3sServer}/bin/ ./bin/ + ln -vsf ${k3sCNIPlugins}/bin/cni ./bin/cni + ln -vsf ${k3sContainerd}/bin/containerd-shim-runc-v2 ./bin + rsync -a --no-perms --chmod u=rwX ${k3sRoot}/etc/ ./etc/ + mkdir -p ./build/static/charts + + cp ${traefikChart} ./build/static/charts + cp ${traefik-crdChart} ./build/static/charts + + export ARCH=$GOARCH + export DRONE_TAG="v${k3sVersion}" + export DRONE_COMMIT="${k3sCommit}" + # use ./scripts/package-cli to run 'go generate' + 'go build' + + ./scripts/package-cli + mkdir -p $out/bin + ''; + + # Otherwise it depends on 'getGoDirs', which is normally set in buildPhase + doCheck = false; + + installPhase = '' + # wildcard to match the arm64 build too + install -m 0755 dist/artifacts/k3s* -D $out/bin/k3s + wrapProgram $out/bin/k3s \ + --prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \ + --prefix PATH : "$out/bin" + ln -s $out/bin/k3s $out/bin/kubectl + ln -s $out/bin/k3s $out/bin/crictl + ln -s $out/bin/k3s $out/bin/ctr + ''; + + doInstallCheck = true; + installCheckPhase = '' + $out/bin/k3s --version | grep -F "v${k3sVersion}" >/dev/null + ''; + + passthru.updateScript = updateScript; + + passthru.mkTests = version: + let k3s_version = "k3s_" + lib.replaceStrings ["."] ["_"] (lib.versions.majorMinor version); + in { + single-node = nixosTests.k3s.single-node.${k3s_version}; + multi-node = nixosTests.k3s.multi-node.${k3s_version}; + }; + passthru.tests = passthru.mkTests k3sVersion; + + + meta = baseMeta; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/default.nix b/nixpkgs/pkgs/applications/networking/cluster/k3s/default.nix new file mode 100644 index 000000000000..934f5a3691cd --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/default.nix @@ -0,0 +1,33 @@ +{ lib, callPackage, ... }@args: + +let + k3s_builder = import ./builder.nix lib; + common = opts: callPackage (k3s_builder opts); + # extraArgs is the extra arguments passed in by the caller to propogate downward. + # This is to allow all-packages.nix to do: + # + # let k3s_1_23 = (callPackage ./path/to/k3s { + # commonK3sArg = .... + # }).k3s_1_23; + extraArgs = builtins.removeAttrs args [ "callPackage" ]; +in +{ + k3s_1_26 = common ((import ./1_26/versions.nix) // { + updateScript = [ ./update-script.sh "26" ]; + }) extraArgs; + + # 1_27 can be built with the same builder as 1_26 + k3s_1_27 = common ((import ./1_27/versions.nix) // { + updateScript = [ ./update-script.sh "27" ]; + }) extraArgs; + + # 1_28 can be built with the same builder as 1_26 + k3s_1_28 = common ((import ./1_28/versions.nix) // { + updateScript = [ ./update-script.sh "28" ]; + }) extraArgs; + + # 1_29 can be built with the same builder as 1_26 + k3s_1_29 = common ((import ./1_29/versions.nix) // { + updateScript = [ ./update-script.sh "29" ]; + }) extraArgs; +} diff --git a/nixpkgs/pkgs/applications/networking/cluster/k3s/update-script.sh b/nixpkgs/pkgs/applications/networking/cluster/k3s/update-script.sh new file mode 100755 index 000000000000..dc41d7325b6c --- /dev/null +++ b/nixpkgs/pkgs/applications/networking/cluster/k3s/update-script.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p curl gnugrep gnused jq yq-go nix-prefetch + +set -x -eu -o pipefail + +MINOR_VERSION="${1:?Must provide a minor version number, like '26', as the only argument}" + +WORKDIR=$(mktemp -d) +trap "rm -rf ${WORKDIR}" EXIT + +NIXPKGS_ROOT="$(git rev-parse --show-toplevel)"/ +NIXPKGS_K3S_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd -P)/ +cd ${NIXPKGS_K3S_PATH} + +cd 1_${MINOR_VERSION} + + +LATEST_TAG_RAWFILE=${WORKDIR}/latest_tag.json +curl --silent -f ${GITHUB_TOKEN:+-u ":$GITHUB_TOKEN"} \ + https://api.github.com/repos/k3s-io/k3s/releases > ${LATEST_TAG_RAWFILE} + +LATEST_TAG_NAME=$(jq 'map(.tag_name)' ${LATEST_TAG_RAWFILE} | \ + grep -v -e rc -e engine | tail -n +2 | head -n -1 | sed 's|[", ]||g' | sort -rV | grep -E "^v1\.${MINOR_VERSION}\." | head -n1) + +K3S_VERSION=$(echo ${LATEST_TAG_NAME} | sed 's/^v//') + +K3S_COMMIT=$(curl --silent -f ${GITHUB_TOKEN:+-u ":$GITHUB_TOKEN"} \ + https://api.github.com/repos/k3s-io/k3s/tags \ + | jq -r "map(select(.name == \"${LATEST_TAG_NAME}\")) | .[0] | .commit.sha") + +K3S_REPO_SHA256=$(nix-prefetch-url --quiet --unpack https://github.com/k3s-io/k3s/archive/refs/tags/${LATEST_TAG_NAME}.tar.gz) + +FILE_SCRIPTS_DOWNLOAD=${WORKDIR}/scripts-download +curl --silent -f https://raw.githubusercontent.com/k3s-io/k3s/${K3S_COMMIT}/scripts/download > $FILE_SCRIPTS_DOWNLOAD + +FILE_SCRIPTS_VERSION=${WORKDIR}/scripts-version.sh +curl --silent -f https://raw.githubusercontent.com/k3s-io/k3s/${K3S_COMMIT}/scripts/version.sh > $FILE_SCRIPTS_VERSION + +FILE_TRAEFIK_MANIFEST=${WORKDIR}/traefik.yml +curl --silent -f -o "$FILE_TRAEFIK_MANIFEST" https://raw.githubusercontent.com/k3s-io/k3s/${K3S_COMMIT}/manifests/traefik.yaml + +CHART_FILES=( $(yq eval --no-doc .spec.chart "$FILE_TRAEFIK_MANIFEST" | xargs -n1 basename) ) +# These files are: +# 1. traefik-crd-20.3.1+up20.3.0.tgz +# 2. traefik-20.3.1+up20.3.0.tgz +# at the time of writing + +if [[ "${#CHART_FILES[@]}" != "2" ]]; then + echo "New manifest charts added, the packaging scripts will need to be updated: ${CHART_FILES}" + exit 1 +fi + +CHARTS_URL=https://k3s.io/k3s-charts/assets +# Get metadata for both files +rm -f chart-versions.nix.update +cat > chart-versions.nix.update <<EOF +{ + traefik-crd = { + url = "${CHARTS_URL}/traefik-crd/${CHART_FILES[0]}"; + sha256 = "$(nix-prefetch-url --quiet "${CHARTS_URL}/traefik-crd/${CHART_FILES[0]}")"; + }; + traefik = { + url = "${CHARTS_URL}/traefik/${CHART_FILES[1]}"; + sha256 = "$(nix-prefetch-url --quiet "${CHARTS_URL}/traefik/${CHART_FILES[1]}")"; + }; +} +EOF +mv chart-versions.nix.update chart-versions.nix + +FILE_GO_MOD=${WORKDIR}/go.mod +curl --silent https://raw.githubusercontent.com/k3s-io/k3s/${K3S_COMMIT}/go.mod > $FILE_GO_MOD + + +K3S_ROOT_VERSION=$(grep 'VERSION_ROOT=' ${FILE_SCRIPTS_VERSION} \ + | cut -d'=' -f2 | sed -e 's/"//g' -e 's/^v//') +K3S_ROOT_SHA256=$(nix-prefetch-url --quiet --unpack \ + "https://github.com/k3s-io/k3s-root/releases/download/v${K3S_ROOT_VERSION}/k3s-root-amd64.tar") + +CNIPLUGINS_VERSION=$(grep 'VERSION_CNIPLUGINS=' ${FILE_SCRIPTS_VERSION} \ + | cut -d'=' -f2 | sed -e 's/"//g' -e 's/^v//') +CNIPLUGINS_SHA256=$(nix-prefetch-url --quiet --unpack \ + "https://github.com/rancher/plugins/archive/refs/tags/v${CNIPLUGINS_VERSION}.tar.gz") + +# mimics https://github.com/k3s-io/k3s/blob/v1.26.5%2Bk3s1/scripts/version.sh#L25 +CONTAINERD_VERSION=$(grep github.com/containerd/containerd ${FILE_GO_MOD} \ + | head -n1 | awk '{print $4}' | sed -e 's/^v//') +CONTAINERD_SHA256=$(nix-prefetch-url --quiet --unpack \ + "https://github.com/k3s-io/containerd/archive/refs/tags/v${CONTAINERD_VERSION}.tar.gz") + +CRI_CTL_VERSION=$(grep github.com/kubernetes-sigs/cri-tools ${FILE_GO_MOD} \ + | head -n1 | awk '{print $4}' | sed -e 's/"//g' -e 's/^v//') + +setKV () { + sed -i "s|$1 = \".*\"|$1 = \"${2:-}\"|" ${NIXPKGS_K3S_PATH}default.nix +} + +FAKE_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + +cat >versions.nix <<EOF +{ + k3sVersion = "${K3S_VERSION}"; + k3sCommit = "${K3S_COMMIT}"; + k3sRepoSha256 = "${K3S_REPO_SHA256}"; + k3sVendorHash = "${FAKE_HASH}"; + chartVersions = import ./chart-versions.nix; + k3sRootVersion = "${K3S_ROOT_VERSION}"; + k3sRootSha256 = "${K3S_ROOT_SHA256}"; + k3sCNIVersion = "${CNIPLUGINS_VERSION}"; + k3sCNISha256 = "${CNIPLUGINS_SHA256}"; + containerdVersion = "${CONTAINERD_VERSION}"; + containerdSha256 = "${CONTAINERD_SHA256}"; + criCtlVersion = "${CRI_CTL_VERSION}"; +} +EOF + +set +e +K3S_VENDOR_HASH=$(nix-prefetch -I nixpkgs=${NIXPKGS_ROOT} "{ sha256 }: (import ${NIXPKGS_ROOT}. {}).k3s_1_${MINOR_VERSION}.goModules.overrideAttrs (_: { vendorHash = sha256; })") +set -e + +if [ -n "${K3S_VENDOR_HASH:-}" ]; then + sed -i "s|${FAKE_HASH}|${K3S_VENDOR_HASH}|g" ./versions.nix +else + echo "Update failed. K3S_VENDOR_HASH is empty." + exit 1 +fi + +# Implement commit +# See https://nixos.org/manual/nixpkgs/stable/#var-passthru-updateScript-commit +OLD_VERSION="$(nix-instantiate --eval -E "with import $NIXPKGS_ROOT. {}; k3s.version or (builtins.parseDrvName k3s.name).version" | tr -d '"')" +cat <<EOF +[{ + "attrPath": "k3s_1_${MINOR_VERSION}", + "oldVersion": "$OLD_VERSION", + "newVersion": "$K3S_VERSION", + "files": ["$PWD/versions.nix","$PWD/chart-versions.nix"] +}] +EOF |