From 457f56813d21053aa3a58a6a6b5d6dcb3b019e6c Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Sat, 20 May 2023 15:49:08 +0900 Subject: k3s: add "1_26" builder which can be used for 1_27 too This is meant to make it easier to maintain multiple versions of the package without having to copy+paste folders every time. We can still copy+paste if the builders diverge enough to warrant it. --- .../networking/cluster/k3s/1_26/default.nix | 337 --------------------- .../networking/cluster/k3s/builders/1_26.nix | 335 ++++++++++++++++++++ .../networking/cluster/k3s/default.nix | 23 ++ 3 files changed, 358 insertions(+), 337 deletions(-) delete mode 100644 pkgs/applications/networking/cluster/k3s/1_26/default.nix create mode 100644 pkgs/applications/networking/cluster/k3s/builders/1_26.nix create mode 100644 pkgs/applications/networking/cluster/k3s/default.nix (limited to 'pkgs/applications/networking/cluster/k3s') diff --git a/pkgs/applications/networking/cluster/k3s/1_26/default.nix b/pkgs/applications/networking/cluster/k3s/1_26/default.nix deleted file mode 100644 index f7dbe7e6839e..000000000000 --- a/pkgs/applications/networking/cluster/k3s/1_26/default.nix +++ /dev/null @@ -1,337 +0,0 @@ -{ stdenv -, lib -, makeWrapper -, socat -, iptables -, iproute2 -, ipset -, bridge-utils -, btrfs-progs -, conntrack-tools -, buildGoModule -, runc -, rsync -, kmod -, libseccomp -, pkg-config -, ethtool -, util-linux -, fetchFromGitHub -, fetchurl -, fetchzip -, fetchgit -, zstd -, yq-go -, sqlite -, nixosTests -, pkgsBuildBuild -}: - -# k3s is a kinda weird derivation. One of the main points of k3s is the -# simplicity of it being one binary that can perform several tasks. -# However, when you have a good package manager (like nix), that doesn't -# actually make much of a difference; you don't really care if it's one binary -# or 10 since with a good package manager, installing and running it is -# identical. -# Since upstream k3s packages itself as one large binary with several -# "personalities" (in the form of subcommands like 'k3s agent' and 'k3s -# kubectl'), it ends up being easiest to mostly mimic upstream packaging, with -# some exceptions. -# K3s also carries patches to some packages (such as containerd and cni -# plugins), so we intentionally use the k3s versions of those binaries for k3s, -# even if the upstream version of those binaries exist in nixpkgs already. In -# the end, that means we have a thick k3s binary that behaves like the upstream -# one for the most part. -# However, k3s also bundles several pieces of unpatched software, from the -# strongswan vpn software, to iptables, to socat, conntrack, busybox, etc. -# Those pieces of software we entirely ignore upstream's handling of, and just -# make sure they're in the path if desired. -let - k3sVersion = "1.26.4+k3s1"; # k3s git tag - k3sCommit = "8d0255af07e95b841952563253d27b0d10bd72f0"; # k3s git commit at the above version - k3sRepoSha256 = "0qlszdnlsvj3hzx2p0wl3zhaw908w8a62z6vlf2g69a3c75f55cs"; - k3sVendorSha256 = "sha256-JXTsZYtTspu/pWMRSS2BcegktawBJ6BK7YEKbz1J/ao="; - - # nix generated by update.sh - # Based on the traefik charts here: https://github.com/k3s-io/k3s/blob/d71ab6317e22dd34673faa307a412a37a16767f6/scripts/download#L29-L32 - # see also https://github.com/k3s-io/k3s/blob/d71ab6317e22dd34673faa307a412a37a16767f6/manifests/traefik.yaml#L8 - # At the time of writing, there are two traefik charts, and that's it - charts = import ./chart-versions.nix; - - # taken from ./scripts/version.sh VERSION_ROOT https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L47 - k3sRootVersion = "0.12.1"; - k3sRootSha256 = "0724yx3zk89m2239fmdgwzf9w672pik71xqrvgb7pdmknmmdn9f4"; - - # taken from ./scripts/version.sh VERSION_CNIPLUGINS https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L45 - k3sCNIVersion = "1.1.1-k3s1"; - k3sCNISha256 = "14mb3zsqibj1sn338gjmsyksbm0mxv9p016dij7zidccx2rzn6nl"; - - # taken from ./scripts/version.sh VERSION_CONTAINERD - containerdVersion = "1.6.19-k3s1"; - containerdSha256 = "12dwqh77wplg30kdi73d90qni23agw2cwxjd2p5lchq86mpmmwwr"; - - # run `grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}'` in the k3s repo at the tag - criCtlVersion = "1.26.0-rc.0-k3s1"; - - baseMeta = with lib; { - description = "A lightweight Kubernetes distribution"; - license = licenses.asl20; - homepage = "https://k3s.io"; - maintainers = with maintainers; [ euank mic92 yajo ]; - platforms = platforms.linux; - }; - - # https://github.com/k3s-io/k3s/blob/5fb370e53e0014dc96183b8ecb2c25a61e891e76/scripts/build#L19-L40 - versionldflags = [ - "-X github.com/rancher/k3s/pkg/version.Version=v${k3sVersion}" - "-X github.com/rancher/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}" - "-X k8s.io/client-go/pkg/version.gitVersion=v${k3sVersion}" - "-X k8s.io/client-go/pkg/version.gitCommit=${k3sCommit}" - "-X k8s.io/client-go/pkg/version.gitTreeState=clean" - "-X k8s.io/client-go/pkg/version.buildDate=1970-01-01T01:01:01Z" - "-X k8s.io/component-base/version.gitVersion=v${k3sVersion}" - "-X k8s.io/component-base/version.gitCommit=${k3sCommit}" - "-X k8s.io/component-base/version.gitTreeState=clean" - "-X k8s.io/component-base/version.buildDate=1970-01-01T01:01:01Z" - "-X github.com/kubernetes-sigs/cri-tools/pkg/version.Version=v${criCtlVersion}" - "-X github.com/containerd/containerd/version.Version=v${containerdVersion}" - "-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd" - ]; - - # bundled into the k3s binary - traefikChart = fetchurl charts.traefik; - traefik-crdChart = fetchurl charts.traefik-crd; - - # so, k3s is a complicated thing to package - # This derivation attempts to avoid including any random binaries from the - # internet. k3s-root is _mostly_ binaries built to be bundled in k3s (which - # we don't care about doing, we can add those as build or runtime - # dependencies using a real package manager). - # In addition to those binaries, it's also configuration though (right now - # mostly strongswan configuration), and k3s does use those files. - # As such, we download it in order to grab 'etc' and bundle it into the final - # k3s binary. - k3sRoot = fetchzip { - # Note: marked as apache 2.0 license - url = "https://github.com/k3s-io/k3s-root/releases/download/v${k3sRootVersion}/k3s-root-amd64.tar"; - sha256 = k3sRootSha256; - stripRoot = false; - }; - k3sCNIPlugins = buildGoModule rec { - pname = "k3s-cni-plugins"; - version = k3sCNIVersion; - vendorSha256 = null; - - subPackages = [ "." ]; - - src = fetchFromGitHub { - owner = "rancher"; - repo = "plugins"; - rev = "v${version}"; - sha256 = k3sCNISha256; - }; - - postInstall = '' - mv $out/bin/plugins $out/bin/cni - ''; - - meta = baseMeta // { - description = "CNI plugins, as patched by rancher for k3s"; - }; - }; - # Grab this separately from a build because it's used by both stages of the - # k3s build. - k3sRepo = fetchgit { - url = "https://github.com/k3s-io/k3s"; - rev = "v${k3sVersion}"; - sha256 = k3sRepoSha256; - }; - # Stage 1 of the k3s build: - # Let's talk about how k3s is structured. - # One of the ideas of k3s is that there's the single "k3s" binary which can - # do everything you need, from running a k3s server, to being a worker node, - # to running kubectl. - # The way that actually works is that k3s is a single go binary that contains - # a bunch of bindata that it unpacks at runtime into directories (either the - # user's home directory or /var/lib/rancher if run as root). - # This bindata includes both binaries and configuration. - # In order to let nixpkgs do all its autostripping/patching/etc, we split this into two derivations. - # First, we build all the binaries that get packed into the thick k3s binary - # (and output them from one derivation so they'll all be suitably patched up). - # Then, we bundle those binaries into our thick k3s binary and use that as - # the final single output. - # This approach was chosen because it ensures the bundled binaries all are - # correctly built to run with nix (we can lean on the existing buildGoModule - # stuff), and we can again lean on that tooling for the final k3s binary too. - # Other alternatives would be to manually run the - # strip/patchelf/remove-references step ourselves in the installPhase of the - # derivation when we've built all the binaries, but haven't bundled them in - # with generated bindata yet. - - k3sServer = buildGoModule rec { - pname = "k3s-server"; - version = k3sVersion; - - src = k3sRepo; - vendorSha256 = k3sVendorSha256; - - nativeBuildInputs = [ pkg-config ]; - buildInputs = [ libseccomp sqlite.dev ]; - - subPackages = [ "cmd/server" ]; - ldflags = versionldflags; - - tags = [ "libsqlite3" "linux" ]; - - # create the multicall symlinks for k3s - postInstall = '' - mv $out/bin/server $out/bin/k3s - pushd $out - # taken verbatim from https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/build#L105-L113 - ln -s k3s ./bin/k3s-agent - ln -s k3s ./bin/k3s-server - ln -s k3s ./bin/k3s-etcd-snapshot - ln -s k3s ./bin/k3s-secrets-encrypt - ln -s k3s ./bin/k3s-certificate - ln -s k3s ./bin/kubectl - ln -s k3s ./bin/crictl - ln -s k3s ./bin/ctr - popd - ''; - - meta = baseMeta // { - description = "The various binaries that get packaged into the final k3s binary"; - }; - }; - k3sContainerd = buildGoModule { - pname = "k3s-containerd"; - version = containerdVersion; - src = fetchFromGitHub { - owner = "k3s-io"; - repo = "containerd"; - rev = "v${containerdVersion}"; - sha256 = containerdSha256; - }; - vendorSha256 = null; - buildInputs = [ btrfs-progs ]; - subPackages = [ "cmd/containerd" "cmd/containerd-shim-runc-v2" ]; - ldflags = versionldflags; - }; -in -buildGoModule rec { - pname = "k3s"; - version = k3sVersion; - - src = k3sRepo; - vendorSha256 = k3sVendorSha256; - - postPatch = '' - # Nix prefers dynamically linked binaries over static binary. - - substituteInPlace scripts/package-cli \ - --replace '"$LDFLAGS $STATIC" -o' \ - '"$LDFLAGS" -o' \ - --replace "STATIC=\"-extldflags \'-static\'\"" \ - "" - - # Upstream codegen fails with trimpath set. Removes "trimpath" for 'go generate': - - substituteInPlace scripts/package-cli \ - --replace '"''${GO}" generate' \ - 'GOFLAGS="" \ - GOOS="${pkgsBuildBuild.go.GOOS}" \ - GOARCH="${pkgsBuildBuild.go.GOARCH}" \ - CC="${pkgsBuildBuild.stdenv.cc}/bin/cc" \ - "''${GO}" generate' - ''; - - # Important utilities used by the kubelet, see - # https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494 - # Note the list in that issue is stale and some aren't relevant for k3s. - k3sRuntimeDeps = [ - kmod - socat - iptables - iproute2 - ipset - bridge-utils - ethtool - util-linux # kubelet wants 'nsenter' from util-linux: https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-705994388 - conntrack-tools - ]; - - buildInputs = k3sRuntimeDeps; - - nativeBuildInputs = [ - makeWrapper - rsync - yq-go - zstd - ]; - - # embedded in the final k3s cli - propagatedBuildInputs = [ - k3sCNIPlugins - k3sContainerd - k3sServer - runc - ]; - - # We override most of buildPhase due to peculiarities in k3s's build. - # Specifically, it has a 'go generate' which runs part of the package. See - # this comment: - # https://github.com/NixOS/nixpkgs/pull/158089#discussion_r799965694 - # So, why do we use buildGoModule at all? For the `vendorSha256` / `go mod download` stuff primarily. - buildPhase = '' - patchShebangs ./scripts/package-cli ./scripts/download ./scripts/build-upload - - # copy needed 'go generate' inputs into place - mkdir -p ./bin/aux - rsync -a --no-perms ${k3sServer}/bin/ ./bin/ - ln -vsf ${runc}/bin/runc ./bin/runc - ln -vsf ${k3sCNIPlugins}/bin/cni ./bin/cni - ln -vsf ${k3sContainerd}/bin/* ./bin/ - rsync -a --no-perms --chmod u=rwX ${k3sRoot}/etc/ ./etc/ - mkdir -p ./build/static/charts - - cp ${traefikChart} ./build/static/charts - cp ${traefik-crdChart} ./build/static/charts - - export ARCH=$GOARCH - export DRONE_TAG="v${k3sVersion}" - export DRONE_COMMIT="${k3sCommit}" - # use ./scripts/package-cli to run 'go generate' + 'go build' - - ./scripts/package-cli - mkdir -p $out/bin - ''; - - # Otherwise it depends on 'getGoDirs', which is normally set in buildPhase - doCheck = false; - - installPhase = '' - # wildcard to match the arm64 build too - install -m 0755 dist/artifacts/k3s* -D $out/bin/k3s - wrapProgram $out/bin/k3s \ - --prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \ - --prefix PATH : "$out/bin" - ''; - - doInstallCheck = true; - installCheckPhase = '' - $out/bin/k3s --version | grep -F "v${k3sVersion}" >/dev/null - ''; - - passthru.updateScript = ./update.sh; - - passthru.mkTests = version: - let k3s_version = "k3s_" + lib.replaceStrings ["."] ["_"] (lib.versions.majorMinor version); - in { - single-node = nixosTests.k3s.single-node.${k3s_version}; - multi-node = nixosTests.k3s.multi-node.${k3s_version}; - }; - passthru.tests = passthru.mkTests k3sVersion; - - - meta = baseMeta; -} diff --git a/pkgs/applications/networking/cluster/k3s/builders/1_26.nix b/pkgs/applications/networking/cluster/k3s/builders/1_26.nix new file mode 100644 index 000000000000..fa34782d39a2 --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/builders/1_26.nix @@ -0,0 +1,335 @@ +{ lib +, makeWrapper +, socat +, iptables +, iproute2 +, ipset +, bridge-utils +, btrfs-progs +, conntrack-tools +, buildGoModule +, runc +, rsync +, kmod +, libseccomp +, pkg-config +, ethtool +, util-linux +, fetchFromGitHub +, fetchurl +, fetchzip +, fetchgit +, zstd +, yq-go +, sqlite +, nixosTests +, pkgsBuildBuild +}: + +{ + # git tag + k3sVersion, + # commit hash + k3sCommit, + k3sRepoSha256 ? lib.fakeHash, + k3sVendorSha256 ? lib.fakeHash, + # taken from ./scripts/version.sh VERSION_ROOT https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L47 + k3sRootVersion, + k3sRootSha256 ? lib.fakeHash, + # Based on the traefik charts here: https://github.com/k3s-io/k3s/blob/d71ab6317e22dd34673faa307a412a37a16767f6/scripts/download#L29-L32 + # see also https://github.com/k3s-io/k3s/blob/d71ab6317e22dd34673faa307a412a37a16767f6/manifests/traefik.yaml#L8 + chartVersions, + # taken from ./scripts/version.sh VERSION_CNIPLUGINS https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L45 + k3sCNIVersion, + k3sCNISha256 ? lib.fakeHash, + # taken from ./scripts/version.sh VERSION_CONTAINERD + containerdVersion, + containerdSha256 ? lib.fakeHash, + # run `grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}'` in the k3s repo at the tag + criCtlVersion, + updateScript ? null, +}: + +# k3s is a kinda weird derivation. One of the main points of k3s is the +# simplicity of it being one binary that can perform several tasks. +# However, when you have a good package manager (like nix), that doesn't +# actually make much of a difference; you don't really care if it's one binary +# or 10 since with a good package manager, installing and running it is +# identical. +# Since upstream k3s packages itself as one large binary with several +# "personalities" (in the form of subcommands like 'k3s agent' and 'k3s +# kubectl'), it ends up being easiest to mostly mimic upstream packaging, with +# some exceptions. +# K3s also carries patches to some packages (such as containerd and cni +# plugins), so we intentionally use the k3s versions of those binaries for k3s, +# even if the upstream version of those binaries exist in nixpkgs already. In +# the end, that means we have a thick k3s binary that behaves like the upstream +# one for the most part. +# However, k3s also bundles several pieces of unpatched software, from the +# strongswan vpn software, to iptables, to socat, conntrack, busybox, etc. +# Those pieces of software we entirely ignore upstream's handling of, and just +# make sure they're in the path if desired. +let + + baseMeta = with lib; { + description = "A lightweight Kubernetes distribution"; + license = licenses.asl20; + homepage = "https://k3s.io"; + maintainers = with maintainers; [ euank mic92 yajo ]; + platforms = platforms.linux; + }; + + # https://github.com/k3s-io/k3s/blob/5fb370e53e0014dc96183b8ecb2c25a61e891e76/scripts/build#L19-L40 + versionldflags = [ + "-X github.com/rancher/k3s/pkg/version.Version=v${k3sVersion}" + "-X github.com/rancher/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitVersion=v${k3sVersion}" + "-X k8s.io/client-go/pkg/version.gitCommit=${k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitTreeState=clean" + "-X k8s.io/client-go/pkg/version.buildDate=1970-01-01T01:01:01Z" + "-X k8s.io/component-base/version.gitVersion=v${k3sVersion}" + "-X k8s.io/component-base/version.gitCommit=${k3sCommit}" + "-X k8s.io/component-base/version.gitTreeState=clean" + "-X k8s.io/component-base/version.buildDate=1970-01-01T01:01:01Z" + "-X github.com/kubernetes-sigs/cri-tools/pkg/version.Version=v${criCtlVersion}" + "-X github.com/containerd/containerd/version.Version=v${containerdVersion}" + "-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd" + ]; + + # bundled into the k3s binary + traefikChart = fetchurl chartVersions.traefik; + traefik-crdChart = fetchurl chartVersions.traefik-crd; + + # so, k3s is a complicated thing to package + # This derivation attempts to avoid including any random binaries from the + # internet. k3s-root is _mostly_ binaries built to be bundled in k3s (which + # we don't care about doing, we can add those as build or runtime + # dependencies using a real package manager). + # In addition to those binaries, it's also configuration though (right now + # mostly strongswan configuration), and k3s does use those files. + # As such, we download it in order to grab 'etc' and bundle it into the final + # k3s binary. + k3sRoot = fetchzip { + # Note: marked as apache 2.0 license + url = "https://github.com/k3s-io/k3s-root/releases/download/v${k3sRootVersion}/k3s-root-amd64.tar"; + sha256 = k3sRootSha256; + stripRoot = false; + }; + k3sCNIPlugins = buildGoModule rec { + pname = "k3s-cni-plugins"; + version = k3sCNIVersion; + vendorSha256 = null; + + subPackages = [ "." ]; + + src = fetchFromGitHub { + owner = "rancher"; + repo = "plugins"; + rev = "v${version}"; + sha256 = k3sCNISha256; + }; + + postInstall = '' + mv $out/bin/plugins $out/bin/cni + ''; + + meta = baseMeta // { + description = "CNI plugins, as patched by rancher for k3s"; + }; + }; + # Grab this separately from a build because it's used by both stages of the + # k3s build. + k3sRepo = fetchgit { + url = "https://github.com/k3s-io/k3s"; + rev = "v${k3sVersion}"; + sha256 = k3sRepoSha256; + }; + # Stage 1 of the k3s build: + # Let's talk about how k3s is structured. + # One of the ideas of k3s is that there's the single "k3s" binary which can + # do everything you need, from running a k3s server, to being a worker node, + # to running kubectl. + # The way that actually works is that k3s is a single go binary that contains + # a bunch of bindata that it unpacks at runtime into directories (either the + # user's home directory or /var/lib/rancher if run as root). + # This bindata includes both binaries and configuration. + # In order to let nixpkgs do all its autostripping/patching/etc, we split this into two derivations. + # First, we build all the binaries that get packed into the thick k3s binary + # (and output them from one derivation so they'll all be suitably patched up). + # Then, we bundle those binaries into our thick k3s binary and use that as + # the final single output. + # This approach was chosen because it ensures the bundled binaries all are + # correctly built to run with nix (we can lean on the existing buildGoModule + # stuff), and we can again lean on that tooling for the final k3s binary too. + # Other alternatives would be to manually run the + # strip/patchelf/remove-references step ourselves in the installPhase of the + # derivation when we've built all the binaries, but haven't bundled them in + # with generated bindata yet. + + k3sServer = buildGoModule { + pname = "k3s-server"; + version = k3sVersion; + + src = k3sRepo; + vendorSha256 = k3sVendorSha256; + + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ libseccomp sqlite.dev ]; + + subPackages = [ "cmd/server" ]; + ldflags = versionldflags; + + tags = [ "libsqlite3" "linux" ]; + + # create the multicall symlinks for k3s + postInstall = '' + mv $out/bin/server $out/bin/k3s + pushd $out + # taken verbatim from https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/build#L105-L113 + ln -s k3s ./bin/k3s-agent + ln -s k3s ./bin/k3s-server + ln -s k3s ./bin/k3s-etcd-snapshot + ln -s k3s ./bin/k3s-secrets-encrypt + ln -s k3s ./bin/k3s-certificate + ln -s k3s ./bin/kubectl + ln -s k3s ./bin/crictl + ln -s k3s ./bin/ctr + popd + ''; + + meta = baseMeta // { + description = "The various binaries that get packaged into the final k3s binary"; + }; + }; + k3sContainerd = buildGoModule { + pname = "k3s-containerd"; + version = containerdVersion; + src = fetchFromGitHub { + owner = "k3s-io"; + repo = "containerd"; + rev = "v${containerdVersion}"; + sha256 = containerdSha256; + }; + vendorSha256 = null; + buildInputs = [ btrfs-progs ]; + subPackages = [ "cmd/containerd" "cmd/containerd-shim-runc-v2" ]; + ldflags = versionldflags; + }; +in +buildGoModule rec { + pname = "k3s"; + version = k3sVersion; + + src = k3sRepo; + vendorSha256 = k3sVendorSha256; + + postPatch = '' + # Nix prefers dynamically linked binaries over static binary. + + substituteInPlace scripts/package-cli \ + --replace '"$LDFLAGS $STATIC" -o' \ + '"$LDFLAGS" -o' \ + --replace "STATIC=\"-extldflags \'-static\'\"" \ + "" + + # Upstream codegen fails with trimpath set. Removes "trimpath" for 'go generate': + + substituteInPlace scripts/package-cli \ + --replace '"''${GO}" generate' \ + 'GOFLAGS="" \ + GOOS="${pkgsBuildBuild.go.GOOS}" \ + GOARCH="${pkgsBuildBuild.go.GOARCH}" \ + CC="${pkgsBuildBuild.stdenv.cc}/bin/cc" \ + "''${GO}" generate' + ''; + + # Important utilities used by the kubelet, see + # https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494 + # Note the list in that issue is stale and some aren't relevant for k3s. + k3sRuntimeDeps = [ + kmod + socat + iptables + iproute2 + ipset + bridge-utils + ethtool + util-linux # kubelet wants 'nsenter' from util-linux: https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-705994388 + conntrack-tools + ]; + + buildInputs = k3sRuntimeDeps; + + nativeBuildInputs = [ + makeWrapper + rsync + yq-go + zstd + ]; + + # embedded in the final k3s cli + propagatedBuildInputs = [ + k3sCNIPlugins + k3sContainerd + k3sServer + runc + ]; + + # We override most of buildPhase due to peculiarities in k3s's build. + # Specifically, it has a 'go generate' which runs part of the package. See + # this comment: + # https://github.com/NixOS/nixpkgs/pull/158089#discussion_r799965694 + # So, why do we use buildGoModule at all? For the `vendorSha256` / `go mod download` stuff primarily. + buildPhase = '' + patchShebangs ./scripts/package-cli ./scripts/download ./scripts/build-upload + + # copy needed 'go generate' inputs into place + mkdir -p ./bin/aux + rsync -a --no-perms ${k3sServer}/bin/ ./bin/ + ln -vsf ${runc}/bin/runc ./bin/runc + ln -vsf ${k3sCNIPlugins}/bin/cni ./bin/cni + ln -vsf ${k3sContainerd}/bin/* ./bin/ + rsync -a --no-perms --chmod u=rwX ${k3sRoot}/etc/ ./etc/ + mkdir -p ./build/static/charts + + cp ${traefikChart} ./build/static/charts + cp ${traefik-crdChart} ./build/static/charts + + export ARCH=$GOARCH + export DRONE_TAG="v${k3sVersion}" + export DRONE_COMMIT="${k3sCommit}" + # use ./scripts/package-cli to run 'go generate' + 'go build' + + ./scripts/package-cli + mkdir -p $out/bin + ''; + + # Otherwise it depends on 'getGoDirs', which is normally set in buildPhase + doCheck = false; + + installPhase = '' + # wildcard to match the arm64 build too + install -m 0755 dist/artifacts/k3s* -D $out/bin/k3s + wrapProgram $out/bin/k3s \ + --prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \ + --prefix PATH : "$out/bin" + ''; + + doInstallCheck = true; + installCheckPhase = '' + $out/bin/k3s --version | grep -F "v${k3sVersion}" >/dev/null + ''; + + passthru.updateScript = updateScript; + + passthru.mkTests = version: + let k3s_version = "k3s_" + lib.replaceStrings ["."] ["_"] (lib.versions.majorMinor version); + in { + single-node = nixosTests.k3s.single-node.${k3s_version}; + multi-node = nixosTests.k3s.multi-node.${k3s_version}; + }; + passthru.tests = passthru.mkTests k3sVersion; + + + meta = baseMeta; +} diff --git a/pkgs/applications/networking/cluster/k3s/default.nix b/pkgs/applications/networking/cluster/k3s/default.nix new file mode 100644 index 000000000000..52beacf7e6c1 --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/default.nix @@ -0,0 +1,23 @@ +{ lib, stdenv, callPackage }: + +let + # The 1_26 builder can compile 1.26, and hopefully newer versions + k3s_1_26_builder = import ./builders/1_26.nix; +in +{ + k3s_1_26 = (callPackage k3s_1_26_builder { }) { + k3sVersion = "1.26.4+k3s1"; + k3sCommit = "8d0255af07e95b841952563253d27b0d10bd72f0"; + k3sRepoSha256 = "0qlszdnlsvj3hzx2p0wl3zhaw908w8a62z6vlf2g69a3c75f55cs"; + k3sVendorSha256 = "sha256-JXTsZYtTspu/pWMRSS2BcegktawBJ6BK7YEKbz1J/ao="; + chartVersions = import ./1_26/chart-versions.nix; + k3sRootVersion = "0.12.1"; + k3sRootSha256 = "0724yx3zk89m2239fmdgwzf9w672pik71xqrvgb7pdmknmmdn9f4"; + k3sCNIVersion = "1.1.1-k3s1"; + k3sCNISha256 = "14mb3zsqibj1sn338gjmsyksbm0mxv9p016dij7zidccx2rzn6nl"; + containerdVersion = "1.6.19-k3s1"; + containerdSha256 = "12dwqh77wplg30kdi73d90qni23agw2cwxjd2p5lchq86mpmmwwr"; + criCtlVersion = "1.26.0-rc.0-k3s1"; + updateScript = ./1_26/update-script.sh; + }; +} -- cgit 1.4.1 From c71ef26ecc9e4353589b4987d27d7c60e3b94755 Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Sat, 20 May 2023 15:55:31 +0900 Subject: k3s: init 1.27.1+k3s1 On unstable, we're now tracking | version | EOL | |---------|-----| | 1.24 | 07/2023 | | 1.25 | 10/2023 | | 1.26 | 02/2024 | | 1.27 | 06/2024 | This commit adds 1.27 using the 1_26 builder introduced in the last commit. Notably, this commit also updates the 'update.sh' script to work for the new setup I have here. --- .../networking/cluster/k3s/1_27/chart-versions.nix | 10 ++ .../networking/cluster/k3s/1_27/update.sh | 127 +++++++++++++++++++++ .../networking/cluster/k3s/1_27/versions.nix | 14 +++ .../networking/cluster/k3s/default.nix | 6 +- pkgs/top-level/all-packages.nix | 7 +- 5 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 pkgs/applications/networking/cluster/k3s/1_27/chart-versions.nix create mode 100755 pkgs/applications/networking/cluster/k3s/1_27/update.sh create mode 100644 pkgs/applications/networking/cluster/k3s/1_27/versions.nix (limited to 'pkgs/applications/networking/cluster/k3s') diff --git a/pkgs/applications/networking/cluster/k3s/1_27/chart-versions.nix b/pkgs/applications/networking/cluster/k3s/1_27/chart-versions.nix new file mode 100644 index 000000000000..8c40604d0f1c --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/1_27/chart-versions.nix @@ -0,0 +1,10 @@ +{ + traefik-crd = { + url = "https://k3s.io/k3s-charts/assets/traefik-crd/traefik-crd-20.3.1+up20.3.0.tgz"; + sha256 = "1775vjldvqvhzdbzanxhbaqbmkih09yb91im651q8bc7z5sb9ckn"; + }; + traefik = { + url = "https://k3s.io/k3s-charts/assets/traefik/traefik-20.3.1+up20.3.0.tgz"; + sha256 = "1rj0f0n0vgjcbzfwzhqmsd501i2f6vw145w9plbp8gwdyzmg2nc6"; + }; +} diff --git a/pkgs/applications/networking/cluster/k3s/1_27/update.sh b/pkgs/applications/networking/cluster/k3s/1_27/update.sh new file mode 100755 index 000000000000..427e2a33d9c9 --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/1_27/update.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p curl gnugrep gnused jq yq-go nix-prefetch + +set -x -eu -o pipefail + +WORKDIR=$(mktemp -d) +trap "rm -rf ${WORKDIR}" EXIT + +NIXPKGS_ROOT="$(git rev-parse --show-toplevel)"/ +NIXPKGS_K3S_PATH=$(cd $(dirname ${BASH_SOURCE[0]}); pwd -P)/ +cd ${NIXPKGS_K3S_PATH} + +LATEST_TAG_RAWFILE=${WORKDIR}/latest_tag.json +curl --silent -f ${GITHUB_TOKEN:+-u ":$GITHUB_TOKEN"} \ + https://api.github.com/repos/k3s-io/k3s/releases > ${LATEST_TAG_RAWFILE} + +LATEST_TAG_NAME=$(jq 'map(.tag_name)' ${LATEST_TAG_RAWFILE} | \ + grep -v -e rc -e engine | tail -n +2 | head -n -1 | sed 's|[", ]||g' | sort -rV | head -n1) + +K3S_VERSION=$(echo ${LATEST_TAG_NAME} | sed 's/^v//') + +K3S_COMMIT=$(curl --silent -f ${GITHUB_TOKEN:+-u ":$GITHUB_TOKEN"} \ + https://api.github.com/repos/k3s-io/k3s/tags \ + | jq -r "map(select(.name == \"${LATEST_TAG_NAME}\")) | .[0] | .commit.sha") + +K3S_REPO_SHA256=$(nix-prefetch-url --quiet --unpack https://github.com/k3s-io/k3s/archive/refs/tags/${LATEST_TAG_NAME}.tar.gz) + +FILE_SCRIPTS_DOWNLOAD=${WORKDIR}/scripts-download +curl --silent -f https://raw.githubusercontent.com/k3s-io/k3s/${K3S_COMMIT}/scripts/download > $FILE_SCRIPTS_DOWNLOAD + +FILE_SCRIPTS_VERSION=${WORKDIR}/scripts-version.sh +curl --silent -f https://raw.githubusercontent.com/k3s-io/k3s/${K3S_COMMIT}/scripts/version.sh > $FILE_SCRIPTS_VERSION + +FILE_TRAEFIK_MANIFEST=${WORKDIR}/traefik.yml +curl --silent -f -o "$FILE_TRAEFIK_MANIFEST" https://raw.githubusercontent.com/k3s-io/k3s/${K3S_COMMIT}/manifests/traefik.yaml + +CHART_FILES=( $(yq eval --no-doc .spec.chart "$FILE_TRAEFIK_MANIFEST" | xargs -n1 basename) ) +# These files are: +# 1. traefik-crd-20.3.1+up20.3.0.tgz +# 2. traefik-20.3.1+up20.3.0.tgz +# at the time of writing + +if [[ "${#CHART_FILES[@]}" != "2" ]]; then + echo "New manifest charts added, the packaging scripts will need to be updated: ${CHART_FILES}" + exit 1 +fi + +CHARTS_URL=https://k3s.io/k3s-charts/assets +# Get metadata for both files +rm -f chart-versions.nix.update +cat > chart-versions.nix.update < $FILE_GO_MOD + + +K3S_ROOT_VERSION=$(grep 'VERSION_ROOT=' ${FILE_SCRIPTS_VERSION} \ + | cut -d'=' -f2 | sed -e 's/"//g' -e 's/^v//') +K3S_ROOT_SHA256=$(nix-prefetch-url --quiet --unpack \ + "https://github.com/k3s-io/k3s-root/releases/download/v${K3S_ROOT_VERSION}/k3s-root-amd64.tar") + +CNIPLUGINS_VERSION=$(grep 'VERSION_CNIPLUGINS=' ${FILE_SCRIPTS_VERSION} \ + | cut -d'=' -f2 | sed -e 's/"//g' -e 's/^v//') +CNIPLUGINS_SHA256=$(nix-prefetch-url --quiet --unpack \ + "https://github.com/rancher/plugins/archive/refs/tags/v${CNIPLUGINS_VERSION}.tar.gz") + +CONTAINERD_VERSION=$(grep 'VERSION_CONTAINERD=' ${FILE_SCRIPTS_VERSION} \ + | cut -d'=' -f2 | sed -e 's/"//g' -e 's/^v//') +CONTAINERD_SHA256=$(nix-prefetch-url --quiet --unpack \ + "https://github.com/k3s-io/containerd/archive/refs/tags/v${CONTAINERD_VERSION}.tar.gz") + +CRI_CTL_VERSION=$(grep github.com/kubernetes-sigs/cri-tools ${FILE_GO_MOD} \ + | head -n1 | awk '{print $4}' | sed -e 's/"//g' -e 's/^v//') + +setKV () { + sed -i "s|$1 = \".*\"|$1 = \"${2:-}\"|" ${NIXPKGS_K3S_PATH}default.nix +} + +FAKE_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + +cat >versions.nix < ${K3S_VERSION}" +fi diff --git a/pkgs/applications/networking/cluster/k3s/1_27/versions.nix b/pkgs/applications/networking/cluster/k3s/1_27/versions.nix new file mode 100644 index 000000000000..b06e30ba5c96 --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/1_27/versions.nix @@ -0,0 +1,14 @@ +{ + k3sVersion = "1.27.1+k3s1"; + k3sCommit = "bc5b42c27908ab430101eff0db0a0b22f870bd7a"; + k3sRepoSha256 = "1xj3blfayrsfbcsljjdaswy49hhz8yiwf1d85arnsgbn8fidswpm"; + k3sVendorSha256 = "sha256-+sM2fjS88kxMQzra2t+jU1IaKCoJpW7p3w7lCOv5mMU="; + chartVersions = import ./chart-versions.nix; + k3sRootVersion = "0.12.1"; + k3sRootSha256 = "0724yx3zk89m2239fmdgwzf9w672pik71xqrvgb7pdmknmmdn9f4"; + k3sCNIVersion = "1.1.1-k3s1"; + k3sCNISha256 = "14mb3zsqibj1sn338gjmsyksbm0mxv9p016dij7zidccx2rzn6nl"; + containerdVersion = "1.6.19-k3s1"; + containerdSha256 = "12dwqh77wplg30kdi73d90qni23agw2cwxjd2p5lchq86mpmmwwr"; + criCtlVersion = "1.26.0-rc.0-k3s1"; +} diff --git a/pkgs/applications/networking/cluster/k3s/default.nix b/pkgs/applications/networking/cluster/k3s/default.nix index 52beacf7e6c1..5a6b68eb5a9a 100644 --- a/pkgs/applications/networking/cluster/k3s/default.nix +++ b/pkgs/applications/networking/cluster/k3s/default.nix @@ -18,6 +18,10 @@ in containerdVersion = "1.6.19-k3s1"; containerdSha256 = "12dwqh77wplg30kdi73d90qni23agw2cwxjd2p5lchq86mpmmwwr"; criCtlVersion = "1.26.0-rc.0-k3s1"; - updateScript = ./1_26/update-script.sh; + }; + + # 1_27 can be built with the same builder as 1_26 + k3s_1_27 = (callPackage k3s_1_26_builder { }) (import ./1_27/versions.nix) // { + updateScript = ./1_27/update-script.sh; }; } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 7db17064d0f3..90b5c3503b9e 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -31674,8 +31674,11 @@ with pkgs; k3s_1_24 = callPackage ../applications/networking/cluster/k3s/1_24 { }; k3s_1_25 = callPackage ../applications/networking/cluster/k3s/1_25 { }; - k3s_1_26 = (callPackage ../applications/networking/cluster/k3s { }).k3s_1_26; - k3s = k3s_1_26; + inherit (callPackage ../applications/networking/cluster/k3s { }) + k3s_1_26 + k3s_1_27 + ; + k3s = k3s_1_27; k3sup = callPackage ../applications/networking/cluster/k3sup { }; -- cgit 1.4.1 From 2257ccd54463d051611862d6a82753238eff22fd Mon Sep 17 00:00:00 2001 From: Euan Kemp Date: Mon, 22 May 2023 17:18:23 +0900 Subject: k3s: name builder less confusingly Per PR feedback --- .../networking/cluster/k3s/builder.nix | 341 +++++++++++++++++++++ .../networking/cluster/k3s/builders/1_26.nix | 335 -------------------- .../networking/cluster/k3s/default.nix | 7 +- 3 files changed, 344 insertions(+), 339 deletions(-) create mode 100644 pkgs/applications/networking/cluster/k3s/builder.nix delete mode 100644 pkgs/applications/networking/cluster/k3s/builders/1_26.nix (limited to 'pkgs/applications/networking/cluster/k3s') diff --git a/pkgs/applications/networking/cluster/k3s/builder.nix b/pkgs/applications/networking/cluster/k3s/builder.nix new file mode 100644 index 000000000000..0d3d127e4844 --- /dev/null +++ b/pkgs/applications/networking/cluster/k3s/builder.nix @@ -0,0 +1,341 @@ +# builder.nix contains a "builder" expression that, given k3s version and hash +# variables, creates a package for that version. +# Due to variance in k3s's build process, this builder only works for k3s 1.26+ +# currently. +# It is likely we will have to split out additional builders for additional +# versions in the future, or customize this one further. +{ lib +, makeWrapper +, socat +, iptables +, iproute2 +, ipset +, bridge-utils +, btrfs-progs +, conntrack-tools +, buildGoModule +, runc +, rsync +, kmod +, libseccomp +, pkg-config +, ethtool +, util-linux +, fetchFromGitHub +, fetchurl +, fetchzip +, fetchgit +, zstd +, yq-go +, sqlite +, nixosTests +, pkgsBuildBuild +}: + +{ + # git tag + k3sVersion, + # commit hash + k3sCommit, + k3sRepoSha256 ? lib.fakeHash, + k3sVendorSha256 ? lib.fakeHash, + # taken from ./scripts/version.sh VERSION_ROOT https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L47 + k3sRootVersion, + k3sRootSha256 ? lib.fakeHash, + # Based on the traefik charts here: https://github.com/k3s-io/k3s/blob/d71ab6317e22dd34673faa307a412a37a16767f6/scripts/download#L29-L32 + # see also https://github.com/k3s-io/k3s/blob/d71ab6317e22dd34673faa307a412a37a16767f6/manifests/traefik.yaml#L8 + chartVersions, + # taken from ./scripts/version.sh VERSION_CNIPLUGINS https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L45 + k3sCNIVersion, + k3sCNISha256 ? lib.fakeHash, + # taken from ./scripts/version.sh VERSION_CONTAINERD + containerdVersion, + containerdSha256 ? lib.fakeHash, + # run `grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}'` in the k3s repo at the tag + criCtlVersion, + updateScript ? null, +}: + +# k3s is a kinda weird derivation. One of the main points of k3s is the +# simplicity of it being one binary that can perform several tasks. +# However, when you have a good package manager (like nix), that doesn't +# actually make much of a difference; you don't really care if it's one binary +# or 10 since with a good package manager, installing and running it is +# identical. +# Since upstream k3s packages itself as one large binary with several +# "personalities" (in the form of subcommands like 'k3s agent' and 'k3s +# kubectl'), it ends up being easiest to mostly mimic upstream packaging, with +# some exceptions. +# K3s also carries patches to some packages (such as containerd and cni +# plugins), so we intentionally use the k3s versions of those binaries for k3s, +# even if the upstream version of those binaries exist in nixpkgs already. In +# the end, that means we have a thick k3s binary that behaves like the upstream +# one for the most part. +# However, k3s also bundles several pieces of unpatched software, from the +# strongswan vpn software, to iptables, to socat, conntrack, busybox, etc. +# Those pieces of software we entirely ignore upstream's handling of, and just +# make sure they're in the path if desired. +let + + baseMeta = with lib; { + description = "A lightweight Kubernetes distribution"; + license = licenses.asl20; + homepage = "https://k3s.io"; + maintainers = with maintainers; [ euank mic92 yajo ]; + platforms = platforms.linux; + }; + + # https://github.com/k3s-io/k3s/blob/5fb370e53e0014dc96183b8ecb2c25a61e891e76/scripts/build#L19-L40 + versionldflags = [ + "-X github.com/rancher/k3s/pkg/version.Version=v${k3sVersion}" + "-X github.com/rancher/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitVersion=v${k3sVersion}" + "-X k8s.io/client-go/pkg/version.gitCommit=${k3sCommit}" + "-X k8s.io/client-go/pkg/version.gitTreeState=clean" + "-X k8s.io/client-go/pkg/version.buildDate=1970-01-01T01:01:01Z" + "-X k8s.io/component-base/version.gitVersion=v${k3sVersion}" + "-X k8s.io/component-base/version.gitCommit=${k3sCommit}" + "-X k8s.io/component-base/version.gitTreeState=clean" + "-X k8s.io/component-base/version.buildDate=1970-01-01T01:01:01Z" + "-X github.com/kubernetes-sigs/cri-tools/pkg/version.Version=v${criCtlVersion}" + "-X github.com/containerd/containerd/version.Version=v${containerdVersion}" + "-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd" + ]; + + # bundled into the k3s binary + traefikChart = fetchurl chartVersions.traefik; + traefik-crdChart = fetchurl chartVersions.traefik-crd; + + # so, k3s is a complicated thing to package + # This derivation attempts to avoid including any random binaries from the + # internet. k3s-root is _mostly_ binaries built to be bundled in k3s (which + # we don't care about doing, we can add those as build or runtime + # dependencies using a real package manager). + # In addition to those binaries, it's also configuration though (right now + # mostly strongswan configuration), and k3s does use those files. + # As such, we download it in order to grab 'etc' and bundle it into the final + # k3s binary. + k3sRoot = fetchzip { + # Note: marked as apache 2.0 license + url = "https://github.com/k3s-io/k3s-root/releases/download/v${k3sRootVersion}/k3s-root-amd64.tar"; + sha256 = k3sRootSha256; + stripRoot = false; + }; + k3sCNIPlugins = buildGoModule rec { + pname = "k3s-cni-plugins"; + version = k3sCNIVersion; + vendorSha256 = null; + + subPackages = [ "." ]; + + src = fetchFromGitHub { + owner = "rancher"; + repo = "plugins"; + rev = "v${version}"; + sha256 = k3sCNISha256; + }; + + postInstall = '' + mv $out/bin/plugins $out/bin/cni + ''; + + meta = baseMeta // { + description = "CNI plugins, as patched by rancher for k3s"; + }; + }; + # Grab this separately from a build because it's used by both stages of the + # k3s build. + k3sRepo = fetchgit { + url = "https://github.com/k3s-io/k3s"; + rev = "v${k3sVersion}"; + sha256 = k3sRepoSha256; + }; + # Stage 1 of the k3s build: + # Let's talk about how k3s is structured. + # One of the ideas of k3s is that there's the single "k3s" binary which can + # do everything you need, from running a k3s server, to being a worker node, + # to running kubectl. + # The way that actually works is that k3s is a single go binary that contains + # a bunch of bindata that it unpacks at runtime into directories (either the + # user's home directory or /var/lib/rancher if run as root). + # This bindata includes both binaries and configuration. + # In order to let nixpkgs do all its autostripping/patching/etc, we split this into two derivations. + # First, we build all the binaries that get packed into the thick k3s binary + # (and output them from one derivation so they'll all be suitably patched up). + # Then, we bundle those binaries into our thick k3s binary and use that as + # the final single output. + # This approach was chosen because it ensures the bundled binaries all are + # correctly built to run with nix (we can lean on the existing buildGoModule + # stuff), and we can again lean on that tooling for the final k3s binary too. + # Other alternatives would be to manually run the + # strip/patchelf/remove-references step ourselves in the installPhase of the + # derivation when we've built all the binaries, but haven't bundled them in + # with generated bindata yet. + + k3sServer = buildGoModule { + pname = "k3s-server"; + version = k3sVersion; + + src = k3sRepo; + vendorSha256 = k3sVendorSha256; + + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ libseccomp sqlite.dev ]; + + subPackages = [ "cmd/server" ]; + ldflags = versionldflags; + + tags = [ "libsqlite3" "linux" ]; + + # create the multicall symlinks for k3s + postInstall = '' + mv $out/bin/server $out/bin/k3s + pushd $out + # taken verbatim from https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/build#L105-L113 + ln -s k3s ./bin/k3s-agent + ln -s k3s ./bin/k3s-server + ln -s k3s ./bin/k3s-etcd-snapshot + ln -s k3s ./bin/k3s-secrets-encrypt + ln -s k3s ./bin/k3s-certificate + ln -s k3s ./bin/kubectl + ln -s k3s ./bin/crictl + ln -s k3s ./bin/ctr + popd + ''; + + meta = baseMeta // { + description = "The various binaries that get packaged into the final k3s binary"; + }; + }; + k3sContainerd = buildGoModule { + pname = "k3s-containerd"; + version = containerdVersion; + src = fetchFromGitHub { + owner = "k3s-io"; + repo = "containerd"; + rev = "v${containerdVersion}"; + sha256 = containerdSha256; + }; + vendorSha256 = null; + buildInputs = [ btrfs-progs ]; + subPackages = [ "cmd/containerd" "cmd/containerd-shim-runc-v2" ]; + ldflags = versionldflags; + }; +in +buildGoModule rec { + pname = "k3s"; + version = k3sVersion; + + src = k3sRepo; + vendorSha256 = k3sVendorSha256; + + postPatch = '' + # Nix prefers dynamically linked binaries over static binary. + + substituteInPlace scripts/package-cli \ + --replace '"$LDFLAGS $STATIC" -o' \ + '"$LDFLAGS" -o' \ + --replace "STATIC=\"-extldflags \'-static\'\"" \ + "" + + # Upstream codegen fails with trimpath set. Removes "trimpath" for 'go generate': + + substituteInPlace scripts/package-cli \ + --replace '"''${GO}" generate' \ + 'GOFLAGS="" \ + GOOS="${pkgsBuildBuild.go.GOOS}" \ + GOARCH="${pkgsBuildBuild.go.GOARCH}" \ + CC="${pkgsBuildBuild.stdenv.cc}/bin/cc" \ + "''${GO}" generate' + ''; + + # Important utilities used by the kubelet, see + # https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494 + # Note the list in that issue is stale and some aren't relevant for k3s. + k3sRuntimeDeps = [ + kmod + socat + iptables + iproute2 + ipset + bridge-utils + ethtool + util-linux # kubelet wants 'nsenter' from util-linux: https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-705994388 + conntrack-tools + ]; + + buildInputs = k3sRuntimeDeps; + + nativeBuildInputs = [ + makeWrapper + rsync + yq-go + zstd + ]; + + # embedded in the final k3s cli + propagatedBuildInputs = [ + k3sCNIPlugins + k3sContainerd + k3sServer + runc + ]; + + # We override most of buildPhase due to peculiarities in k3s's build. + # Specifically, it has a 'go generate' which runs part of the package. See + # this comment: + # https://github.com/NixOS/nixpkgs/pull/158089#discussion_r799965694 + # So, why do we use buildGoModule at all? For the `vendorSha256` / `go mod download` stuff primarily. + buildPhase = '' + patchShebangs ./scripts/package-cli ./scripts/download ./scripts/build-upload + + # copy needed 'go generate' inputs into place + mkdir -p ./bin/aux + rsync -a --no-perms ${k3sServer}/bin/ ./bin/ + ln -vsf ${runc}/bin/runc ./bin/runc + ln -vsf ${k3sCNIPlugins}/bin/cni ./bin/cni + ln -vsf ${k3sContainerd}/bin/* ./bin/ + rsync -a --no-perms --chmod u=rwX ${k3sRoot}/etc/ ./etc/ + mkdir -p ./build/static/charts + + cp ${traefikChart} ./build/static/charts + cp ${traefik-crdChart} ./build/static/charts + + export ARCH=$GOARCH + export DRONE_TAG="v${k3sVersion}" + export DRONE_COMMIT="${k3sCommit}" + # use ./scripts/package-cli to run 'go generate' + 'go build' + + ./scripts/package-cli + mkdir -p $out/bin + ''; + + # Otherwise it depends on 'getGoDirs', which is normally set in buildPhase + doCheck = false; + + installPhase = '' + # wildcard to match the arm64 build too + install -m 0755 dist/artifacts/k3s* -D $out/bin/k3s + wrapProgram $out/bin/k3s \ + --prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \ + --prefix PATH : "$out/bin" + ''; + + doInstallCheck = true; + installCheckPhase = '' + $out/bin/k3s --version | grep -F "v${k3sVersion}" >/dev/null + ''; + + passthru.updateScript = updateScript; + + passthru.mkTests = version: + let k3s_version = "k3s_" + lib.replaceStrings ["."] ["_"] (lib.versions.majorMinor version); + in { + single-node = nixosTests.k3s.single-node.${k3s_version}; + multi-node = nixosTests.k3s.multi-node.${k3s_version}; + }; + passthru.tests = passthru.mkTests k3sVersion; + + + meta = baseMeta; +} diff --git a/pkgs/applications/networking/cluster/k3s/builders/1_26.nix b/pkgs/applications/networking/cluster/k3s/builders/1_26.nix deleted file mode 100644 index fa34782d39a2..000000000000 --- a/pkgs/applications/networking/cluster/k3s/builders/1_26.nix +++ /dev/null @@ -1,335 +0,0 @@ -{ lib -, makeWrapper -, socat -, iptables -, iproute2 -, ipset -, bridge-utils -, btrfs-progs -, conntrack-tools -, buildGoModule -, runc -, rsync -, kmod -, libseccomp -, pkg-config -, ethtool -, util-linux -, fetchFromGitHub -, fetchurl -, fetchzip -, fetchgit -, zstd -, yq-go -, sqlite -, nixosTests -, pkgsBuildBuild -}: - -{ - # git tag - k3sVersion, - # commit hash - k3sCommit, - k3sRepoSha256 ? lib.fakeHash, - k3sVendorSha256 ? lib.fakeHash, - # taken from ./scripts/version.sh VERSION_ROOT https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L47 - k3sRootVersion, - k3sRootSha256 ? lib.fakeHash, - # Based on the traefik charts here: https://github.com/k3s-io/k3s/blob/d71ab6317e22dd34673faa307a412a37a16767f6/scripts/download#L29-L32 - # see also https://github.com/k3s-io/k3s/blob/d71ab6317e22dd34673faa307a412a37a16767f6/manifests/traefik.yaml#L8 - chartVersions, - # taken from ./scripts/version.sh VERSION_CNIPLUGINS https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/version.sh#L45 - k3sCNIVersion, - k3sCNISha256 ? lib.fakeHash, - # taken from ./scripts/version.sh VERSION_CONTAINERD - containerdVersion, - containerdSha256 ? lib.fakeHash, - # run `grep github.com/kubernetes-sigs/cri-tools go.mod | head -n1 | awk '{print $4}'` in the k3s repo at the tag - criCtlVersion, - updateScript ? null, -}: - -# k3s is a kinda weird derivation. One of the main points of k3s is the -# simplicity of it being one binary that can perform several tasks. -# However, when you have a good package manager (like nix), that doesn't -# actually make much of a difference; you don't really care if it's one binary -# or 10 since with a good package manager, installing and running it is -# identical. -# Since upstream k3s packages itself as one large binary with several -# "personalities" (in the form of subcommands like 'k3s agent' and 'k3s -# kubectl'), it ends up being easiest to mostly mimic upstream packaging, with -# some exceptions. -# K3s also carries patches to some packages (such as containerd and cni -# plugins), so we intentionally use the k3s versions of those binaries for k3s, -# even if the upstream version of those binaries exist in nixpkgs already. In -# the end, that means we have a thick k3s binary that behaves like the upstream -# one for the most part. -# However, k3s also bundles several pieces of unpatched software, from the -# strongswan vpn software, to iptables, to socat, conntrack, busybox, etc. -# Those pieces of software we entirely ignore upstream's handling of, and just -# make sure they're in the path if desired. -let - - baseMeta = with lib; { - description = "A lightweight Kubernetes distribution"; - license = licenses.asl20; - homepage = "https://k3s.io"; - maintainers = with maintainers; [ euank mic92 yajo ]; - platforms = platforms.linux; - }; - - # https://github.com/k3s-io/k3s/blob/5fb370e53e0014dc96183b8ecb2c25a61e891e76/scripts/build#L19-L40 - versionldflags = [ - "-X github.com/rancher/k3s/pkg/version.Version=v${k3sVersion}" - "-X github.com/rancher/k3s/pkg/version.GitCommit=${lib.substring 0 8 k3sCommit}" - "-X k8s.io/client-go/pkg/version.gitVersion=v${k3sVersion}" - "-X k8s.io/client-go/pkg/version.gitCommit=${k3sCommit}" - "-X k8s.io/client-go/pkg/version.gitTreeState=clean" - "-X k8s.io/client-go/pkg/version.buildDate=1970-01-01T01:01:01Z" - "-X k8s.io/component-base/version.gitVersion=v${k3sVersion}" - "-X k8s.io/component-base/version.gitCommit=${k3sCommit}" - "-X k8s.io/component-base/version.gitTreeState=clean" - "-X k8s.io/component-base/version.buildDate=1970-01-01T01:01:01Z" - "-X github.com/kubernetes-sigs/cri-tools/pkg/version.Version=v${criCtlVersion}" - "-X github.com/containerd/containerd/version.Version=v${containerdVersion}" - "-X github.com/containerd/containerd/version.Package=github.com/k3s-io/containerd" - ]; - - # bundled into the k3s binary - traefikChart = fetchurl chartVersions.traefik; - traefik-crdChart = fetchurl chartVersions.traefik-crd; - - # so, k3s is a complicated thing to package - # This derivation attempts to avoid including any random binaries from the - # internet. k3s-root is _mostly_ binaries built to be bundled in k3s (which - # we don't care about doing, we can add those as build or runtime - # dependencies using a real package manager). - # In addition to those binaries, it's also configuration though (right now - # mostly strongswan configuration), and k3s does use those files. - # As such, we download it in order to grab 'etc' and bundle it into the final - # k3s binary. - k3sRoot = fetchzip { - # Note: marked as apache 2.0 license - url = "https://github.com/k3s-io/k3s-root/releases/download/v${k3sRootVersion}/k3s-root-amd64.tar"; - sha256 = k3sRootSha256; - stripRoot = false; - }; - k3sCNIPlugins = buildGoModule rec { - pname = "k3s-cni-plugins"; - version = k3sCNIVersion; - vendorSha256 = null; - - subPackages = [ "." ]; - - src = fetchFromGitHub { - owner = "rancher"; - repo = "plugins"; - rev = "v${version}"; - sha256 = k3sCNISha256; - }; - - postInstall = '' - mv $out/bin/plugins $out/bin/cni - ''; - - meta = baseMeta // { - description = "CNI plugins, as patched by rancher for k3s"; - }; - }; - # Grab this separately from a build because it's used by both stages of the - # k3s build. - k3sRepo = fetchgit { - url = "https://github.com/k3s-io/k3s"; - rev = "v${k3sVersion}"; - sha256 = k3sRepoSha256; - }; - # Stage 1 of the k3s build: - # Let's talk about how k3s is structured. - # One of the ideas of k3s is that there's the single "k3s" binary which can - # do everything you need, from running a k3s server, to being a worker node, - # to running kubectl. - # The way that actually works is that k3s is a single go binary that contains - # a bunch of bindata that it unpacks at runtime into directories (either the - # user's home directory or /var/lib/rancher if run as root). - # This bindata includes both binaries and configuration. - # In order to let nixpkgs do all its autostripping/patching/etc, we split this into two derivations. - # First, we build all the binaries that get packed into the thick k3s binary - # (and output them from one derivation so they'll all be suitably patched up). - # Then, we bundle those binaries into our thick k3s binary and use that as - # the final single output. - # This approach was chosen because it ensures the bundled binaries all are - # correctly built to run with nix (we can lean on the existing buildGoModule - # stuff), and we can again lean on that tooling for the final k3s binary too. - # Other alternatives would be to manually run the - # strip/patchelf/remove-references step ourselves in the installPhase of the - # derivation when we've built all the binaries, but haven't bundled them in - # with generated bindata yet. - - k3sServer = buildGoModule { - pname = "k3s-server"; - version = k3sVersion; - - src = k3sRepo; - vendorSha256 = k3sVendorSha256; - - nativeBuildInputs = [ pkg-config ]; - buildInputs = [ libseccomp sqlite.dev ]; - - subPackages = [ "cmd/server" ]; - ldflags = versionldflags; - - tags = [ "libsqlite3" "linux" ]; - - # create the multicall symlinks for k3s - postInstall = '' - mv $out/bin/server $out/bin/k3s - pushd $out - # taken verbatim from https://github.com/k3s-io/k3s/blob/v1.23.3%2Bk3s1/scripts/build#L105-L113 - ln -s k3s ./bin/k3s-agent - ln -s k3s ./bin/k3s-server - ln -s k3s ./bin/k3s-etcd-snapshot - ln -s k3s ./bin/k3s-secrets-encrypt - ln -s k3s ./bin/k3s-certificate - ln -s k3s ./bin/kubectl - ln -s k3s ./bin/crictl - ln -s k3s ./bin/ctr - popd - ''; - - meta = baseMeta // { - description = "The various binaries that get packaged into the final k3s binary"; - }; - }; - k3sContainerd = buildGoModule { - pname = "k3s-containerd"; - version = containerdVersion; - src = fetchFromGitHub { - owner = "k3s-io"; - repo = "containerd"; - rev = "v${containerdVersion}"; - sha256 = containerdSha256; - }; - vendorSha256 = null; - buildInputs = [ btrfs-progs ]; - subPackages = [ "cmd/containerd" "cmd/containerd-shim-runc-v2" ]; - ldflags = versionldflags; - }; -in -buildGoModule rec { - pname = "k3s"; - version = k3sVersion; - - src = k3sRepo; - vendorSha256 = k3sVendorSha256; - - postPatch = '' - # Nix prefers dynamically linked binaries over static binary. - - substituteInPlace scripts/package-cli \ - --replace '"$LDFLAGS $STATIC" -o' \ - '"$LDFLAGS" -o' \ - --replace "STATIC=\"-extldflags \'-static\'\"" \ - "" - - # Upstream codegen fails with trimpath set. Removes "trimpath" for 'go generate': - - substituteInPlace scripts/package-cli \ - --replace '"''${GO}" generate' \ - 'GOFLAGS="" \ - GOOS="${pkgsBuildBuild.go.GOOS}" \ - GOARCH="${pkgsBuildBuild.go.GOARCH}" \ - CC="${pkgsBuildBuild.stdenv.cc}/bin/cc" \ - "''${GO}" generate' - ''; - - # Important utilities used by the kubelet, see - # https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-237202494 - # Note the list in that issue is stale and some aren't relevant for k3s. - k3sRuntimeDeps = [ - kmod - socat - iptables - iproute2 - ipset - bridge-utils - ethtool - util-linux # kubelet wants 'nsenter' from util-linux: https://github.com/kubernetes/kubernetes/issues/26093#issuecomment-705994388 - conntrack-tools - ]; - - buildInputs = k3sRuntimeDeps; - - nativeBuildInputs = [ - makeWrapper - rsync - yq-go - zstd - ]; - - # embedded in the final k3s cli - propagatedBuildInputs = [ - k3sCNIPlugins - k3sContainerd - k3sServer - runc - ]; - - # We override most of buildPhase due to peculiarities in k3s's build. - # Specifically, it has a 'go generate' which runs part of the package. See - # this comment: - # https://github.com/NixOS/nixpkgs/pull/158089#discussion_r799965694 - # So, why do we use buildGoModule at all? For the `vendorSha256` / `go mod download` stuff primarily. - buildPhase = '' - patchShebangs ./scripts/package-cli ./scripts/download ./scripts/build-upload - - # copy needed 'go generate' inputs into place - mkdir -p ./bin/aux - rsync -a --no-perms ${k3sServer}/bin/ ./bin/ - ln -vsf ${runc}/bin/runc ./bin/runc - ln -vsf ${k3sCNIPlugins}/bin/cni ./bin/cni - ln -vsf ${k3sContainerd}/bin/* ./bin/ - rsync -a --no-perms --chmod u=rwX ${k3sRoot}/etc/ ./etc/ - mkdir -p ./build/static/charts - - cp ${traefikChart} ./build/static/charts - cp ${traefik-crdChart} ./build/static/charts - - export ARCH=$GOARCH - export DRONE_TAG="v${k3sVersion}" - export DRONE_COMMIT="${k3sCommit}" - # use ./scripts/package-cli to run 'go generate' + 'go build' - - ./scripts/package-cli - mkdir -p $out/bin - ''; - - # Otherwise it depends on 'getGoDirs', which is normally set in buildPhase - doCheck = false; - - installPhase = '' - # wildcard to match the arm64 build too - install -m 0755 dist/artifacts/k3s* -D $out/bin/k3s - wrapProgram $out/bin/k3s \ - --prefix PATH : ${lib.makeBinPath k3sRuntimeDeps} \ - --prefix PATH : "$out/bin" - ''; - - doInstallCheck = true; - installCheckPhase = '' - $out/bin/k3s --version | grep -F "v${k3sVersion}" >/dev/null - ''; - - passthru.updateScript = updateScript; - - passthru.mkTests = version: - let k3s_version = "k3s_" + lib.replaceStrings ["."] ["_"] (lib.versions.majorMinor version); - in { - single-node = nixosTests.k3s.single-node.${k3s_version}; - multi-node = nixosTests.k3s.multi-node.${k3s_version}; - }; - passthru.tests = passthru.mkTests k3sVersion; - - - meta = baseMeta; -} diff --git a/pkgs/applications/networking/cluster/k3s/default.nix b/pkgs/applications/networking/cluster/k3s/default.nix index 5a6b68eb5a9a..d8ecd79b5cae 100644 --- a/pkgs/applications/networking/cluster/k3s/default.nix +++ b/pkgs/applications/networking/cluster/k3s/default.nix @@ -1,11 +1,10 @@ { lib, stdenv, callPackage }: let - # The 1_26 builder can compile 1.26, and hopefully newer versions - k3s_1_26_builder = import ./builders/1_26.nix; + k3s_builder = import ./builder.nix; in { - k3s_1_26 = (callPackage k3s_1_26_builder { }) { + k3s_1_26 = (callPackage k3s_builder { }) { k3sVersion = "1.26.4+k3s1"; k3sCommit = "8d0255af07e95b841952563253d27b0d10bd72f0"; k3sRepoSha256 = "0qlszdnlsvj3hzx2p0wl3zhaw908w8a62z6vlf2g69a3c75f55cs"; @@ -21,7 +20,7 @@ in }; # 1_27 can be built with the same builder as 1_26 - k3s_1_27 = (callPackage k3s_1_26_builder { }) (import ./1_27/versions.nix) // { + k3s_1_27 = (callPackage k3s_builder { }) (import ./1_27/versions.nix) // { updateScript = ./1_27/update-script.sh; }; } -- cgit 1.4.1