# Systemd services for lxd. { config, lib, pkgs, ... }: let cfg = config.virtualisation.lxd; preseedFormat = pkgs.formats.yaml {}; in { imports = [ (lib.mkRemovedOptionModule [ "virtualisation" "lxd" "zfsPackage" ] "Override zfs in an overlay instead to override it globally") ]; ###### interface options = { virtualisation.lxd = { enable = lib.mkOption { type = lib.types.bool; default = false; description = lib.mdDoc '' This option enables lxd, a daemon that manages containers. Users in the "lxd" group can interact with the daemon (e.g. to start or stop containers) using the {command}`lxc` command line tool, among others. Most of the time, you'll also want to start lxcfs, so that containers can "see" the limits: ``` virtualisation.lxc.lxcfs.enable = true; ``` ''; }; package = lib.mkOption { type = lib.types.package; default = pkgs.lxd; defaultText = lib.literalExpression "pkgs.lxd"; description = lib.mdDoc '' The LXD package to use. ''; }; lxcPackage = lib.mkOption { type = lib.types.package; default = pkgs.lxc; defaultText = lib.literalExpression "pkgs.lxc"; description = lib.mdDoc '' The LXC package to use with LXD (required for AppArmor profiles). ''; }; zfsSupport = lib.mkOption { type = lib.types.bool; default = config.boot.zfs.enabled; defaultText = lib.literalExpression "config.boot.zfs.enabled"; description = lib.mdDoc '' Enables lxd to use zfs as a storage for containers. This option is enabled by default if a zfs pool is configured with nixos. ''; }; recommendedSysctlSettings = lib.mkOption { type = lib.types.bool; default = false; description = lib.mdDoc '' Enables various settings to avoid common pitfalls when running containers requiring many file operations. Fixes errors like "Too many open files" or "neighbour: ndisc_cache: neighbor table overflow!". See https://lxd.readthedocs.io/en/latest/production-setup/ for details. ''; }; preseed = lib.mkOption { type = lib.types.nullOr (lib.types.submodule { freeformType = preseedFormat.type; }); default = null; description = lib.mdDoc '' Configuration for LXD preseed, see for supported values. Changes to this will be re-applied to LXD which will overwrite existing entities or create missing ones, but entities will *not* be removed by preseed. ''; example = lib.literalExpression '' { networks = [ { name = "lxdbr0"; type = "bridge"; config = { "ipv4.address" = "10.0.100.1/24"; "ipv4.nat" = "true"; }; } ]; profiles = [ { name = "default"; devices = { eth0 = { name = "eth0"; network = "lxdbr0"; type = "nic"; }; root = { path = "/"; pool = "default"; size = "35GiB"; type = "disk"; }; }; } ]; storage_pools = [ { name = "default"; driver = "dir"; config = { source = "/var/lib/lxd/storage-pools/default"; }; } ]; } ''; }; startTimeout = lib.mkOption { type = lib.types.int; default = 600; apply = toString; description = lib.mdDoc '' Time to wait (in seconds) for LXD to become ready to process requests. If LXD does not reply within the configured time, lxd.service will be considered failed and systemd will attempt to restart it. ''; }; ui = { enable = lib.mkEnableOption (lib.mdDoc "(experimental) LXD UI"); package = lib.mkPackageOption pkgs.lxd-unwrapped "ui" { }; }; }; }; ###### implementation config = lib.mkIf cfg.enable { environment.systemPackages = [ cfg.package ]; # Note: the following options are also declared in virtualisation.lxc, but # the latter can't be simply enabled to reuse the formers, because it # does a bunch of unrelated things. systemd.tmpfiles.rules = [ "d /var/lib/lxc/rootfs 0755 root root -" ]; security.apparmor = { packages = [ cfg.lxcPackage ]; policies = { "bin.lxc-start".profile = '' include ${cfg.lxcPackage}/etc/apparmor.d/usr.bin.lxc-start ''; "lxc-containers".profile = '' include ${cfg.lxcPackage}/etc/apparmor.d/lxc-containers ''; }; }; # TODO: remove once LXD gets proper support for cgroupsv2 # (currently most of the e.g. CPU accounting stuff doesn't work) systemd.enableUnifiedCgroupHierarchy = false; systemd.sockets.lxd = { description = "LXD UNIX socket"; wantedBy = [ "sockets.target" ]; socketConfig = { ListenStream = "/var/lib/lxd/unix.socket"; SocketMode = "0660"; SocketGroup = "lxd"; Service = "lxd.service"; }; }; systemd.services.lxd = { description = "LXD Container Management Daemon"; wantedBy = [ "multi-user.target" ]; after = [ "network-online.target" (lib.mkIf config.virtualisation.lxc.lxcfs.enable "lxcfs.service") ]; requires = [ "network-online.target" "lxd.socket" (lib.mkIf config.virtualisation.lxc.lxcfs.enable "lxcfs.service") ]; documentation = [ "man:lxd(1)" ]; path = [ pkgs.util-linux ] ++ lib.optional cfg.zfsSupport config.boot.zfs.package; environment = lib.mkIf (cfg.ui.enable) { "LXD_UI" = cfg.ui.package; }; serviceConfig = { ExecStart = "@${cfg.package}/bin/lxd lxd --group lxd"; ExecStartPost = "${cfg.package}/bin/lxd waitready --timeout=${cfg.startTimeout}"; ExecStop = "${cfg.package}/bin/lxd shutdown"; KillMode = "process"; # when stopping, leave the containers alone LimitMEMLOCK = "infinity"; LimitNOFILE = "1048576"; LimitNPROC = "infinity"; TasksMax = "infinity"; Restart = "on-failure"; TimeoutStartSec = "${cfg.startTimeout}s"; TimeoutStopSec = "30s"; # By default, `lxd` loads configuration files from hard-coded # `/usr/share/lxc/config` - since this is a no-go for us, we have to # explicitly tell it where the actual configuration files are Environment = lib.mkIf (config.virtualisation.lxc.lxcfs.enable) "LXD_LXC_TEMPLATE_CONFIG=${pkgs.lxcfs}/share/lxc/config"; }; }; systemd.services.lxd-preseed = lib.mkIf (cfg.preseed != null) { description = "LXD initialization with preseed file"; wantedBy = ["multi-user.target"]; requires = ["lxd.service"]; after = ["lxd.service"]; script = '' ${pkgs.coreutils}/bin/cat ${preseedFormat.generate "lxd-preseed.yaml" cfg.preseed} | ${cfg.package}/bin/lxd init --preseed ''; serviceConfig = { Type = "oneshot"; }; }; users.groups.lxd = {}; users.users.root = { subUidRanges = [ { startUid = 1000000; count = 65536; } ]; subGidRanges = [ { startGid = 1000000; count = 65536; } ]; }; boot.kernel.sysctl = lib.mkIf cfg.recommendedSysctlSettings { "fs.inotify.max_queued_events" = 1048576; "fs.inotify.max_user_instances" = 1048576; "fs.inotify.max_user_watches" = 1048576; "vm.max_map_count" = 262144; # TODO: Default vm.max_map_count has been increased system-wide "kernel.dmesg_restrict" = 1; "net.ipv4.neigh.default.gc_thresh3" = 8192; "net.ipv6.neigh.default.gc_thresh3" = 8192; "kernel.keys.maxkeys" = 2000; }; boot.kernelModules = [ "veth" "xt_comment" "xt_CHECKSUM" "xt_MASQUERADE" "vhost_vsock" ] ++ lib.optionals (!config.networking.nftables.enable) [ "iptable_mangle" ]; }; }