From 67e0777f6258ded0029bbfeec65fbedf36d6d4c8 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Sat, 2 Nov 2019 15:01:39 +0100 Subject: nixos/ceph: run unprivileged, use StateDirectory and tmpfiles, don't pass extraServiceConfig Don't pass user and group to ceph, and rely on it to drop ceps, but let systemd handle running it as the appropriate user. This also inlines the extraServiceConfig into the makeService function, as we have conditionals depending on daemonType there anyways. Use StateDirectory to create directories in /var/lib/ceph/${daemonType}/${clusterName}-${daemonId}. There previously was a condition on daemonType being one of mds,mon,rgw or mgr. We only instantiate makeServices with these types, and "osd" was special. In the osd case, test examples suggest it'd be in something like /var/lib/ceph/osd/ceph-${cfg.osd0.name} - so it's not special at all, but exactly like the pattern for the others. During initialization, we also need these folders, before the unit is started up. Move the mkdir -p commands in the vm tests to the line immediately before they're required. --- .../modules/services/network-filesystems/ceph.nix | 57 +++++++++++----------- 1 file changed, 29 insertions(+), 28 deletions(-) (limited to 'nixos/modules/services') diff --git a/nixos/modules/services/network-filesystems/ceph.nix b/nixos/modules/services/network-filesystems/ceph.nix index 887a6ff925d2..ef0d2d90a1a1 100644 --- a/nixos/modules/services/network-filesystems/ceph.nix +++ b/nixos/modules/services/network-filesystems/ceph.nix @@ -9,12 +9,12 @@ let expandCamelCase = replaceStrings upperChars (map (s: " ${s}") lowerChars); expandCamelCaseAttrs = mapAttrs' (name: value: nameValuePair (expandCamelCase name) value); - makeServices = (daemonType: daemonIds: extraServiceConfig: + makeServices = (daemonType: daemonIds: mkMerge (map (daemonId: - { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph extraServiceConfig; }) + { "ceph-${daemonType}-${daemonId}" = makeService daemonType daemonId cfg.global.clusterName pkgs.ceph; }) daemonIds)); - makeService = (daemonType: daemonId: clusterName: ceph: extraServiceConfig: { + makeService = (daemonType: daemonId: clusterName: ceph: { enable = true; description = "Ceph ${builtins.replaceStrings lowerChars upperChars daemonType} daemon ${daemonId}"; after = [ "network-online.target" "time-sync.target" ] ++ optional (daemonType == "osd") "ceph-mon.target"; @@ -22,6 +22,8 @@ let partOf = [ "ceph-${daemonType}.target" ]; wantedBy = [ "ceph-${daemonType}.target" ]; + path = [ pkgs.getopt ]; + serviceConfig = { LimitNOFILE = 1048576; LimitNPROC = 1048576; @@ -34,22 +36,22 @@ let Restart = "on-failure"; StartLimitBurst = "5"; StartLimitInterval = "30min"; + StateDirectory = "ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}"; + User = "ceph"; + Group = if daemonType == "osd" then "disk" else "ceph"; ExecStart = ''${ceph.out}/bin/${if daemonType == "rgw" then "radosgw" else "ceph-${daemonType}"} \ - -f --cluster ${clusterName} --id ${daemonId} --setuser ceph \ - --setgroup ${if daemonType == "osd" then "disk" else "ceph"}''; - } // extraServiceConfig - // optionalAttrs (daemonType == "osd") { ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh \ - --id ${daemonId} --cluster ${clusterName}''; }; - } // optionalAttrs (builtins.elem daemonType [ "mds" "mon" "rgw" "mgr" ]) { - preStart = '' - daemonPath="/var/lib/ceph/${if daemonType == "rgw" then "radosgw" else daemonType}/${clusterName}-${daemonId}" - if [ ! -d $daemonPath ]; then - mkdir -m 755 -p $daemonPath - chown -R ceph:ceph $daemonPath - fi - ''; - } // optionalAttrs (daemonType == "osd") { path = [ pkgs.getopt ]; } - ); + -f --cluster ${clusterName} --id ${daemonId}''; + } // optionalAttrs (daemonType == "osd") { + ExecStartPre = ''${ceph.lib}/libexec/ceph/ceph-osd-prestart.sh --id ${daemonId} --cluster ${clusterName}''; + StartLimitBurst = "30"; + RestartSec = "20s"; + PrivateDevices = "no"; # osd needs disk access + } // optionalAttrs ( daemonType == "mon") { + RestartSec = "10"; + } // optionalAttrs (lib.elem daemonType ["mgr" "mds"]) { + StartLimitBurst = "3"; + }; + }); makeTarget = (daemonType: { @@ -377,14 +379,11 @@ in systemd.services = let services = [] - ++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons { RestartSec = "10"; }) - ++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons { StartLimitBurst = "3"; }) - ++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons { StartLimitBurst = "30"; - RestartSec = "20s"; - PrivateDevices = "no"; # osd needs disk access - }) - ++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons { }) - ++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons { StartLimitBurst = "3"; }); + ++ optional cfg.mon.enable (makeServices "mon" cfg.mon.daemons) + ++ optional cfg.mds.enable (makeServices "mds" cfg.mds.daemons) + ++ optional cfg.osd.enable (makeServices "osd" cfg.osd.daemons) + ++ optional cfg.rgw.enable (makeServices "rgw" cfg.rgw.daemons) + ++ optional cfg.mgr.enable (makeServices "mgr" cfg.mgr.daemons); in mkMerge services; @@ -403,7 +402,9 @@ in systemd.tmpfiles.rules = [ "d /etc/ceph - ceph ceph - -" "d /run/ceph 0770 ceph ceph -" - "d /var/lib/ceph - ceph ceph - -" - ]; + "d /var/lib/ceph - ceph ceph - -"] + ++ optionals cfg.mgr.enable [ "d /var/lib/ceph/mgr - ceph ceph - -"] + ++ optionals cfg.mon.enable [ "d /var/lib/ceph/mon - ceph ceph - -"] + ++ optionals cfg.osd.enable [ "d /var/lib/ceph/osd - ceph ceph - -"]; }; } -- cgit 1.4.1