Merge pull request #17106 from spacefrogg/openafs

openafs: Refactor and introduce server module

4 files changed, 527 insertions, 99 deletions

diff --git a/nixos/modules/services/network-filesystems/openafs-client/default.nix b/nixos/modules/services/network-filesystems/openafs-client/default.nix
deleted file mode 100644
index 0946e379e796..000000000000
--- a/nixos/modules/services/network-filesystems/openafs-client/default.nix
+++ /dev/null
@@ -1,99 +0,0 @@
-{ config, pkgs, lib, ... }:
-
-let
-  inherit (lib) mkOption mkIf;
-
-  cfg = config.services.openafsClient;
-
-  cellServDB = pkgs.fetchurl {
-    url = http://dl.central.org/dl/cellservdb/CellServDB.2017-03-14;
-    sha256 = "1197z6c5xrijgf66rhaymnm5cvyg2yiy1i20y4ah4mrzmjx0m7sc";
-  };
-
-  afsConfig = pkgs.runCommand "afsconfig" {} ''
-    mkdir -p $out
-    echo ${cfg.cellName} > $out/ThisCell
-    cp ${cellServDB} $out/CellServDB
-    echo "/afs:${cfg.cacheDirectory}:${cfg.cacheSize}" > $out/cacheinfo
-  '';
-
-  openafsPkgs = config.boot.kernelPackages.openafsClient;
-in
-{
-  ###### interface
-
-  options = {
-
-    services.openafsClient = {
-
-      enable = mkOption {
-        default = false;
-        description = "Whether to enable the OpenAFS client.";
-      };
-
-      cellName = mkOption {
-        default = "grand.central.org";
-        description = "Cell name.";
-      };
-
-      cacheSize = mkOption {
-        default = "100000";
-        description = "Cache size.";
-      };
-
-      cacheDirectory = mkOption {
-        default = "/var/cache/openafs";
-        description = "Cache directory.";
-      };
-
-      crypt = mkOption {
-        default = false;
-        description = "Whether to enable (weak) protocol encryption.";
-      };
-
-      sparse = mkOption {
-        default = false;
-        description = "Minimal cell list in /afs.";
-      };
-
-    };
-  };
-
-
-  ###### implementation
-
-  config = mkIf cfg.enable {
-
-    environment.systemPackages = [ openafsPkgs ];
-
-    environment.etc = [
-      { source = afsConfig;
-        target = "openafs";
-      }
-    ];
-
-    systemd.services.afsd = {
-      description = "AFS client";
-      wantedBy = [ "multi-user.target" ];
-      after = [ "network.target" ];
-      serviceConfig = { RemainAfterExit = true; };
-
-      preStart = ''
-        mkdir -p -m 0755 /afs
-        mkdir -m 0700 -p ${cfg.cacheDirectory}
-        ${pkgs.kmod}/bin/insmod ${openafsPkgs}/lib/openafs/libafs-*.ko || true
-        ${openafsPkgs}/sbin/afsd -confdir ${afsConfig} -cachedir ${cfg.cacheDirectory} ${if cfg.sparse then "-dynroot-sparse" else "-dynroot"} -fakestat -afsdb
-        ${openafsPkgs}/bin/fs setcrypt ${if cfg.crypt then "on" else "off"}
-      '';
-
-      # Doing this in preStop, because after these commands AFS is basically
-      # stopped, so systemd has nothing to do, just noticing it.  If done in
-      # postStop, then we get a hang + kernel oops, because AFS can't be
-      # stopped simply by sending signals to processes.
-      preStop = ''
-        ${pkgs.utillinux}/bin/umount /afs
-        ${openafsPkgs}/sbin/afsd -shutdown
-      '';
-    };
-  };
-}
diff --git a/nixos/modules/services/network-filesystems/openafs/client.nix b/nixos/modules/services/network-filesystems/openafs/client.nix
new file mode 100644
index 000000000000..3826fe3edfd0
--- /dev/null
+++ b/nixos/modules/services/network-filesystems/openafs/client.nix
@@ -0,0 +1,239 @@
+{ config, pkgs, lib, ... }:
+
+with import ./lib.nix { inherit lib; };
+
+let
+  inherit (lib) getBin mkOption mkIf optionalString singleton types;
+
+  cfg = config.services.openafsClient;
+
+  cellServDB = pkgs.fetchurl {
+    url = http://dl.central.org/dl/cellservdb/CellServDB.2017-03-14;
+    sha256 = "1197z6c5xrijgf66rhaymnm5cvyg2yiy1i20y4ah4mrzmjx0m7sc";
+  };
+
+  clientServDB = pkgs.writeText "client-cellServDB-${cfg.cellName}" (mkCellServDB cfg.cellName cfg.cellServDB);
+
+  afsConfig = pkgs.runCommand "afsconfig" {} ''
+    mkdir -p $out
+    echo ${cfg.cellName} > $out/ThisCell
+    cat ${cellServDB} ${clientServDB} > $out/CellServDB
+    echo "${cfg.mountPoint}:${cfg.cache.directory}:${toString cfg.cache.blocks}" > $out/cacheinfo
+  '';
+
+  openafsMod = config.boot.kernelPackages.openafs;
+  openafsBin = lib.getBin pkgs.openafs;
+in
+{
+  ###### interface
+
+  options = {
+
+    services.openafsClient = {
+
+      enable = mkOption {
+        default = false;
+        type = types.bool;
+        description = "Whether to enable the OpenAFS client.";
+      };
+
+      afsdb = mkOption {
+        default = true;
+        type = types.bool;
+        description = "Resolve cells via AFSDB DNS records.";
+      };
+
+      cellName = mkOption {
+        default = "";
+        type = types.str;
+        description = "Cell name.";
+        example = "grand.central.org";
+      };
+
+      cellServDB = mkOption {
+        default = [];
+        type = with types; listOf (submodule { options = cellServDBConfig; });
+        description = ''
+          This cell's database server records, added to the global
+          CellServDB. See CellServDB(5) man page for syntax. Ignored when
+          <literal>afsdb</literal> is set to <literal>true</literal>.
+        '';
+        example = ''
+          [ { ip = "1.2.3.4"; dnsname = "first.afsdb.server.dns.fqdn.org"; }
+            { ip = "2.3.4.5"; dnsname = "second.afsdb.server.dns.fqdn.org"; }
+          ]
+        '';
+      };
+
+      cache = {
+        blocks = mkOption {
+          default = 100000;
+          type = types.int;
+          description = "Cache size in 1KB blocks.";
+        };
+
+        chunksize = mkOption {
+          default = 0;
+          type = types.ints.between 0 30;
+          description = ''
+            Size of each cache chunk given in powers of
+            2. <literal>0</literal> resets the chunk size to its default
+            values (13 (8 KB) for memcache, 18-20 (256 KB to 1 MB) for
+            diskcache). Maximum value is 30. Important performance
+            parameter. Set to higher values when dealing with large files.
+          '';
+        };
+
+        directory = mkOption {
+          default = "/var/cache/openafs";
+          type = types.str;
+          description = "Cache directory.";
+        };
+
+        diskless = mkOption {
+          default = false;
+          type = types.bool;
+          description = ''
+            Use in-memory cache for diskless machines. Has no real
+            performance benefit anymore.
+          '';
+        };
+      };
+
+      crypt = mkOption {
+        default = true;
+        type = types.bool;
+        description = "Whether to enable (weak) protocol encryption.";
+      };
+
+      daemons = mkOption {
+        default = 2;
+        type = types.int;
+        description = ''
+          Number of daemons to serve user requests. Numbers higher than 6
+          usually do no increase performance. Default is sufficient for up
+          to five concurrent users.
+        '';
+      };
+
+      fakestat = mkOption {
+        default = false;
+        type = types.bool;
+        description = ''
+          Return fake data on stat() calls. If <literal>true</literal>,
+          always do so. If <literal>false</literal>, only do so for
+          cross-cell mounts (as these are potentially expensive).
+        '';
+      };
+
+      inumcalc = mkOption {
+        default = "compat";
+        type = types.strMatching "compat|md5";
+        description = ''
+          Inode calculation method. <literal>compat</literal> is
+          computationally less expensive, but <literal>md5</literal> greatly
+          reduces the likelihood of inode collisions in larger scenarios
+          involving multiple cells mounted into one AFS space.
+        '';
+      };
+
+      mountPoint = mkOption {
+        default = "/afs";
+        type = types.str;
+        description = ''
+          Mountpoint of the AFS file tree, conventionally
+          <literal>/afs</literal>. When set to a different value, only
+          cross-cells that use the same value can be accessed.
+        '';
+      };
+
+      sparse = mkOption {
+        default = true;
+        type = types.bool;
+        description = "Minimal cell list in /afs.";
+      };
+
+      startDisconnected = mkOption {
+        default = false;
+        type = types.bool;
+        description = ''
+          Start up in disconnected mode.  You need to execute
+          <literal>fs disco online</literal> (as root) to switch to
+          connected mode. Useful for roaming devices.
+        '';
+      };
+
+    };
+  };
+
+
+  ###### implementation
+
+  config = mkIf cfg.enable {
+
+    assertions = [
+      { assertion = cfg.afsdb || cfg.cellServDB != [];
+        message = "You should specify all cell-local database servers in config.services.openafsClient.cellServDB or set config.services.openafsClient.afsdb.";
+      }
+      { assertion = cfg.cellName != "";
+        message = "You must specify the local cell name in config.services.openafsClient.cellName.";
+      }
+    ];
+
+    environment.systemPackages = [ pkgs.openafs ];
+
+    environment.etc = {
+      clientCellServDB = {
+        source = pkgs.runCommand "CellServDB" {} ''
+          cat ${cellServDB} ${clientServDB} > $out
+        '';
+        target = "openafs/CellServDB";
+        mode = "0644";
+      };
+      clientCell = {
+        text = ''
+          ${cfg.cellName}
+        '';
+        target = "openafs/ThisCell";
+        mode = "0644";
+      };
+    };
+
+    systemd.services.afsd = {
+      description = "AFS client";
+      wantedBy = [ "multi-user.target" ];
+      after = singleton (if cfg.startDisconnected then  "network.target" else "network-online.target");
+      serviceConfig = { RemainAfterExit = true; };
+      restartIfChanged = false;
+
+      preStart = ''
+        mkdir -p -m 0755 ${cfg.mountPoint}
+        mkdir -m 0700 -p ${cfg.cache.directory}
+        ${pkgs.kmod}/bin/insmod ${openafsMod}/lib/modules/*/extra/openafs/libafs.ko.xz
+        ${openafsBin}/sbin/afsd \
+          -mountdir ${cfg.mountPoint} \
+          -confdir ${afsConfig} \
+          ${optionalString (!cfg.cache.diskless) "-cachedir ${cfg.cache.directory}"} \
+          -blocks ${toString cfg.cache.blocks} \
+          -chunksize ${toString cfg.cache.chunksize} \
+          ${optionalString cfg.cache.diskless "-memcache"} \
+          -inumcalc ${cfg.inumcalc} \
+          ${if cfg.fakestat then "-fakestat-all" else "-fakestat"} \
+          ${if cfg.sparse then "-dynroot-sparse" else "-dynroot"} \
+          ${optionalString cfg.afsdb "-afsdb"}
+        ${openafsBin}/bin/fs setcrypt ${if cfg.crypt then "on" else "off"}
+        ${optionalString cfg.startDisconnected "${openafsBin}/bin/fs discon offline"}
+      '';
+
+      # Doing this in preStop, because after these commands AFS is basically
+      # stopped, so systemd has nothing to do, just noticing it.  If done in
+      # postStop, then we get a hang + kernel oops, because AFS can't be
+      # stopped simply by sending signals to processes.
+      preStop = ''
+        ${pkgs.utillinux}/bin/umount ${cfg.mountPoint}
+        ${openafsBin}/sbin/afsd -shutdown
+        ${pkgs.kmod}/sbin/rmmod libafs
+      '';
+    };
+  };
+}
diff --git a/nixos/modules/services/network-filesystems/openafs/lib.nix b/nixos/modules/services/network-filesystems/openafs/lib.nix
new file mode 100644
index 000000000000..ecfc72d2eaf9
--- /dev/null
+++ b/nixos/modules/services/network-filesystems/openafs/lib.nix
@@ -0,0 +1,28 @@
+{ lib, ...}:
+
+let
+  inherit (lib) concatStringsSep mkOption types;
+
+in rec {
+
+  mkCellServDB = cellName: db: ''
+    >${cellName}
+  '' + (concatStringsSep "\n" (map (dbm: if (dbm.ip != "" && dbm.dnsname != "") then dbm.ip + " #" + dbm.dnsname else "")
+                                   db));
+
+  # CellServDB configuration type
+  cellServDBConfig = {
+    ip = mkOption {
+      type = types.str;
+      default = "";
+      example = "1.2.3.4";
+      description = "IP Address of a database server";
+    };
+    dnsname = mkOption {
+      type = types.str;
+      default = "";
+      example = "afs.example.org";
+      description = "DNS full-qualified domain name of a database server";
+    };
+  };
+}
diff --git a/nixos/modules/services/network-filesystems/openafs/server.nix b/nixos/modules/services/network-filesystems/openafs/server.nix
new file mode 100644
index 000000000000..429eb945ac9e
--- /dev/null
+++ b/nixos/modules/services/network-filesystems/openafs/server.nix
@@ -0,0 +1,260 @@
+{ config, pkgs, lib, ... }:
+
+with import ./lib.nix { inherit lib; };
+
+let
+  inherit (lib) concatStringsSep intersperse mapAttrsToList mkForce mkIf mkMerge mkOption optionalString types;
+
+  bosConfig = pkgs.writeText "BosConfig" (''
+    restrictmode 1
+    restarttime 16 0 0 0 0
+    checkbintime 3 0 5 0 0
+  '' + (optionalString cfg.roles.database.enable ''
+    bnode simple vlserver 1
+    parm ${openafsBin}/libexec/openafs/vlserver ${optionalString cfg.dottedPrincipals "-allow-dotted-principals"} ${cfg.roles.database.vlserverArgs}
+    end
+    bnode simple ptserver 1
+    parm ${openafsBin}/libexec/openafs/ptserver ${optionalString cfg.dottedPrincipals "-allow-dotted-principals"} ${cfg.roles.database.ptserverArgs}
+    end
+  '') + (optionalString cfg.roles.fileserver.enable ''
+    bnode dafs dafs 1
+    parm ${openafsBin}/libexec/openafs/dafileserver ${optionalString cfg.dottedPrincipals "-allow-dotted-principals"} -udpsize ${udpSizeStr} ${cfg.roles.fileserver.fileserverArgs}
+    parm ${openafsBin}/libexec/openafs/davolserver ${optionalString cfg.dottedPrincipals "-allow-dotted-principals"} -udpsize ${udpSizeStr} ${cfg.roles.fileserver.volserverArgs}
+    parm ${openafsBin}/libexec/openafs/salvageserver ${cfg.roles.fileserver.salvageserverArgs}
+    parm ${openafsBin}/libexec/openafs/dasalvager ${cfg.roles.fileserver.salvagerArgs}
+    end
+  '') + (optionalString (cfg.roles.database.enable && cfg.roles.backup.enable) ''
+    bnode simple buserver 1
+    parm ${openafsBin}/libexec/openafs/buserver ${cfg.roles.backup.buserverArgs} ${optionalString (cfg.roles.backup.cellServDB != []) "-cellservdb /etc/openafs/backup/"}
+    end
+  ''));
+
+  netInfo = if (cfg.advertisedAddresses != []) then
+    pkgs.writeText "NetInfo" ((concatStringsSep "\nf " cfg.advertisedAddresses) + "\n")
+  else null;
+
+  buCellServDB = pkgs.writeText "backup-cellServDB-${cfg.cellName}" (mkCellServDB cfg.cellName cfg.roles.backup.cellServDB);
+
+  cfg = config.services.openafsServer;
+
+  udpSizeStr = toString cfg.udpPacketSize;
+
+  openafsBin = lib.getBin pkgs.openafs;
+
+in {
+
+  options = {
+
+    services.openafsServer = {
+
+      enable = mkOption {
+        default = false;
+        type = types.bool;
+        description = ''
+          Whether to enable the OpenAFS server. An OpenAFS server needs a
+          complex setup. So, be aware that enabling this service and setting
+          some options does not give you a turn-key-ready solution. You need
+          at least a running Kerberos 5 setup, as OpenAFS relies on it for
+          authentication. See the Guide "QuickStartUnix" coming with
+          <literal>pkgs.openafs.doc</literal> for complete setup
+          instructions.
+        '';
+      };
+
+      advertisedAddresses = mkOption {
+        default = [];
+        description = "List of IP addresses this server is advertised under. See NetInfo(5)";
+      };
+
+      cellName = mkOption {
+        default = "";
+        type = types.str;
+        description = "Cell name, this server will serve.";
+        example = "grand.central.org";
+      };
+
+      cellServDB = mkOption {
+        default = [];
+        type = with types; listOf (submodule [ { options = cellServDBConfig;} ]);
+        description = "Definition of all cell-local database server machines.";
+      };
+
+      roles = {
+        fileserver = {
+          enable = mkOption {
+            default = true;
+            type = types.bool;
+            description = "Fileserver role, serves files and volumes from its local storage.";
+          };
+
+          fileserverArgs = mkOption {
+            default = "-vattachpar 128 -vhashsize 11 -L -rxpck 400 -cb 1000000";
+            type = types.str;
+            description = "Arguments to the dafileserver process. See its man page.";
+          };
+
+          volserverArgs = mkOption {
+            default = "";
+            type = types.str;
+            description = "Arguments to the davolserver process. See its man page.";
+            example = "-sync never";
+          };
+
+          salvageserverArgs = mkOption {
+            default = "";
+            type = types.str;
+            description = "Arguments to the salvageserver process. See its man page.";
+            example = "-showlog";
+          };
+
+          salvagerArgs = mkOption {
+            default = "";
+            type = types.str;
+            description = "Arguments to the dasalvager process. See its man page.";
+            example = "-showlog -showmounts";
+          };
+        };
+
+        database = {
+          enable = mkOption {
+            default = true;
+            type = types.bool;
+            description = ''
+              Database server role, maintains the Volume Location Database,
+              Protection Database (and Backup Database, see
+              <literal>backup</literal> role). There can be multiple
+              servers in the database role for replication, which then need
+              reliable network connection to each other.
+
+              Servers in this role appear in AFSDB DNS records or the
+              CellServDB.
+            '';
+          };
+
+          vlserverArgs = mkOption {
+            default = "";
+            type = types.str;
+            description = "Arguments to the vlserver process. See its man page.";
+            example = "-rxbind";
+          };
+
+          ptserverArgs = mkOption {
+            default = "";
+            type = types.str;
+            description = "Arguments to the ptserver process. See its man page.";
+            example = "-restricted -default_access S---- S-M---";
+          };
+        };
+
+        backup = {
+          enable = mkOption {
+            default = false;
+            type = types.bool;
+            description = ''
+              Backup server role. Use in conjunction with the
+              <literal>database</literal> role to maintain the Backup
+              Database. Normally only used in conjunction with tape storage
+              or IBM's Tivoli Storage Manager.
+            '';
+          };
+
+          buserverArgs = mkOption {
+            default = "";
+            type = types.str;
+            description = "Arguments to the buserver process. See its man page.";
+            example = "-p 8";
+          };
+
+          cellServDB = mkOption {
+            default = [];
+            type = with types; listOf (submodule [ { options = cellServDBConfig;} ]);
+            description = ''
+              Definition of all cell-local backup database server machines.
+              Use this when your cell uses less backup database servers than
+              other database server machines.
+            '';
+          };
+        };
+      };
+
+      dottedPrincipals= mkOption {
+        default = false;
+        type = types.bool;
+        description = ''
+          If enabled, allow principal names containing (.) dots. Enabling
+          this has security implications!
+        '';
+      };
+
+      udpPacketSize = mkOption {
+        default = 1310720;
+        type = types.int;
+        description = ''
+          UDP packet size to use in Bytes. Higher values can speed up
+          communications. The default of 1 MB is a sufficient in most
+          cases. Make sure to increase the kernel's UDP buffer size
+          accordingly via <literal>net.core(w|r|opt)mem_max</literal>
+          sysctl.
+        '';
+      };
+
+    };
+
+  };
+
+  config = mkIf cfg.enable {
+
+    assertions = [
+      { assertion = cfg.cellServDB != [];
+        message = "You must specify all cell-local database servers in config.services.openafsServer.cellServDB.";
+      }
+      { assertion = cfg.cellName != "";
+        message = "You must specify the local cell name in config.services.openafsServer.cellName.";
+      }
+    ];
+
+    environment.systemPackages = [ pkgs.openafs ];
+
+    environment.etc = {
+      bosConfig = {
+        source = bosConfig;
+        target = "openafs/BosConfig";
+        mode = "0644";
+      };
+      cellServDB = {
+        text = mkCellServDB cfg.cellName cfg.cellServDB;
+        target = "openafs/server/CellServDB";
+        mode = "0644";
+      };
+      thisCell = {
+        text = cfg.cellName;
+        target = "openafs/server/ThisCell";
+        mode = "0644";
+      };
+      buCellServDB = {
+        enable = (cfg.roles.backup.cellServDB != []);
+        text = mkCellServDB cfg.cellName cfg.roles.backup.cellServDB;
+        target = "openafs/backup/CellServDB";
+      };
+    };
+
+    systemd.services = {
+      openafs-server = {
+        description = "OpenAFS server";
+        after = [ "syslog.target" "network.target" ];
+        wantedBy = [ "multi-user.target" ];
+        restartIfChanged = false;
+        unitConfig.ConditionPathExists = [ "/etc/openafs/server/rxkad.keytab" ];
+        preStart = ''
+          mkdir -m 0755 -p /var/openafs
+          ${optionalString (netInfo != null) "cp ${netInfo} /var/openafs/netInfo"}
+          ${optionalString (cfg.roles.backup.cellServDB != []) "cp ${buCellServDB}"}
+        '';
+        serviceConfig = {
+          ExecStart = "${openafsBin}/bin/bosserver -nofork";
+          ExecStop = "${openafsBin}/bin/bos shutdown localhost -wait -localauth";
+        };
+      };
+    };
+  };
+}