1 files changed, 58 insertions, 0 deletions
diff --git a/nixos/modules/hardware/infiniband.nix b/nixos/modules/hardware/infiniband.nix
new file mode 100644
index 000000000000..962883fa7972
--- /dev/null
+++ b/nixos/modules/hardware/infiniband.nix
@@ -0,0 +1,58 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+
+  cfg = config.hardware.infiniband;
+  opensm-services = {
+    "opensm@" = {
+      enable = true;
+      description = "Starts OpenSM Infiniband fabric Subnet Managers";
+      before = [ "network.target"];
+      unitConfig = {
+        ConditionPathExists = "/sys/class/infiniband_mad/abi_version";
+      };
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = "${pkgs.opensm}/bin/opensm --guid %I --log_file /var/log/opensm.%I.log";
+      };
+    };
+  } // (builtins.listToAttrs (map (guid: {
+    name = "opensm@${guid}";
+    value = {
+      enable = true;
+      wantedBy = [ "machines.target" ];
+      overrideStrategy = "asDropin";
+    };
+  } ) cfg.guids));
+
+in
+
+{
+  options.hardware.infiniband = {
+    enable = mkEnableOption "Infiniband support";
+    guids = mkOption {
+      type = with types; listOf str;
+      default = [];
+      example = [ "0xe8ebd30000eee2e1" ];
+      description = lib.mdDoc ''
+        A list of infiniband port guids on the system. This is discoverable using `ibstat -p`
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    boot.initrd.kernelModules = [
+      "mlx5_core" "mlx5_ib" "ib_cm"
+      "rdma_cm" "rdma_ucm" "rpcrdma"
+      "ib_ipoib" "ib_isert" "ib_umad" "ib_uverbs"
+    ];
+    # rdma-core exposes ibstat, mstflint exposes mstconfig (which can be needed for
+    # setting link configurations), qperf needed to affirm link speeds
+    environment.systemPackages = with pkgs; [
+      rdma-core mstflint qperf
+    ];
+    systemd.services = opensm-services;
+  };
+}