From b8c871475afd0e529f1e2bc6f56763d2b9487f25 Mon Sep 17 00:00:00 2001
From: Edward Tjörnhammar <ed@cflags.cc>
Date: Wed, 6 Sep 2023 13:39:04 +0200
Subject: nixos/infiniband: add support for configurable guids

---
 nixos/doc/manual/release-notes/rl-2311.section.md |  2 +
 nixos/modules/hardware/infiniband.nix             | 58 +++++++++++++++++++++++
 nixos/modules/module-list.nix                     |  1 +
 3 files changed, 61 insertions(+)
 create mode 100644 nixos/modules/hardware/infiniband.nix

(limited to 'nixos')

diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md
index 4d559b9ca40a..05fcff30b07d 100644
--- a/nixos/doc/manual/release-notes/rl-2311.section.md
+++ b/nixos/doc/manual/release-notes/rl-2311.section.md
@@ -56,6 +56,8 @@
 
 - [eris-server](https://codeberg.org/eris/eris-go). [ERIS](https://eris.codeberg.page/) is an encoding for immutable storage and this server provides block exchange as well as content decoding over HTTP and through a FUSE file-system. Available as [services.eris-server](#opt-services.eris-server.enable).
 
+- hardware/infiniband.nix adds infiniband subnet manager support using an [opensm](https://github.com/linux-rdma/opensm) systemd-template service, instantiated on card guids. The module also adds kernel modules and cli tooling to help administrators debug and measure performance. Available as [hardware.infiniband.enable](#opt-hardware.infiniband.enable).
+
 - [Honk](https://humungus.tedunangst.com/r/honk), a complete ActivityPub server with minimal setup and support costs.
   Available as [services.honk](#opt-services.honk.enable).
 
diff --git a/nixos/modules/hardware/infiniband.nix b/nixos/modules/hardware/infiniband.nix
new file mode 100644
index 000000000000..962883fa7972
--- /dev/null
+++ b/nixos/modules/hardware/infiniband.nix
@@ -0,0 +1,58 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+
+let
+
+  cfg = config.hardware.infiniband;
+  opensm-services = {
+    "opensm@" = {
+      enable = true;
+      description = "Starts OpenSM Infiniband fabric Subnet Managers";
+      before = [ "network.target"];
+      unitConfig = {
+        ConditionPathExists = "/sys/class/infiniband_mad/abi_version";
+      };
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = "${pkgs.opensm}/bin/opensm --guid %I --log_file /var/log/opensm.%I.log";
+      };
+    };
+  } // (builtins.listToAttrs (map (guid: {
+    name = "opensm@${guid}";
+    value = {
+      enable = true;
+      wantedBy = [ "machines.target" ];
+      overrideStrategy = "asDropin";
+    };
+  } ) cfg.guids));
+
+in
+
+{
+  options.hardware.infiniband = {
+    enable = mkEnableOption "Infiniband support";
+    guids = mkOption {
+      type = with types; listOf str;
+      default = [];
+      example = [ "0xe8ebd30000eee2e1" ];
+      description = lib.mdDoc ''
+        A list of infiniband port guids on the system. This is discoverable using `ibstat -p`
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    boot.initrd.kernelModules = [
+      "mlx5_core" "mlx5_ib" "ib_cm"
+      "rdma_cm" "rdma_ucm" "rpcrdma"
+      "ib_ipoib" "ib_isert" "ib_umad" "ib_uverbs"
+    ];
+    # rdma-core exposes ibstat, mstflint exposes mstconfig (which can be needed for
+    # setting link configurations), qperf needed to affirm link speeds
+    environment.systemPackages = with pkgs; [
+      rdma-core mstflint qperf
+    ];
+    systemd.services = opensm-services;
+  };
+}
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix
index b7582d4a6324..6ce8a2d88ab4 100644
--- a/nixos/modules/module-list.nix
+++ b/nixos/modules/module-list.nix
@@ -63,6 +63,7 @@
   ./hardware/gpgsmartcards.nix
   ./hardware/hackrf.nix
   ./hardware/i2c.nix
+  ./hardware/infiniband.nix
   ./hardware/keyboard/qmk.nix
   ./hardware/keyboard/teck.nix
   ./hardware/keyboard/uhk.nix
-- 
cgit 1.4.1