diff options
author | Edward Tjörnhammar <ed@cflags.cc> | 2023-09-06 13:39:04 +0200 |
---|---|---|
committer | Edward Tjörnhammar <ed@cflags.cc> | 2023-09-07 19:59:33 +0200 |
commit | b8c871475afd0e529f1e2bc6f56763d2b9487f25 (patch) | |
tree | 43a20cb45aa17c172364682fff8a5a7eb35e054b /nixos | |
parent | d5597ff02e360342b86e906368a05e9daa1d9cc9 (diff) | |
download | nixlib-b8c871475afd0e529f1e2bc6f56763d2b9487f25.tar nixlib-b8c871475afd0e529f1e2bc6f56763d2b9487f25.tar.gz nixlib-b8c871475afd0e529f1e2bc6f56763d2b9487f25.tar.bz2 nixlib-b8c871475afd0e529f1e2bc6f56763d2b9487f25.tar.lz nixlib-b8c871475afd0e529f1e2bc6f56763d2b9487f25.tar.xz nixlib-b8c871475afd0e529f1e2bc6f56763d2b9487f25.tar.zst nixlib-b8c871475afd0e529f1e2bc6f56763d2b9487f25.zip |
nixos/infiniband: add support for configurable guids
Diffstat (limited to 'nixos')
-rw-r--r-- | nixos/doc/manual/release-notes/rl-2311.section.md | 2 | ||||
-rw-r--r-- | nixos/modules/hardware/infiniband.nix | 58 | ||||
-rw-r--r-- | nixos/modules/module-list.nix | 1 |
3 files changed, 61 insertions, 0 deletions
diff --git a/nixos/doc/manual/release-notes/rl-2311.section.md b/nixos/doc/manual/release-notes/rl-2311.section.md index 4d559b9ca40a..05fcff30b07d 100644 --- a/nixos/doc/manual/release-notes/rl-2311.section.md +++ b/nixos/doc/manual/release-notes/rl-2311.section.md @@ -56,6 +56,8 @@ - [eris-server](https://codeberg.org/eris/eris-go). [ERIS](https://eris.codeberg.page/) is an encoding for immutable storage and this server provides block exchange as well as content decoding over HTTP and through a FUSE file-system. Available as [services.eris-server](#opt-services.eris-server.enable). +- hardware/infiniband.nix adds infiniband subnet manager support using an [opensm](https://github.com/linux-rdma/opensm) systemd-template service, instantiated on card guids. The module also adds kernel modules and cli tooling to help administrators debug and measure performance. Available as [hardware.infiniband.enable](#opt-hardware.infiniband.enable). + - [Honk](https://humungus.tedunangst.com/r/honk), a complete ActivityPub server with minimal setup and support costs. Available as [services.honk](#opt-services.honk.enable). diff --git a/nixos/modules/hardware/infiniband.nix b/nixos/modules/hardware/infiniband.nix new file mode 100644 index 000000000000..962883fa7972 --- /dev/null +++ b/nixos/modules/hardware/infiniband.nix @@ -0,0 +1,58 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + + cfg = config.hardware.infiniband; + opensm-services = { + "opensm@" = { + enable = true; + description = "Starts OpenSM Infiniband fabric Subnet Managers"; + before = [ "network.target"]; + unitConfig = { + ConditionPathExists = "/sys/class/infiniband_mad/abi_version"; + }; + serviceConfig = { + Type = "simple"; + ExecStart = "${pkgs.opensm}/bin/opensm --guid %I --log_file /var/log/opensm.%I.log"; + }; + }; + } // (builtins.listToAttrs (map (guid: { + name = "opensm@${guid}"; + value = { + enable = true; + wantedBy = [ "machines.target" ]; + overrideStrategy = "asDropin"; + }; + } ) cfg.guids)); + +in + +{ + options.hardware.infiniband = { + enable = mkEnableOption "Infiniband support"; + guids = mkOption { + type = with types; listOf str; + default = []; + example = [ "0xe8ebd30000eee2e1" ]; + description = lib.mdDoc '' + A list of infiniband port guids on the system. This is discoverable using `ibstat -p` + ''; + }; + }; + + config = mkIf cfg.enable { + boot.initrd.kernelModules = [ + "mlx5_core" "mlx5_ib" "ib_cm" + "rdma_cm" "rdma_ucm" "rpcrdma" + "ib_ipoib" "ib_isert" "ib_umad" "ib_uverbs" + ]; + # rdma-core exposes ibstat, mstflint exposes mstconfig (which can be needed for + # setting link configurations), qperf needed to affirm link speeds + environment.systemPackages = with pkgs; [ + rdma-core mstflint qperf + ]; + systemd.services = opensm-services; + }; +} diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index b7582d4a6324..6ce8a2d88ab4 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -63,6 +63,7 @@ ./hardware/gpgsmartcards.nix ./hardware/hackrf.nix ./hardware/i2c.nix + ./hardware/infiniband.nix ./hardware/keyboard/qmk.nix ./hardware/keyboard/teck.nix ./hardware/keyboard/uhk.nix |