diff options
| -rw-r--r-- | nixos/modules/module-list.nix | 1 | ||||
| -rw-r--r-- | nixos/modules/services/networking/rxe.nix | 63 | ||||
| -rw-r--r-- | nixos/release.nix | 1 | ||||
| -rw-r--r-- | nixos/tests/rxe.nix | 53 |
4 files changed, 118 insertions, 0 deletions
diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index c0c72f2bdb9d..2ef8684d7f95 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -531,6 +531,7 @@ ./services/networking/redsocks.nix ./services/networking/resilio.nix ./services/networking/rpcbind.nix + ./services/networking/rxe.nix ./services/networking/sabnzbd.nix ./services/networking/searx.nix ./services/networking/seeks.nix diff --git a/nixos/modules/services/networking/rxe.nix b/nixos/modules/services/networking/rxe.nix new file mode 100644 index 000000000000..a6a069ec50c0 --- /dev/null +++ b/nixos/modules/services/networking/rxe.nix @@ -0,0 +1,63 @@ +{ config, lib, pkgs, ... }: + +with lib; + +let + cfg = config.networking.rxe; + + runRxeCmd = cmd: ifcs: + concatStrings ( map (x: "${pkgs.rdma-core}/bin/rxe_cfg -n ${cmd} ${x};") ifcs); + + startScript = pkgs.writeShellScriptBin "rxe-start" '' + ${pkgs.rdma-core}/bin/rxe_cfg -n start + ${runRxeCmd "add" cfg.interfaces} + ${pkgs.rdma-core}/bin/rxe_cfg + ''; + + stopScript = pkgs.writeShellScriptBin "rxe-stop" '' + ${runRxeCmd "remove" cfg.interfaces } + ${pkgs.rdma-core}/bin/rxe_cfg -n stop + ''; + +in { + ###### interface + + options = { + networking.rxe = { + enable = mkEnableOption "RDMA over converged ethernet"; + interfaces = mkOption { + type = types.listOf types.str; + default = [ ]; + example = [ "eth0" ]; + description = '' + Enable RDMA on the listed interfaces. The corresponding virtual + RDMA interfaces will be named rxe0 ... rxeN where the ordering + will be as they are named in the list. UDP port 4791 must be + open on the respective ethernet interfaces. + ''; + }; + }; + }; + + ###### implementation + + config = mkIf cfg.enable { + + systemd.services.rxe = { + path = with pkgs; [ kmod rdma-core ]; + description = "RoCE interfaces"; + + wantedBy = [ "multi-user.target" ]; + after = [ "systemd-modules-load.service" "network-online.target" ]; + wants = [ "network-pre.target" ]; + + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = "${startScript}/bin/rxe-start"; + ExecStop = "${stopScript}/bin/rxe-stop"; + }; + }; + }; +} + diff --git a/nixos/release.nix b/nixos/release.nix index 8057e2d50fa6..b778258da63d 100644 --- a/nixos/release.nix +++ b/nixos/release.nix @@ -336,6 +336,7 @@ in rec { tests.radicale = callTest tests/radicale.nix {}; tests.rspamd = callSubTests tests/rspamd.nix {}; tests.runInMachine = callTest tests/run-in-machine.nix {}; + tests.rxe = callTest tests/rxe.nix {}; tests.samba = callTest tests/samba.nix {}; tests.sddm = callSubTests tests/sddm.nix {}; tests.simple = callTest tests/simple.nix {}; diff --git a/nixos/tests/rxe.nix b/nixos/tests/rxe.nix new file mode 100644 index 000000000000..cfe64a75a635 --- /dev/null +++ b/nixos/tests/rxe.nix @@ -0,0 +1,53 @@ +import ./make-test.nix ({ pkgs, ... } : + +let + node = { config, pkgs, lib, ... } : { + networking = { + firewall = { + allowedUDPPorts = [ 4791 ]; # open RoCE port + allowedTCPPorts = [ 4800 ]; # port for test utils + }; + rxe = { + enable = true; + interfaces = [ "eth1" ]; + }; + }; + + environment.systemPackages = with pkgs; [ rdma-core screen ]; + }; + +in { + name = "rxe"; + + nodes = { + server = node; + client = node; + }; + + testScript = '' + # Test if rxe interface comes up + $server->waitForUnit("default.target"); + $server->succeed("systemctl status rxe.service"); + $server->succeed("ibv_devices | grep rxe0"); + + $client->waitForUnit("default.target"); + + # ping pong test + $server->succeed("screen -dmS rc_pingpong ibv_rc_pingpong -p 4800 -g0"); + $client->succeed("sleep 2; ibv_rc_pingpong -p 4800 -g0 server"); + + $server->succeed("screen -dmS uc_pingpong ibv_uc_pingpong -p 4800 -g0"); + $client->succeed("sleep 2; ibv_uc_pingpong -p 4800 -g0 server"); + + $server->succeed("screen -dmS ud_pingpong ibv_ud_pingpong -p 4800 -s 1024 -g0"); + $client->succeed("sleep 2; ibv_ud_pingpong -p 4800 -s 1024 -g0 server"); + + $server->succeed("screen -dmS srq_pingpong ibv_srq_pingpong -p 4800 -g0"); + $client->succeed("sleep 2; ibv_srq_pingpong -p 4800 -g0 server"); + + $server->succeed("screen -dmS rping rping -s -a server -C 10"); + $client->succeed("sleep 2; rping -c -a server -C 10"); + ''; +}) + + |