blob: 305d4538e89a07785a965f3f98aa97855c3f8396 (
plain) (
blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
|
{ config, lib, pkgs, utils, ... }:
let
cfg = config.services.llama-cpp;
in {
options = {
services.llama-cpp = {
enable = lib.mkEnableOption "LLaMA C++ server";
package = lib.mkPackageOption pkgs "llama-cpp" { };
model = lib.mkOption {
type = lib.types.path;
example = "/models/mistral-instruct-7b/ggml-model-q4_0.gguf";
description = "Model path.";
};
extraFlags = lib.mkOption {
type = lib.types.listOf lib.types.str;
description = "Extra flags passed to llama-cpp-server.";
example = ["-c" "4096" "-ngl" "32" "--numa"];
default = [];
};
host = lib.mkOption {
type = lib.types.str;
default = "127.0.0.1";
example = "0.0.0.0";
description = "IP address the LLaMA C++ server listens on.";
};
port = lib.mkOption {
type = lib.types.port;
default = 8080;
description = "Listen port for LLaMA C++ server.";
};
openFirewall = lib.mkOption {
type = lib.types.bool;
default = false;
description = "Open ports in the firewall for LLaMA C++ server.";
};
};
};
config = lib.mkIf cfg.enable {
systemd.services.llama-cpp = {
description = "LLaMA C++ server";
after = ["network.target"];
wantedBy = ["multi-user.target"];
serviceConfig = {
Type = "idle";
KillSignal = "SIGINT";
ExecStart = "${cfg.package}/bin/llama-server --log-disable --host ${cfg.host} --port ${builtins.toString cfg.port} -m ${cfg.model} ${utils.escapeSystemdExecArgs cfg.extraFlags}";
Restart = "on-failure";
RestartSec = 300;
# for GPU acceleration
PrivateDevices = false;
# hardening
DynamicUser = true;
CapabilityBoundingSet = "";
RestrictAddressFamilies = [
"AF_INET"
"AF_INET6"
"AF_UNIX"
];
NoNewPrivileges = true;
PrivateMounts = true;
PrivateTmp = true;
PrivateUsers = true;
ProtectClock = true;
ProtectControlGroups = true;
ProtectHome = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectSystem = "strict";
MemoryDenyWriteExecute = true;
LockPersonality = true;
RemoveIPC = true;
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
SystemCallFilter = [
"@system-service"
"~@privileged"
"~@resources"
];
SystemCallErrorNumber = "EPERM";
ProtectProc = "invisible";
ProtectHostname = true;
ProcSubset = "pid";
};
};
networking.firewall = lib.mkIf cfg.openFirewall {
allowedTCPPorts = [ cfg.port ];
};
};
meta.maintainers = with lib.maintainers; [ newam ];
}
|