blob: 28aa141f762a75fa701f091735ce7676f182ecc0 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
|
{
config,
pkgs,
inputs,
ips,
lib,
...
}: {
services.llama-cpp = {
enable = true;
package = pkgs.llama-cpp-rocm;
settings = {
model = "/mnt/llms/Qwen3-Coder-30B-A3B-Instruct-UD-Q4_K_XL.gguf";
host = "127.0.0.1";
port = 1337;
temp = 0.7;
top-k = 20;
top-p = 0.8;
repeat-penalty = 1.05;
n-gpu-layers = "auto";
fit = "on";
flash-attn = "auto";
cache-type-k = "q8_0";
cache-type-v = "q8_0";
batch-size = 512;
ubatch-size = 256;
# ui-mcp-proxy = true;
# webui-mcp-proxy = true;
metrics = true;
keep = -1;
context-shift = true;
ctx-size = 131072;
models-max = 1;
sleep-idle-seconds = 600;
models-preset = "/etc/llama-cpp/models.ini";
models-dir = "/mnt/llms";
# kv-offload = false;
};
};
environment.etc."llama-cpp/models.ini".source = ../../dotfiles/llama-cpp/model-presets.ini;
}
|
