mjallen/nix-config
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

llama-cpp

Browse Source
This commit is contained in:
mjallen18
2025-03-18 13:42:31 -05:00
parent a537a36f7b
commit 6914eda2fb
1 changed files with 15 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
hosts/nas/apps/ollama/default.nix
View File

@@ -11,13 +11,21 @@
home = "/media/nas/ssd/nix-app-data/ollama";
};
# services.llama-cpp = {
# enable = true;
# port = 10400;
# host = "0.0.0.0";
# openFirewall = true;
# model = "/media/nas/ssd/nix-app-data/llama-cpp/models/functionary-small-v3.2-GGUF/functionary-small-v3.2.Q8_0.gguf";
# };
services.llama-cpp = {
enable = true;
port = 8127;
host = "0.0.0.0";
openFirewall = true;
model = "/media/nas/ssd/nix-app-data/llama-cpp/models/functionary-small-v3.2-GGUF/functionary-small-v3.2.Q4_0.gguf";
extraFlags = [
"--n_gpu-layers"
"500"
"-c"
"0"
"--numa"
"numactl"
]
};
services.open-webui = {
enable = true;