From 6914eda2fb7c12c9f58a66dcb530bafb1fa64b46 Mon Sep 17 00:00:00 2001
From: mjallen18 <matt.l.jallen@gmail.com>
Date: Tue, 18 Mar 2025 13:42:31 -0500
Subject: [PATCH] llama-cpp

---
 hosts/nas/apps/ollama/default.nix | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/hosts/nas/apps/ollama/default.nix b/hosts/nas/apps/ollama/default.nix
index a34409c..903b4d4 100755
--- a/hosts/nas/apps/ollama/default.nix
+++ b/hosts/nas/apps/ollama/default.nix
@@ -11,13 +11,21 @@
     home = "/media/nas/ssd/nix-app-data/ollama";
   };
 
-  # services.llama-cpp = {
-  #   enable = true;
-  #   port = 10400;
-  #   host = "0.0.0.0";
-  #   openFirewall = true;
-  #   model = "/media/nas/ssd/nix-app-data/llama-cpp/models/functionary-small-v3.2-GGUF/functionary-small-v3.2.Q8_0.gguf";
-  # };
+  services.llama-cpp = {
+    enable = true;
+    port = 8127;
+    host = "0.0.0.0";
+    openFirewall = true;
+    model = "/media/nas/ssd/nix-app-data/llama-cpp/models/functionary-small-v3.2-GGUF/functionary-small-v3.2.Q4_0.gguf";
+    extraFlags = [
+      "--n_gpu-layers"
+      "500"
+      "-c"
+      "0"
+      "--numa"
+      "numactl"
+    ]
+  };
 
   services.open-webui = {
     enable = true;