stuffs

modules/nixos/services/ai/default.nix

@@ -26,24 +26,30 @@ let
           group = "jallen-nas";
           openFirewall = cfg.openFirewall;
           rocmOverrideGfx = "11.0.2";
-          loadModels = [ "mistral:instruct" ];
+          loadModels = [ ];
           home = "${cfg.configDir}/ollama";
         };
 
         llama-cpp = {
-          enable = false;
+          enable = true;
           port = 8127;
           host = "0.0.0.0";
           openFirewall = cfg.openFirewall;
-          # model = "${cfg.configDir}/llama-cpp/models/functionary-small-v3.2-GGUF/functionary-small-v3.2.Q4_0.gguf";
+          model = "${cfg.configDir}/llama-cpp/models/Qwen3-Coder-Next-Q4_0.gguf";
           package = pkgs.llama-cpp-rocm;
           extraFlags = [
-            "--n_gpu-layers"
-            "500"
-            "-c"
-            "0"
-            "--numa"
-            "numactl"
+            "--fit"
+            "on"
+            "--seed"
+            "3407"
+            "--temp"
+            "1.0"
+            "--top-p"
+            "0.95"
+            "--min-p"
+            "0.01"
+            "--top-k"
+            "40"
             "--jinja"
           ];
         };
@@ -78,4 +84,11 @@ let
 in
 {
   imports = [ aiConfig ];
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = with pkgs; [
+      amdgpu_top
+      python3Packages.huggingface-hub
+    ];
+  };
 }