71 lines
1.9 KiB
Nix
Executable File
71 lines
1.9 KiB
Nix
Executable File
{ config, pkgs, ... }:
|
|
let
|
|
llamaPackage = pkgs.llama-cpp.overrideAttrs (old: {
|
|
src = pkgs.fetchFromGitHub {
|
|
owner = "ggml-org";
|
|
repo = "llama.cpp";
|
|
rev = "b4920";
|
|
sha256 = "sha256-SnQIeY74JpAPRMxWcpklDH5D4CQvAgi0GYx5+ECk2J4=";
|
|
};
|
|
# Optionally override other attributes if you need to
|
|
# version = "my-fork-version";
|
|
# pname = "llama-cpp-custom";
|
|
});
|
|
in
|
|
{
|
|
services.ollama = {
|
|
enable = true;
|
|
port = 11434;
|
|
host = "0.0.0.0";
|
|
user = "nix-apps";
|
|
group = "jallen-nas";
|
|
openFirewall = true;
|
|
acceleration = "cuda";
|
|
home = "/media/nas/ssd/nix-app-data/ollama";
|
|
};
|
|
|
|
environment.systemPackages = [ llamaPackage ];
|
|
|
|
services.llama-cpp = {
|
|
enable = true;
|
|
port = 8127;
|
|
host = "0.0.0.0";
|
|
openFirewall = true;
|
|
model = "/media/nas/ssd/nix-app-data/llama-cpp/models/functionary-small-v3.2-GGUF/functionary-small-v3.2.Q4_0.gguf";
|
|
package = llamaPackage; # pkgs.unstable.llama-cpp;
|
|
extraFlags = [
|
|
"--n_gpu-layers"
|
|
"500"
|
|
"-c"
|
|
"0"
|
|
"--numa"
|
|
"numactl"
|
|
"--jinja"
|
|
];
|
|
};
|
|
|
|
services.open-webui = {
|
|
enable = false;
|
|
host = "0.0.0.0";
|
|
port = 8888;
|
|
openFirewall = true;
|
|
# stateDir = "/media/nas/ssd/nix-app-data/open-webui";
|
|
environmentFile = config.sops.secrets."jallen-nas/open-webui".path;
|
|
environment = {
|
|
OPENID_PROVIDER_URL = "https://authentik.mjallen.dev/application/o/chat/.well-known/openid-configuration";
|
|
OAUTH_PROVIDER_NAME = "authentik";
|
|
OPENID_REDIRECT_URI = "https://chat.mjallen.dev/oauth/oidc/callback";
|
|
ENABLE_OAUTH_SIGNUP = "False";
|
|
OAUTH_MERGE_ACCOUNTS_BY_EMAIL = "True";
|
|
ENABLE_SIGNUP = "False";
|
|
ENABLE_LOGIN_FORM = "False";
|
|
ANONYMIZED_TELEMETRY = "False";
|
|
DO_NOT_TRACK = "True";
|
|
SCARF_NO_ANALYTICS = "True";
|
|
OLLAMA_API_BASE_URL = "http://127.0.0.1:11434";
|
|
LOCAL_FILES_ONLY = "False";
|
|
WEBUI_AUTH = "False";
|
|
};
|
|
};
|
|
}
|