This commit is contained in:
mjallen18
2026-03-24 14:41:22 -05:00
parent 4cc58ab381
commit 84eb2e3734
11 changed files with 799 additions and 48 deletions

View File

@@ -1,13 +1,96 @@
{
config,
lib,
pkgs,
namespace,
...
}:
with lib;
let
inherit (lib.${namespace}) mkOpt;
inherit (lib.${namespace}) mkOpt mkBoolOpt;
cfg = config.${namespace}.power.ups;
# Script called by upsmon for every UPS event. Reads NTFY_USER and
# NTFY_PASSWORD from the environment (injected via EnvironmentFile on the
# upsmon systemd service). upsmon passes the event type as the first
# argument (e.g. ONBATT, ONLINE, LOWBATT, FSD, COMMOK, COMMBAD, etc).
upsNotifyScript = pkgs.writeShellScript "ups-ntfy-notify" ''
EVENT="$1"
HOST="$(${pkgs.hostname}/bin/hostname)"
SERVER="https://ntfy.mjallen.dev"
TOPIC="ups"
case "$EVENT" in
ONBATT)
TITLE="UPS on battery: $HOST"
PRIORITY="high"
TAGS="battery,rotating_light"
MESSAGE="Power failure detected. UPS is now running on battery."
;;
ONLINE)
TITLE="UPS back on mains: $HOST"
PRIORITY="low"
TAGS="electric_plug,white_check_mark"
MESSAGE="Power restored. UPS is back on mains power."
;;
LOWBATT)
TITLE="UPS battery LOW: $HOST"
PRIORITY="urgent"
TAGS="battery,sos"
MESSAGE="UPS battery is critically low. Shutdown imminent."
;;
FSD)
TITLE="UPS forced shutdown: $HOST"
PRIORITY="urgent"
TAGS="warning,sos"
MESSAGE="Forced shutdown initiated by UPS."
;;
COMMOK)
TITLE="UPS comms restored: $HOST"
PRIORITY="low"
TAGS="electric_plug,white_check_mark"
MESSAGE="Communication with UPS restored."
;;
COMMBAD)
TITLE="UPS comms lost: $HOST"
PRIORITY="high"
TAGS="warning,rotating_light"
MESSAGE="Lost communication with UPS."
;;
SHUTDOWN)
TITLE="UPS shutdown in progress: $HOST"
PRIORITY="urgent"
TAGS="warning,sos"
MESSAGE="System is shutting down due to UPS condition."
;;
REPLBATT)
TITLE="UPS battery needs replacement: $HOST"
PRIORITY="default"
TAGS="battery,warning"
MESSAGE="UPS reports battery needs replacement."
;;
NOCOMM)
TITLE="UPS unreachable: $HOST"
PRIORITY="high"
TAGS="warning,rotating_light"
MESSAGE="UPS is not reachable."
;;
*)
TITLE="UPS event on $HOST: $EVENT"
PRIORITY="default"
TAGS="electric_plug"
MESSAGE="UPS event: $EVENT"
;;
esac
${pkgs.curl}/bin/curl -sf \
--user "$NTFY_USER:$NTFY_PASSWORD" \
-H "Title: $TITLE" \
-H "Priority: $PRIORITY" \
-H "Tags: $TAGS" \
-d "$MESSAGE" \
"$SERVER/$TOPIC" || true
'';
in
{
options.${namespace}.power.ups = {
@@ -17,6 +100,11 @@ in
upsUser = mkOpt types.str "nas-admin" "Name of the ups user";
upsdPort = mkOpt types.int 3493 "Port for upsd";
ntfy = {
enable = mkBoolOpt false "Send ntfy notifications on UPS events";
envFile = mkOpt types.str "" "Path to env file containing NTFY_USER and NTFY_PASSWORD";
};
};
config = mkIf cfg.enable {
@@ -61,6 +149,49 @@ in
passwordFile = config.sops.secrets."jallen-nas/ups_password".path;
user = cfg.upsUser;
};
# Call the notify script for all event types we care about.
settings = mkIf cfg.ntfy.enable {
NOTIFYCMD = "${upsNotifyScript}";
NOTIFYFLAG = [
[
"ONLINE"
"SYSLOG+WALL+EXEC"
]
[
"ONBATT"
"SYSLOG+WALL+EXEC"
]
[
"LOWBATT"
"SYSLOG+WALL+EXEC"
]
[
"FSD"
"SYSLOG+WALL+EXEC"
]
[
"COMMOK"
"SYSLOG+WALL+EXEC"
]
[
"COMMBAD"
"SYSLOG+WALL+EXEC"
]
[
"SHUTDOWN"
"SYSLOG+WALL+EXEC"
]
[
"REPLBATT"
"SYSLOG+WALL+EXEC"
]
[
"NOCOMM"
"SYSLOG+WALL+EXEC"
]
];
};
};
upsd = {
@@ -74,5 +205,8 @@ in
};
};
# Inject ntfy credentials into the upsmon service so the notify script
# can read NTFY_USER and NTFY_PASSWORD from the environment.
systemd.services.upsmon.serviceConfig.EnvironmentFile = mkIf cfg.ntfy.enable [ cfg.ntfy.envFile ];
};
}

View File

@@ -11,6 +11,17 @@ let
cfg = config.${namespace}.services.ai;
ntfyModelFailScript = pkgs.writeShellScript "update-qwen-model-notify-failure" ''
HOST="$(${pkgs.hostname}/bin/hostname)"
${pkgs.curl}/bin/curl -sf \
--user "$NTFY_USER:$NTFY_PASSWORD" \
-H "Title: Qwen model update FAILED on $HOST" \
-H "Priority: high" \
-H "Tags: rotating_light,robot_face" \
-d "The daily update-qwen-model job failed. Check: journalctl -u update-qwen-model.service" \
"https://ntfy.mjallen.dev/builds" || true
'';
aiConfig = lib.${namespace}.mkModule {
inherit config;
name = "ai";
@@ -127,11 +138,22 @@ let
''}";
User = "nix-apps";
Group = "jallen-nas";
EnvironmentFile = [ config.sops.templates."ntfy.env".path ];
};
unitConfig.OnFailure = "update-qwen-model-notify-failure.service";
# Run daily at 3 AM
startAt = "*-*-* 03:00:00";
};
systemd.services.update-qwen-model-notify-failure = {
description = "Notify ntfy on update-qwen-model failure";
serviceConfig = {
Type = "oneshot";
ExecStart = "${ntfyModelFailScript}";
EnvironmentFile = [ config.sops.templates."ntfy.env".path ];
};
};
# Ensure model is available before llama-cpp starts
systemd.services.llama-cpp = {
after = [ "update-qwen-model.service" ];

View File

@@ -10,6 +10,17 @@ let
name = "attic";
cfg = config.${namespace}.services.${name};
ntfyFailScript = pkgs.writeShellScript "nix-rebuild-cache-notify-failure" ''
HOST="$(${pkgs.hostname}/bin/hostname)"
${pkgs.curl}/bin/curl -sf \
--user "$NTFY_USER:$NTFY_PASSWORD" \
-H "Title: Nix cache rebuild FAILED on $HOST" \
-H "Priority: high" \
-H "Tags: rotating_light,nix_snowflake" \
-d "The weekly nix-rebuild-cache job failed. Check: journalctl -u nix-rebuild-cache.service" \
"https://ntfy.mjallen.dev/builds" || true
'';
atticConfig = lib.${namespace}.mkModule {
inherit config name;
description = "attic Service";
@@ -60,7 +71,9 @@ let
StandardError = "journal+console";
Restart = "no";
TimeoutStartSec = "2h";
EnvironmentFile = [ config.sops.templates."ntfy.env".path ];
};
unitConfig.OnFailure = "nix-rebuild-cache-notify-failure.service";
path = with pkgs; [
nix
git
@@ -112,6 +125,15 @@ let
fi;
'';
};
nix-rebuild-cache-notify-failure = {
description = "Notify ntfy on nix-rebuild-cache failure";
serviceConfig = {
Type = "oneshot";
ExecStart = "${ntfyFailScript}";
EnvironmentFile = [ config.sops.templates."ntfy.env".path ];
};
};
};
# Include timers for cache rebuilds

View File

@@ -6,15 +6,51 @@
...
}:
let
inherit (lib.${namespace}) mkOpt;
inherit (lib.${namespace}) mkOpt mkBoolOpt;
name = "crowdsec";
cfg = config.${namespace}.services.${name};
ntfyServer = "https://ntfy.mjallen.dev";
ntfyTopic = "crowdsec";
# CrowdSec HTTP notification plugin config — written to
# /etc/crowdsec/notifications/ntfy.yaml at runtime. Credentials are
# injected via EnvironmentFile so the plugin can reference them with
# {{env "NTFY_USER"}} / {{env "NTFY_PASSWORD"}} in the URL.
ntfyPluginConfig = pkgs.writeText "crowdsec-ntfy.yaml" ''
type: http
name: ntfy_plugin
log_level: info
format: |
{{range . -}}
CrowdSec blocked: {{.Scenario}}
Source IP: {{.Source.Value}}
Country: {{.Source.Cn}}
Decisions: {{.Decisions | len}}
{{range .Decisions -}}
Action: {{.Type}} for {{.Duration}}
{{end}}
{{- end}}
url: ${ntfyServer}/${ntfyTopic}
method: POST
headers:
Title: "CrowdSec: {{(index . 0).Scenario}}"
Priority: "high"
Tags: "rotating_light,shield"
Authorization: "Basic {{b64enc (print (env "NTFY_USER") ":" (env "NTFY_PASSWORD"))}}"
skip_tls_verify: false
timeout: 10s
'';
crowdsecConfig = lib.${namespace}.mkModule {
inherit config name;
description = "crowdsec";
options = with lib; {
apiKey = mkOpt types.str "" "API key for crowdsec bouncer";
ntfy = {
enable = mkBoolOpt false "Send ntfy notifications on new CrowdSec alerts";
envFile = mkOpt types.str "" "Path to env file containing NTFY_USER and NTFY_PASSWORD";
};
};
moduleConfig = {
services = {
@@ -199,6 +235,57 @@ let
user = "crowdsec";
group = "crowdsec";
};
# ---------------------------------------------------------------------------
# ntfy notifications via the CrowdSec HTTP notification plugin
# ---------------------------------------------------------------------------
# Drop the plugin config YAML into /etc/crowdsec/notifications/.
# CrowdSec scans this directory on startup and registers any plugin
# config files it finds.
environment.etc."crowdsec/notifications/ntfy.yaml" = lib.mkIf cfg.ntfy.enable {
source = ntfyPluginConfig;
mode = "0440";
user = "crowdsec";
group = "crowdsec";
};
# CrowdSec profiles.yaml: route every alert to the ntfy plugin.
# This replaces the default "do nothing" profile.
environment.etc."crowdsec/profiles.yaml" = lib.mkIf cfg.ntfy.enable {
text = ''
name: default_ip_remediation
filters:
- Alert.Remediation == true && Alert.GetScope() == "Ip"
decisions:
- type: ban
duration: 4h
notifications:
- ntfy_plugin
on_success: break
---
name: default_range_remediation
filters:
- Alert.Remediation == true && Alert.GetScope() == "Range"
decisions:
- type: ban
duration: 4h
notifications:
- ntfy_plugin
on_success: break
'';
mode = "0440";
user = "crowdsec";
group = "crowdsec";
};
# Inject NTFY_USER and NTFY_PASSWORD into the crowdsec service so the
# HTTP plugin template can reference them. The plugin config uses
# {{env "NTFY_BASIC_AUTH"}} — a pre-encoded "user:pass" base64 string
# for the Authorization: Basic header — computed in ExecStartPre.
systemd.services.crowdsec.serviceConfig.EnvironmentFile = lib.mkIf cfg.ntfy.enable [
cfg.ntfy.envFile
];
};
};
in

View File

@@ -151,7 +151,7 @@ let
'';
giteaPort = config.${namespace}.services.gitea.port;
resticPort = config.${namespace}.services.restic.port;
resticPort = config.${namespace}.services.restic-server.port;
nextcloudPort = config.${namespace}.services.nextcloud.port;
grafanaConfig = lib.${namespace}.mkModule {
@@ -392,27 +392,462 @@ let
httpMethod: POST
timeInterval: 15s
'';
# Provide empty-but-valid alerting provisioning documents.
# Without these, the NixOS module serialises `null` YAML which
# Grafana 12's provisioner fails to parse, producing a spurious
# "data source not found" error at startup.
# ---------------------------------------------------------------------------
# Alerting provisioning
# ---------------------------------------------------------------------------
alerting = {
rules.settings = {
apiVersion = 1;
groups = [ ];
};
# ── Contact points ──────────────────────────────────────────────────
# ntfy via the Grafana webhook contact point. Grafana POSTs a JSON
# body; ntfy accepts any body as the message text. We use the
# message template below to format it nicely.
# Basic auth credentials are read from the SOPS secret at runtime
# via Grafana's $__file{} provider.
contactPoints.settings = {
apiVersion = 1;
contactPoints = [ ];
};
policies.settings = {
apiVersion = 1;
policies = [ ];
contactPoints = [
{
name = "ntfy";
receivers = [
{
uid = "ntfy-webhook";
type = "webhook";
settings = {
url = "https://ntfy.mjallen.dev/grafana-alerts";
httpMethod = "POST";
username = "$__file{${config.sops.secrets."jallen-nas/ntfy/user".path}}";
password = "$__file{${config.sops.secrets."jallen-nas/ntfy/password".path}}";
# Pass alert title and state as ntfy headers via the
# custom message template (defined below).
httpHeaders = {
"Tags" = "chart,bell";
};
};
disableResolveMessage = false;
}
];
}
];
};
# ── Notification message template ───────────────────────────────────
# Grafana sends the rendered template body as the POST body.
# ntfy treats the body as the message text.
templates.settings = {
apiVersion = 1;
templates = [ ];
templates = [
{
name = "ntfy_message";
template = ''
{{ define "ntfy_message" -}}
{{ .CommonAnnotations.summary | default .GroupLabels.alertname }}
{{ range .Alerts -}}
Status: {{ .Status | title }}
Alert: {{ .Labels.alertname }}
Severity: {{ .Labels.severity | default "unknown" }}
Instance: {{ .Labels.instance | default "unknown" }}
{{ if .Annotations.description -}}
Details: {{ .Annotations.description }}
{{ end -}}
{{ end -}}
{{ end }}
'';
}
];
};
# ── Notification routing policy ─────────────────────────────────────
policies.settings = {
apiVersion = 1;
policies = [
{
receiver = "ntfy";
group_by = [
"alertname"
"severity"
];
group_wait = "30s";
group_interval = "5m";
repeat_interval = "4h";
routes = [
# Critical alerts: repeat every 1h, no grouping wait
{
receiver = "ntfy";
matchers = [ "severity = critical" ];
group_wait = "0s";
repeat_interval = "1h";
}
];
}
];
};
# ── Alert rules ─────────────────────────────────────────────────────
rules.settings = {
apiVersion = 1;
groups = [
{
name = "nas-system";
folder = "NAS Alerts";
interval = "1m";
rules = [
# Disk usage > 85% warning, > 95% critical
{
uid = "nas-disk-warning";
title = "Disk usage high";
condition = "C";
data = [
{
refId = "A";
datasourceUid = "prometheus";
model = {
expr = ''
(
node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint!~"/boot.*"}
- node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint!~"/boot.*"}
)
/ node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint!~"/boot.*"}
* 100
'';
intervalMs = 60000;
maxDataPoints = 43200;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
refId = "B";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "threshold";
refId = "C";
expression = "B";
conditions = [
{
evaluator = {
type = "gt";
params = [ 85 ];
};
}
];
};
}
];
noDataState = "NoData";
execErrState = "Error";
for = "5m";
annotations = {
summary = "Disk usage above 85%";
description = "Filesystem {{ $labels.mountpoint }} is {{ $values.B | printf \"%.1f\" }}% full.";
};
labels = {
severity = "warning";
};
isPaused = false;
}
# Memory usage > 90%
{
uid = "nas-memory-high";
title = "Memory usage high";
condition = "C";
data = [
{
refId = "A";
datasourceUid = "prometheus";
model = {
expr = ''
(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100
'';
intervalMs = 60000;
maxDataPoints = 43200;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
refId = "B";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "threshold";
refId = "C";
expression = "B";
conditions = [
{
evaluator = {
type = "gt";
params = [ 90 ];
};
}
];
};
}
];
noDataState = "NoData";
execErrState = "Error";
for = "5m";
annotations = {
summary = "Memory usage above 90%";
description = "Memory usage is {{ $values.B | printf \"%.1f\" }}%.";
};
labels = {
severity = "warning";
};
isPaused = false;
}
# CPU > 90% sustained for 10m
{
uid = "nas-cpu-high";
title = "CPU usage sustained high";
condition = "C";
data = [
{
refId = "A";
datasourceUid = "prometheus";
model = {
expr = ''
100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
'';
intervalMs = 60000;
maxDataPoints = 43200;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
refId = "B";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "threshold";
refId = "C";
expression = "B";
conditions = [
{
evaluator = {
type = "gt";
params = [ 90 ];
};
}
];
};
}
];
noDataState = "NoData";
execErrState = "Error";
for = "10m";
annotations = {
summary = "CPU sustained above 90%";
description = "CPU usage has been above 90% for 10 minutes (currently {{ $values.B | printf \"%.1f\" }}%).";
};
labels = {
severity = "warning";
};
isPaused = false;
}
# UPS on battery (network_ups_tools_ups_status == 0 means OB/on-battery)
{
uid = "nas-ups-onbatt";
title = "UPS on battery";
condition = "C";
data = [
{
refId = "A";
datasourceUid = "prometheus";
model = {
expr = "network_ups_tools_ups_status";
intervalMs = 60000;
maxDataPoints = 43200;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
refId = "B";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "threshold";
refId = "C";
expression = "B";
# status 0 = OB (on battery), 1 = OL (online)
conditions = [
{
evaluator = {
type = "lt";
params = [ 1 ];
};
}
];
};
}
];
noDataState = "NoData";
execErrState = "Error";
for = "1m";
annotations = {
summary = "UPS is running on battery";
description = "Mains power failure detected. UPS battery charge: {{ with query \"network_ups_tools_battery_charge\" }}{{ . | first | value | printf \"%.0f\" }}%{{ end }}.";
};
labels = {
severity = "critical";
};
isPaused = false;
}
# UPS battery charge < 30%
{
uid = "nas-ups-lowbatt";
title = "UPS battery low";
condition = "C";
data = [
{
refId = "A";
datasourceUid = "prometheus";
model = {
expr = "network_ups_tools_battery_charge";
intervalMs = 60000;
maxDataPoints = 43200;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
refId = "B";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "threshold";
refId = "C";
expression = "B";
conditions = [
{
evaluator = {
type = "lt";
params = [ 30 ];
};
}
];
};
}
];
noDataState = "NoData";
execErrState = "Error";
for = "2m";
annotations = {
summary = "UPS battery charge below 30%";
description = "UPS battery is at {{ $values.B | printf \"%.0f\" }}%. Shutdown may be imminent.";
};
labels = {
severity = "critical";
};
isPaused = false;
}
# PostgreSQL not responding
{
uid = "nas-postgres-down";
title = "PostgreSQL down";
condition = "C";
data = [
{
refId = "A";
datasourceUid = "prometheus";
model = {
expr = "pg_up";
intervalMs = 60000;
maxDataPoints = 43200;
refId = "A";
};
}
{
refId = "B";
datasourceUid = "__expr__";
model = {
type = "reduce";
refId = "B";
expression = "A";
reducer = "last";
};
}
{
refId = "C";
datasourceUid = "__expr__";
model = {
type = "threshold";
refId = "C";
expression = "B";
conditions = [
{
evaluator = {
type = "lt";
params = [ 1 ];
};
}
];
};
}
];
noDataState = "Alerting";
execErrState = "Error";
for = "2m";
annotations = {
summary = "PostgreSQL is down";
description = "The PostgreSQL exporter reports pg_up=0. Database may be unavailable.";
};
labels = {
severity = "critical";
};
isPaused = false;
}
];
}
];
};
muteTimings.settings = {
apiVersion = 1;
muteTimes = [ ];

File diff suppressed because one or more lines are too long

View File

@@ -87,6 +87,10 @@ in
enable = true;
port = 8181;
apiKey = config.sops.secrets."jallen-nas/crowdsec-capi".path;
ntfy = {
enable = true;
envFile = config.sops.templates."ntfy.env".path;
};
};
dispatcharr = {
enable = false;
@@ -208,7 +212,7 @@ in
smtpPort = 1025;
imapPort = 1143;
};
restic = {
restic-server = {
enable = true;
port = 8008;
};

View File

@@ -192,7 +192,13 @@ in
# # Power # #
# ###################################################
power.ups = enabled;
power.ups = {
enable = true;
ntfy = {
enable = true;
envFile = config.sops.templates."ntfy.env".path;
};
};
# ###################################################
# # Samba # #
@@ -304,31 +310,34 @@ in
# Configure environment
environment = {
systemPackages = with pkgs; [
attic-client
bcachefs-tools
cryptsetup
clevis
deconz
duperemove
efibootmgr
ffmpeg
ipset
keyutils
nut
packagekit
pass
protonmail-bridge
protonvpn-gui
qrencode
sbctl
systemctl-tui
tigervnc
tpm2-tools
tpm2-tss
] ++ (with pkgs.${namespace}; [
nebula-sign-cert
]);
systemPackages =
with pkgs;
[
attic-client
bcachefs-tools
cryptsetup
clevis
deconz
duperemove
efibootmgr
ffmpeg
ipset
keyutils
nut
packagekit
pass
protonmail-bridge
protonvpn-gui
qrencode
sbctl
systemctl-tui
tigervnc
tpm2-tools
tpm2-tss
]
++ (with pkgs.${namespace}; [
nebula-sign-cert
]);
persistence."/media/nas/main/persist" = {
hideMounts = true;
directories = [

View File

@@ -55,7 +55,7 @@ in
paperless = mkForce disabled;
paperless-ai = mkForce disabled;
protonmail-bridge = mkForce disabled;
restic = mkForce disabled;
restic-server = mkForce disabled;
sunshine = mkForce disabled;
tdarr = mkForce disabled;
unmanic = mkForce disabled;

View File

@@ -72,7 +72,7 @@ in
"paperless"
"paperless-ai"
"protonmail-bridge"
"restic"
"restic-server"
"sparky-fitness"
"sparky-fitness-server"
"sunshine"

View File

@@ -259,6 +259,28 @@ in
"jallen-nas/ntfy/auth-users" = {
sopsFile = defaultSops;
};
"jallen-nas/ntfy/user" = {
sopsFile = defaultSops;
mode = "0440";
owner = "grafana";
group = "keys";
restartUnits = [
"grafana.service"
"crowdsec.service"
"upsmon.service"
];
};
"jallen-nas/ntfy/password" = {
sopsFile = defaultSops;
mode = "0440";
owner = "grafana";
group = "keys";
restartUnits = [
"grafana.service"
"crowdsec.service"
"upsmon.service"
];
};
# ------------------------------
# sparky-fitness
@@ -330,6 +352,20 @@ in
restartUnits = [ "podman-authenticRac.service" ];
};
"ntfy.env" = {
content = ''
NTFY_USER=${config.sops.placeholder."jallen-nas/ntfy/user"}
NTFY_PASSWORD=${config.sops.placeholder."jallen-nas/ntfy/password"}
'';
mode = "0600";
restartUnits = [
"crowdsec.service"
"upsmon.service"
"nix-rebuild-cache.service"
"update-qwen-model.service"
];
};
"paperless.env" = {
content = ''
PAPERLESS_ADMIN_USER = "mjallen"