diff --git a/modules/nixos/power/default.nix b/modules/nixos/power/default.nix index 05cdaa6..646b124 100644 --- a/modules/nixos/power/default.nix +++ b/modules/nixos/power/default.nix @@ -1,13 +1,96 @@ { config, lib, + pkgs, namespace, ... }: with lib; let - inherit (lib.${namespace}) mkOpt; + inherit (lib.${namespace}) mkOpt mkBoolOpt; cfg = config.${namespace}.power.ups; + + # Script called by upsmon for every UPS event. Reads NTFY_USER and + # NTFY_PASSWORD from the environment (injected via EnvironmentFile on the + # upsmon systemd service). upsmon passes the event type as the first + # argument (e.g. ONBATT, ONLINE, LOWBATT, FSD, COMMOK, COMMBAD, etc). + upsNotifyScript = pkgs.writeShellScript "ups-ntfy-notify" '' + EVENT="$1" + HOST="$(${pkgs.hostname}/bin/hostname)" + SERVER="https://ntfy.mjallen.dev" + TOPIC="ups" + + case "$EVENT" in + ONBATT) + TITLE="UPS on battery: $HOST" + PRIORITY="high" + TAGS="battery,rotating_light" + MESSAGE="Power failure detected. UPS is now running on battery." + ;; + ONLINE) + TITLE="UPS back on mains: $HOST" + PRIORITY="low" + TAGS="electric_plug,white_check_mark" + MESSAGE="Power restored. UPS is back on mains power." + ;; + LOWBATT) + TITLE="UPS battery LOW: $HOST" + PRIORITY="urgent" + TAGS="battery,sos" + MESSAGE="UPS battery is critically low. Shutdown imminent." + ;; + FSD) + TITLE="UPS forced shutdown: $HOST" + PRIORITY="urgent" + TAGS="warning,sos" + MESSAGE="Forced shutdown initiated by UPS." + ;; + COMMOK) + TITLE="UPS comms restored: $HOST" + PRIORITY="low" + TAGS="electric_plug,white_check_mark" + MESSAGE="Communication with UPS restored." + ;; + COMMBAD) + TITLE="UPS comms lost: $HOST" + PRIORITY="high" + TAGS="warning,rotating_light" + MESSAGE="Lost communication with UPS." + ;; + SHUTDOWN) + TITLE="UPS shutdown in progress: $HOST" + PRIORITY="urgent" + TAGS="warning,sos" + MESSAGE="System is shutting down due to UPS condition." + ;; + REPLBATT) + TITLE="UPS battery needs replacement: $HOST" + PRIORITY="default" + TAGS="battery,warning" + MESSAGE="UPS reports battery needs replacement." + ;; + NOCOMM) + TITLE="UPS unreachable: $HOST" + PRIORITY="high" + TAGS="warning,rotating_light" + MESSAGE="UPS is not reachable." + ;; + *) + TITLE="UPS event on $HOST: $EVENT" + PRIORITY="default" + TAGS="electric_plug" + MESSAGE="UPS event: $EVENT" + ;; + esac + + ${pkgs.curl}/bin/curl -sf \ + --user "$NTFY_USER:$NTFY_PASSWORD" \ + -H "Title: $TITLE" \ + -H "Priority: $PRIORITY" \ + -H "Tags: $TAGS" \ + -d "$MESSAGE" \ + "$SERVER/$TOPIC" || true + ''; in { options.${namespace}.power.ups = { @@ -17,6 +100,11 @@ in upsUser = mkOpt types.str "nas-admin" "Name of the ups user"; upsdPort = mkOpt types.int 3493 "Port for upsd"; + + ntfy = { + enable = mkBoolOpt false "Send ntfy notifications on UPS events"; + envFile = mkOpt types.str "" "Path to env file containing NTFY_USER and NTFY_PASSWORD"; + }; }; config = mkIf cfg.enable { @@ -61,6 +149,49 @@ in passwordFile = config.sops.secrets."jallen-nas/ups_password".path; user = cfg.upsUser; }; + + # Call the notify script for all event types we care about. + settings = mkIf cfg.ntfy.enable { + NOTIFYCMD = "${upsNotifyScript}"; + NOTIFYFLAG = [ + [ + "ONLINE" + "SYSLOG+WALL+EXEC" + ] + [ + "ONBATT" + "SYSLOG+WALL+EXEC" + ] + [ + "LOWBATT" + "SYSLOG+WALL+EXEC" + ] + [ + "FSD" + "SYSLOG+WALL+EXEC" + ] + [ + "COMMOK" + "SYSLOG+WALL+EXEC" + ] + [ + "COMMBAD" + "SYSLOG+WALL+EXEC" + ] + [ + "SHUTDOWN" + "SYSLOG+WALL+EXEC" + ] + [ + "REPLBATT" + "SYSLOG+WALL+EXEC" + ] + [ + "NOCOMM" + "SYSLOG+WALL+EXEC" + ] + ]; + }; }; upsd = { @@ -74,5 +205,8 @@ in }; }; + # Inject ntfy credentials into the upsmon service so the notify script + # can read NTFY_USER and NTFY_PASSWORD from the environment. + systemd.services.upsmon.serviceConfig.EnvironmentFile = mkIf cfg.ntfy.enable [ cfg.ntfy.envFile ]; }; } diff --git a/modules/nixos/services/ai/default.nix b/modules/nixos/services/ai/default.nix index 34762cf..d662ace 100755 --- a/modules/nixos/services/ai/default.nix +++ b/modules/nixos/services/ai/default.nix @@ -11,6 +11,17 @@ let cfg = config.${namespace}.services.ai; + ntfyModelFailScript = pkgs.writeShellScript "update-qwen-model-notify-failure" '' + HOST="$(${pkgs.hostname}/bin/hostname)" + ${pkgs.curl}/bin/curl -sf \ + --user "$NTFY_USER:$NTFY_PASSWORD" \ + -H "Title: Qwen model update FAILED on $HOST" \ + -H "Priority: high" \ + -H "Tags: rotating_light,robot_face" \ + -d "The daily update-qwen-model job failed. Check: journalctl -u update-qwen-model.service" \ + "https://ntfy.mjallen.dev/builds" || true + ''; + aiConfig = lib.${namespace}.mkModule { inherit config; name = "ai"; @@ -127,11 +138,22 @@ let ''}"; User = "nix-apps"; Group = "jallen-nas"; + EnvironmentFile = [ config.sops.templates."ntfy.env".path ]; }; + unitConfig.OnFailure = "update-qwen-model-notify-failure.service"; # Run daily at 3 AM startAt = "*-*-* 03:00:00"; }; + systemd.services.update-qwen-model-notify-failure = { + description = "Notify ntfy on update-qwen-model failure"; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${ntfyModelFailScript}"; + EnvironmentFile = [ config.sops.templates."ntfy.env".path ]; + }; + }; + # Ensure model is available before llama-cpp starts systemd.services.llama-cpp = { after = [ "update-qwen-model.service" ]; diff --git a/modules/nixos/services/attic/default.nix b/modules/nixos/services/attic/default.nix index a5b147d..7a61746 100644 --- a/modules/nixos/services/attic/default.nix +++ b/modules/nixos/services/attic/default.nix @@ -10,6 +10,17 @@ let name = "attic"; cfg = config.${namespace}.services.${name}; + ntfyFailScript = pkgs.writeShellScript "nix-rebuild-cache-notify-failure" '' + HOST="$(${pkgs.hostname}/bin/hostname)" + ${pkgs.curl}/bin/curl -sf \ + --user "$NTFY_USER:$NTFY_PASSWORD" \ + -H "Title: Nix cache rebuild FAILED on $HOST" \ + -H "Priority: high" \ + -H "Tags: rotating_light,nix_snowflake" \ + -d "The weekly nix-rebuild-cache job failed. Check: journalctl -u nix-rebuild-cache.service" \ + "https://ntfy.mjallen.dev/builds" || true + ''; + atticConfig = lib.${namespace}.mkModule { inherit config name; description = "attic Service"; @@ -60,7 +71,9 @@ let StandardError = "journal+console"; Restart = "no"; TimeoutStartSec = "2h"; + EnvironmentFile = [ config.sops.templates."ntfy.env".path ]; }; + unitConfig.OnFailure = "nix-rebuild-cache-notify-failure.service"; path = with pkgs; [ nix git @@ -112,6 +125,15 @@ let fi; ''; }; + + nix-rebuild-cache-notify-failure = { + description = "Notify ntfy on nix-rebuild-cache failure"; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${ntfyFailScript}"; + EnvironmentFile = [ config.sops.templates."ntfy.env".path ]; + }; + }; }; # Include timers for cache rebuilds diff --git a/modules/nixos/services/crowdsec/default.nix b/modules/nixos/services/crowdsec/default.nix index 9aca81e..acf8aaa 100755 --- a/modules/nixos/services/crowdsec/default.nix +++ b/modules/nixos/services/crowdsec/default.nix @@ -6,15 +6,51 @@ ... }: let - inherit (lib.${namespace}) mkOpt; + inherit (lib.${namespace}) mkOpt mkBoolOpt; name = "crowdsec"; cfg = config.${namespace}.services.${name}; + ntfyServer = "https://ntfy.mjallen.dev"; + ntfyTopic = "crowdsec"; + + # CrowdSec HTTP notification plugin config — written to + # /etc/crowdsec/notifications/ntfy.yaml at runtime. Credentials are + # injected via EnvironmentFile so the plugin can reference them with + # {{env "NTFY_USER"}} / {{env "NTFY_PASSWORD"}} in the URL. + ntfyPluginConfig = pkgs.writeText "crowdsec-ntfy.yaml" '' + type: http + name: ntfy_plugin + log_level: info + format: | + {{range . -}} + CrowdSec blocked: {{.Scenario}} + Source IP: {{.Source.Value}} + Country: {{.Source.Cn}} + Decisions: {{.Decisions | len}} + {{range .Decisions -}} + Action: {{.Type}} for {{.Duration}} + {{end}} + {{- end}} + url: ${ntfyServer}/${ntfyTopic} + method: POST + headers: + Title: "CrowdSec: {{(index . 0).Scenario}}" + Priority: "high" + Tags: "rotating_light,shield" + Authorization: "Basic {{b64enc (print (env "NTFY_USER") ":" (env "NTFY_PASSWORD"))}}" + skip_tls_verify: false + timeout: 10s + ''; + crowdsecConfig = lib.${namespace}.mkModule { inherit config name; description = "crowdsec"; options = with lib; { apiKey = mkOpt types.str "" "API key for crowdsec bouncer"; + ntfy = { + enable = mkBoolOpt false "Send ntfy notifications on new CrowdSec alerts"; + envFile = mkOpt types.str "" "Path to env file containing NTFY_USER and NTFY_PASSWORD"; + }; }; moduleConfig = { services = { @@ -199,6 +235,57 @@ let user = "crowdsec"; group = "crowdsec"; }; + + # --------------------------------------------------------------------------- + # ntfy notifications via the CrowdSec HTTP notification plugin + # --------------------------------------------------------------------------- + + # Drop the plugin config YAML into /etc/crowdsec/notifications/. + # CrowdSec scans this directory on startup and registers any plugin + # config files it finds. + environment.etc."crowdsec/notifications/ntfy.yaml" = lib.mkIf cfg.ntfy.enable { + source = ntfyPluginConfig; + mode = "0440"; + user = "crowdsec"; + group = "crowdsec"; + }; + + # CrowdSec profiles.yaml: route every alert to the ntfy plugin. + # This replaces the default "do nothing" profile. + environment.etc."crowdsec/profiles.yaml" = lib.mkIf cfg.ntfy.enable { + text = '' + name: default_ip_remediation + filters: + - Alert.Remediation == true && Alert.GetScope() == "Ip" + decisions: + - type: ban + duration: 4h + notifications: + - ntfy_plugin + on_success: break + --- + name: default_range_remediation + filters: + - Alert.Remediation == true && Alert.GetScope() == "Range" + decisions: + - type: ban + duration: 4h + notifications: + - ntfy_plugin + on_success: break + ''; + mode = "0440"; + user = "crowdsec"; + group = "crowdsec"; + }; + + # Inject NTFY_USER and NTFY_PASSWORD into the crowdsec service so the + # HTTP plugin template can reference them. The plugin config uses + # {{env "NTFY_BASIC_AUTH"}} — a pre-encoded "user:pass" base64 string + # for the Authorization: Basic header — computed in ExecStartPre. + systemd.services.crowdsec.serviceConfig.EnvironmentFile = lib.mkIf cfg.ntfy.enable [ + cfg.ntfy.envFile + ]; }; }; in diff --git a/modules/nixos/services/grafana/default.nix b/modules/nixos/services/grafana/default.nix index ddc5cee..fce4e57 100755 --- a/modules/nixos/services/grafana/default.nix +++ b/modules/nixos/services/grafana/default.nix @@ -151,7 +151,7 @@ let ''; giteaPort = config.${namespace}.services.gitea.port; - resticPort = config.${namespace}.services.restic.port; + resticPort = config.${namespace}.services.restic-server.port; nextcloudPort = config.${namespace}.services.nextcloud.port; grafanaConfig = lib.${namespace}.mkModule { @@ -392,27 +392,462 @@ let httpMethod: POST timeInterval: 15s ''; - # Provide empty-but-valid alerting provisioning documents. - # Without these, the NixOS module serialises `null` YAML which - # Grafana 12's provisioner fails to parse, producing a spurious - # "data source not found" error at startup. + # --------------------------------------------------------------------------- + # Alerting provisioning + # --------------------------------------------------------------------------- alerting = { - rules.settings = { - apiVersion = 1; - groups = [ ]; - }; + # ── Contact points ────────────────────────────────────────────────── + # ntfy via the Grafana webhook contact point. Grafana POSTs a JSON + # body; ntfy accepts any body as the message text. We use the + # message template below to format it nicely. + # Basic auth credentials are read from the SOPS secret at runtime + # via Grafana's $__file{} provider. contactPoints.settings = { apiVersion = 1; - contactPoints = [ ]; - }; - policies.settings = { - apiVersion = 1; - policies = [ ]; + contactPoints = [ + { + name = "ntfy"; + receivers = [ + { + uid = "ntfy-webhook"; + type = "webhook"; + settings = { + url = "https://ntfy.mjallen.dev/grafana-alerts"; + httpMethod = "POST"; + username = "$__file{${config.sops.secrets."jallen-nas/ntfy/user".path}}"; + password = "$__file{${config.sops.secrets."jallen-nas/ntfy/password".path}}"; + # Pass alert title and state as ntfy headers via the + # custom message template (defined below). + httpHeaders = { + "Tags" = "chart,bell"; + }; + }; + disableResolveMessage = false; + } + ]; + } + ]; }; + + # ── Notification message template ─────────────────────────────────── + # Grafana sends the rendered template body as the POST body. + # ntfy treats the body as the message text. templates.settings = { apiVersion = 1; - templates = [ ]; + templates = [ + { + name = "ntfy_message"; + template = '' + {{ define "ntfy_message" -}} + {{ .CommonAnnotations.summary | default .GroupLabels.alertname }} + {{ range .Alerts -}} + Status: {{ .Status | title }} + Alert: {{ .Labels.alertname }} + Severity: {{ .Labels.severity | default "unknown" }} + Instance: {{ .Labels.instance | default "unknown" }} + {{ if .Annotations.description -}} + Details: {{ .Annotations.description }} + {{ end -}} + {{ end -}} + {{ end }} + ''; + } + ]; }; + + # ── Notification routing policy ───────────────────────────────────── + policies.settings = { + apiVersion = 1; + policies = [ + { + receiver = "ntfy"; + group_by = [ + "alertname" + "severity" + ]; + group_wait = "30s"; + group_interval = "5m"; + repeat_interval = "4h"; + routes = [ + # Critical alerts: repeat every 1h, no grouping wait + { + receiver = "ntfy"; + matchers = [ "severity = critical" ]; + group_wait = "0s"; + repeat_interval = "1h"; + } + ]; + } + ]; + }; + + # ── Alert rules ───────────────────────────────────────────────────── + rules.settings = { + apiVersion = 1; + groups = [ + { + name = "nas-system"; + folder = "NAS Alerts"; + interval = "1m"; + rules = [ + # Disk usage > 85% warning, > 95% critical + { + uid = "nas-disk-warning"; + title = "Disk usage high"; + condition = "C"; + data = [ + { + refId = "A"; + datasourceUid = "prometheus"; + model = { + expr = '' + ( + node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint!~"/boot.*"} + - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint!~"/boot.*"} + ) + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs",mountpoint!~"/boot.*"} + * 100 + ''; + intervalMs = 60000; + maxDataPoints = 43200; + refId = "A"; + }; + } + { + refId = "B"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + refId = "B"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "threshold"; + refId = "C"; + expression = "B"; + conditions = [ + { + evaluator = { + type = "gt"; + params = [ 85 ]; + }; + } + ]; + }; + } + ]; + noDataState = "NoData"; + execErrState = "Error"; + for = "5m"; + annotations = { + summary = "Disk usage above 85%"; + description = "Filesystem {{ $labels.mountpoint }} is {{ $values.B | printf \"%.1f\" }}% full."; + }; + labels = { + severity = "warning"; + }; + isPaused = false; + } + + # Memory usage > 90% + { + uid = "nas-memory-high"; + title = "Memory usage high"; + condition = "C"; + data = [ + { + refId = "A"; + datasourceUid = "prometheus"; + model = { + expr = '' + (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 + ''; + intervalMs = 60000; + maxDataPoints = 43200; + refId = "A"; + }; + } + { + refId = "B"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + refId = "B"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "threshold"; + refId = "C"; + expression = "B"; + conditions = [ + { + evaluator = { + type = "gt"; + params = [ 90 ]; + }; + } + ]; + }; + } + ]; + noDataState = "NoData"; + execErrState = "Error"; + for = "5m"; + annotations = { + summary = "Memory usage above 90%"; + description = "Memory usage is {{ $values.B | printf \"%.1f\" }}%."; + }; + labels = { + severity = "warning"; + }; + isPaused = false; + } + + # CPU > 90% sustained for 10m + { + uid = "nas-cpu-high"; + title = "CPU usage sustained high"; + condition = "C"; + data = [ + { + refId = "A"; + datasourceUid = "prometheus"; + model = { + expr = '' + 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) + ''; + intervalMs = 60000; + maxDataPoints = 43200; + refId = "A"; + }; + } + { + refId = "B"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + refId = "B"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "threshold"; + refId = "C"; + expression = "B"; + conditions = [ + { + evaluator = { + type = "gt"; + params = [ 90 ]; + }; + } + ]; + }; + } + ]; + noDataState = "NoData"; + execErrState = "Error"; + for = "10m"; + annotations = { + summary = "CPU sustained above 90%"; + description = "CPU usage has been above 90% for 10 minutes (currently {{ $values.B | printf \"%.1f\" }}%)."; + }; + labels = { + severity = "warning"; + }; + isPaused = false; + } + + # UPS on battery (network_ups_tools_ups_status == 0 means OB/on-battery) + { + uid = "nas-ups-onbatt"; + title = "UPS on battery"; + condition = "C"; + data = [ + { + refId = "A"; + datasourceUid = "prometheus"; + model = { + expr = "network_ups_tools_ups_status"; + intervalMs = 60000; + maxDataPoints = 43200; + refId = "A"; + }; + } + { + refId = "B"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + refId = "B"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "threshold"; + refId = "C"; + expression = "B"; + # status 0 = OB (on battery), 1 = OL (online) + conditions = [ + { + evaluator = { + type = "lt"; + params = [ 1 ]; + }; + } + ]; + }; + } + ]; + noDataState = "NoData"; + execErrState = "Error"; + for = "1m"; + annotations = { + summary = "UPS is running on battery"; + description = "Mains power failure detected. UPS battery charge: {{ with query \"network_ups_tools_battery_charge\" }}{{ . | first | value | printf \"%.0f\" }}%{{ end }}."; + }; + labels = { + severity = "critical"; + }; + isPaused = false; + } + + # UPS battery charge < 30% + { + uid = "nas-ups-lowbatt"; + title = "UPS battery low"; + condition = "C"; + data = [ + { + refId = "A"; + datasourceUid = "prometheus"; + model = { + expr = "network_ups_tools_battery_charge"; + intervalMs = 60000; + maxDataPoints = 43200; + refId = "A"; + }; + } + { + refId = "B"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + refId = "B"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "threshold"; + refId = "C"; + expression = "B"; + conditions = [ + { + evaluator = { + type = "lt"; + params = [ 30 ]; + }; + } + ]; + }; + } + ]; + noDataState = "NoData"; + execErrState = "Error"; + for = "2m"; + annotations = { + summary = "UPS battery charge below 30%"; + description = "UPS battery is at {{ $values.B | printf \"%.0f\" }}%. Shutdown may be imminent."; + }; + labels = { + severity = "critical"; + }; + isPaused = false; + } + + # PostgreSQL not responding + { + uid = "nas-postgres-down"; + title = "PostgreSQL down"; + condition = "C"; + data = [ + { + refId = "A"; + datasourceUid = "prometheus"; + model = { + expr = "pg_up"; + intervalMs = 60000; + maxDataPoints = 43200; + refId = "A"; + }; + } + { + refId = "B"; + datasourceUid = "__expr__"; + model = { + type = "reduce"; + refId = "B"; + expression = "A"; + reducer = "last"; + }; + } + { + refId = "C"; + datasourceUid = "__expr__"; + model = { + type = "threshold"; + refId = "C"; + expression = "B"; + conditions = [ + { + evaluator = { + type = "lt"; + params = [ 1 ]; + }; + } + ]; + }; + } + ]; + noDataState = "Alerting"; + execErrState = "Error"; + for = "2m"; + annotations = { + summary = "PostgreSQL is down"; + description = "The PostgreSQL exporter reports pg_up=0. Database may be unavailable."; + }; + labels = { + severity = "critical"; + }; + isPaused = false; + } + ]; + } + ]; + }; + muteTimings.settings = { apiVersion = 1; muteTimes = [ ]; diff --git a/secrets/nas-secrets.yaml b/secrets/nas-secrets.yaml index d3136e7..aad98bb 100644 --- a/secrets/nas-secrets.yaml +++ b/secrets/nas-secrets.yaml @@ -60,6 +60,8 @@ jallen-nas: attic-key: ENC[AES256_GCM,data:SgWW1G/1bSplv2MEShWog5Hm+vPCBJd2ggURt5K/s16Z6jRxBuFCbQ3XGHJ2bqQMymKnltIux/6zguJTqi7/THvL+YEm4ls7zOYO3XlVA5G2n1sXkX/hkZK5acRzTXFiffZfZyH/iE3kFTk+68WBmYoDqT6zwJvYDDjGvtXqCj471tzVwgmc1hewAwkCVo48z113zV6Qr6egfNN+Asj8ZL86Q6CsTFIaqqttXwdDHlS93ZeqVYIGkBfuaaKTUNgFgTJ/2B8FewDy/vNdwkbWSsAAESggvwYcVB2NHjpCzND0jExXmHuzun6ebSS8StvWt+yAJLvbBgJyKj+JmF1zWRRlKVc1UG+d6cqFgr5F5/tf2zYl9RoCM5ZFEilFzmfnl1l0AI8N3wjQw71brKb/nO1YBrlKL48vq2P1aSIau6YN/vy5+0dl55Zn9Fyt7DFpVZdwLZefE1A3CJqmRbksDCidQMYaq80v6qCB8d/vNBny7h5v0+04aXhHK9gBVj0E/RcbKHZ/XKefumxxIe3Zay+aKjhTVkTkC8JtzrpPWbcxYZsglc+1K6k3DQdism2luM5eNncjPvlnmV1VkxP3UStquJ98ot2ePLMlRVyfYIzF3/M8JUjf3I8DQuRcLD1AhduCMvlTn+x1qaRx92AHFApbWpJ8FZ44qv7QlpoVGBgSW8c4puFUMFhnMCY0uJfT4opniGZi3RSABSU1JzXWGSKgM3cqgweWnIe9oJJC7ymfRp1Vk07w/mHDxMXghoW9rAxBMaiADRGLwTxgUXDhm0f+J3zfTc2hp6dnIYvOfnQVsNbJoNeAW5oojvR4DXwq4BpFOhmHzBKszAdhGM/X1/+F4O0kb52zJzjZxGrPhnsbAoUfuFH90pCdRznm6dfST/M9v6NLE59Xig3pyEihUTMvIzZzzOSzPSIpA4URqx7llQaAo0NRpNr71c2geco0mS/WuHX82jC5fPMCXYFbuOM3dn2eyUKLdQ6b4TrpYRsHiQ5URz2kkrUPNCwdXQE4+kLpBNoBowGnDDN3fNoS0SplV0Gwkk8o9EJaG7aj6V9OqUYIP7L2Ud8UbJyaZiEQ2oeSGVjLpYGFhbPZBlK4dQ7wSSj8HRwaLrJHpmYySYSr8QnmN2p7vwHx4gi3FC4pFs1g935gNuJpwGDVNr/XIeAV2s+DCeMROz8M2x4gID6LarC1eA/0eKdL5eq47MV1+nihNEq708DJY0mY9KeVXBF/tLpF2Fd2DE/OMBvRU1osML+0U3X79dbdYs1hprzmHxM0tjJF7oRpWCLL4bB4QNNd+VB5t4fdSe0mS/6aFeVTdY6dEcQ7FsxTG3Lk0QLVzhNffdcY5UfTI62BnVaC9wdSNSRFwajRzaeBq5o2tSNO3NZKbOWsSgsdv7CwS1WSnKQaCBfx6XQxtIk9dqODdjxh3tS1XVmCUFm/gDM2w3OxnQRc4LYSB7ApeV7Haw+V6YDAgKG97UHaL6M5Bo7/iS7mI7QBrgxecjd6a8o1arVV60Q5LPvov15tqtxk65hKxvsAZjERNHamHpWjVRHNcqQIoL9FaSLoJ7FD3rJsGS+86r7NUVPnV0XtkdRilE5mydc1BltvRUqaknmmNWL5kgrgnDO02l7WZkrHr9Dv0uagfR47S/REpEPgoErd0xoqsfdeTkQviyjjEuZp3U3M5VQ/ud7T6FvR1yYRj8UWWedOstchJnInn1Pn//8lM41ZTqRD/Pd9dEN5aItFAVknp7w16GrsgCtjohlLodpU7xKSPW7r4YuZR6Jnj80W/FEVE2bhRuB0HKeAkJzEHvWpHRxa6yGzyZH+S4mqLFndHpUVQUpX/l+RH5LpkGaxjqkzbpyO3zpwHf9RgyDio7FZx57896Owb0fYvpvxd24A5xFNmNjPRNpBiMmtHlWO93tJJUV6TkdhEOa8l0Ifm/lJfAXc0WeUBWZO0PeCz7m/DeltwPKLOGVBCeHPmdu9IfLxZ4TfLSdaS3W6rbVIK2kwMfTrLa48QJqGYfqb4KZWqJInPptNnXiDoH6pjrjjwAhRwvIaClUlVJ9NWCtXSC3BJs4XA3B625ScYW0rlE4r4C2zXqr+I2xmHxFMdUadNgbuybFx59zwo+/hDmF+bovjt2Xr6b7440rt18C8s0eusFeOuqfl+e375v6iA4RFS5nIa0Xagokjvla38B/4S0OM7mN8/v01GSX7vv22U3uf081BBqd6N6tySzWEAza4HpL857EU5fPFz303J4VA2GzrxbZWJHcVUBD0lI3aTXnKPm1BJEaYRD/xHvW2l1sSZsy2/+9jg1nGS5hkY+AUHn3dRPgcm2dXUh6/5HgpSW36wSn4lUCWElp4dcf/hEjmMZaB/m9PPRaVCKy9LiVwFa/3RVAxucRxPJb54jDA+hEXNIK+hzoGbRfUlKpe20QHtr2RfskbxgUIKMq8AdMVb4XRXHKYdf7iAll3Eip1uim4HD+pQY1+erqbRSlNGy1F6s8HxK071sA48ratPqZeLLgjmv6kjnl+X61sN6zTOw1mImYeayybLPFP+jDCwxYffqX9amD0hnFRPKBTrzOPFCB3spOD+S6EdXUC6+O3vV7wE4p5/F74kDCxF0fQllIBgxhl/g5XjOTr4X1bg8CislAz72c84+/T1BGPabGLsZvH/sN+ordNzcTWO6Xw9q8nc0ChxAC7DaKu2o2scHtitWRR0rfnFt5tkGqFAZxWZ92buLD2AicwI0GA739cgWevUpOcxYjBq/h20vK+c2Si9YywKsX43yPriIKr9RObCf8BPHn3Jdz6Ws9OYmLrhA/IMXI8iJ8X35VFjpjggy6c0oBu+ip9ipUfcmck152lXRgETqzDn+ulTk5WGlSgd0Mm/Fy+QdOXi0aYat1d+l6oDVrb4fgychDktLQpcNz+oIWuUSaqYU26iiM95eoSY2B0m9uEgmYVUQ4s7vqH14BDFHXSQVqkMf8RKDRv0L2ELg8vF7Ly8cHbWbBJh52mS0UnH6PtN7SL69JEvr31HUtlDrvZ7wLagtYAGJxde6wh4zZy07+bd1EY2VVuHPU21TLs0d8qdowLKd8zAxy6ss9LNLETGYzkfqDn2cTN/f9ogr7TeuWUYHCh2LtIwGEFWLFlSfD3slul2lqb22D90J9QS91RB3YB2Lp+DU60IxDve2Of8NF3t6kTkQGYKEcbN+/QeZq1VGuu8xI3IDYm093J3Ji1Ai+t3wm/ttErcryY/aXfGSV2XRuQhorAMpTBt3aJS2fFAoA08T3A6nYPINb8lEKebmz3Som5nsOc8CISTsw5nDyUbuNA4BJ5wH70J3t3gh6Eaz86+Vw72ZzJAbWeVA4aUTAqipv1/M4uCMM98Wn9lFJtm+ZbFySpxzXJnjZ6KuKLEg7w+XunEzuo/nWMsBa9FPE/Q5s+Y1KK2y48RHFJ3L0W+NpcYRzJaUYWTa0VyEMRgWqq5UYbuSkzqWMUwDuEQKQ7z7fSDKU3vJsT0ZF9jEatg7Tu2CVSMgTeimw8RxSMcHdduOpGKS0NEz0MB2jQiIDMOnR0knWANHrnB5at/oIhup9wvRFHkn8dr4xeGH2IJ1Kvm16XXWkqUGyA9v+DnapJs8ZKQm59DZoPPeqixUWl8UIAnaGKIJw96SVmVOJ2mYhZpkrC2tm2Dx+iAYeoqxP/4PqORnK+8LzJLLvjOtXi+9T/D7G56OAerEqF5r8dNFkcoEEymuDvt6Bt/FmrFGTc7mo3iQhvbtKW+jHCa75QS2NWDaA8TfySjClYsrvMj3mBqC261V6JUTrgZ3fg4xDH1lJQ5IY4xA1Ujr2cA7ExYDRdzXWQHpqCQjJjrdCH5DyWz43ywlfertFz92AUUfyJozUuoK9rFIvNxQdO5S7QSWTEHKK6h5hgxUqBFc5bUSFryO3LhfnIbZ1gYrJkusjvcHSH9PzQ4s1o57adoetKgg/VqewlD0ZLFfrDllgKzS7dE0IG3QTmUXpZeWnW44W5cCSvRtdw3kxKM6gvyk/s+8d+HzjjopJxc0fQ+T8bu5n51VhTOQg323Ai4ds4l1VLhFgVCynbh8jzBsHi4wFabzsNbyXyN5UTuJrw4WCwcnNwmdEIIQvKTfZhbT+4mCWAR8ZiDV8yEXCfoyfT4jqVzjfZ2fsW/IQcq76L/eCFKB98ZPjV9AMmhqK7E4s6i12DSXxrSPNhoqqz4GdfvOjcs1Mvq0FEYc2UEPcqBZkLA1OSZ0qRK8wCBot9W+tiGMh3hpLNPIXs8Ar9etUp6x2ppEehPzmNj4uS+hk1BtVSUzLzu9plWiYum6bUikJaYzPJvevZ0MzgHgNu6PQJgsK/OSC+qoZ1saVaAjmMtZsvo5QVSOflTgBDHSjHfvbmgPAHBVMBvO5mH1mSEraJka2N24VW6jJeILnbOGr6ijmCd3sROl2TKn0aJWil2rWCG/Lk5pSvoypzzb1lgI3bwZanD1JmtkP8LAWg7t/nzUByQ0vQmwS0NAWIWapbbpTBHCYS84Llt1Ao49tw5vf3vEITqrW9PN1RAuSPmI/jZh0KLxvcfvoKIMNSMO9IykMxZWrOjw+am0LkK+PxLf6bYuctne3LxKmjdeoDUOR1fqM9QP/qEhM6KCPxVJra55BCGFGzB/pUM/+d8uSMTP8C9yrZDQwbgdhQpMK1rmJSVHSf63KtIkf32t/CyXbYAlwRcX7y1nixju7bwoXenImscjCYYaV70Ux0S69qfPipUj/zG8+6bbm4d8oFwBn+C/8N9+dAp4bTQQD8D9hgkk+hhDihOH++Dd/ZUDGwRa8DgRc0MQWAnDguwFwRdPsDGbdPtaotPoUBV4gpn21TpC/SMywTg5ltsOpKszs8EuA2RVdIIne6G63TxknYj9EikqjT64VfxHOacqlos/JFLjH3PDfKCaTZFruTRjiCx8pcsqwWJEGQgMepIo+ybM7isdcl3mw5TEk0YBtbWUW6VaLj/19aEbN6etqaK/8oP9br8Bn/sEmn5haSas7JtgSTwB0Nzi75VAYs2gATzwSe+l59DW2uGZFcCXowbg+/xlnr6j00DY/IbnyfCbm61YQ0VvfE0jvvAISOf9ckfKicojff4MHe8D26HL3RMjiBRyajcbcHdPPoICONc3YkKHDDVb7wpWNfhCy5MrgPD9lRAiBF3J/CTtB6up80bsM1mGZ5gFQUDqruxeXqQqF71bW7y9xphwuGbQ2YDohNBmKDVVHbNQUG9IrRaB4VTU0INzzs8SogGint97LJRM4g7aXCX3GtCb7KiCu3I0S4zccHYLiTfleRqRrHMAYXtMVzZzKL0asKr3la+s37Zb6z6VKm33BjGzU/12rTuS50XKZt8/GeuXHE2v2eiPN+6yqmuW4S6P73P1bL7XSN0LUHgcwXHp0IByTGEkblVnXFDCaX/3ITaPqO4Cg5gginnVz90JNhNaMdATjArMflpH/DqfKgnSJSaoQReii+PjPlthSu+L8fVQkFjoNE8vOkX+nqWgrp/2vyk6odyYKyxO5G0lmgDGqYyx0VIohdoe2ipgvQTvV7Yhtzp+2aYPr11d4tcnUcMjzBc6skcPKA/CAiX/7SIBUl2risDhRBCM3PRfWPcwQb7L/OoIZz6CjiT4/BRK5zEPccO/5z8OZDnbd1NIGG78Sbip5p5c9+eVj2qaqpAXMWasZbpMGAqM7Ds9Us6MspNk9qGJVN2yljoz87ukub1Kfhv3i1p3/OVkD+spjla0Ihoq1bLJTuV9yXZiHPuZnD4hFQ8VktxD6FaoZJ4alMrK548R5vRxtXu7mgeylqG/uQL8RE8MvLRuGbF+ovYQ==,iv:/Qwb4B4uS0aimH3WaVPh4D0iRhQneDZKiSes2eXR6Ws=,tag:WH7C6VGckzqSycEXfYLqkA==,type:str] ntfy: auth-users: ENC[AES256_GCM,data:5k2a8GxQ76tGFv0kSlnS2Cr3te0pkKjLlswtnK7m3JOuBMN4hFxOuleZJgy/gbcYvxtKgs5zx6l1pVJVUBnaSZxzANK/LWjbYPaM8VOkzTFxCpLWjhCOlLn0gao=,iv:7BrNN929jGkkquMVnRx1kjnDNg1F47MdCFkYK8fCPL0=,tag:lpMUK9rLmHUYOh7LSpXsVA==,type:str] + user: ENC[AES256_GCM,data:tfKPY8bKBA==,iv:c5SC7PJ4PkPiYnVspmyOertK35/5zuKnOcDz4zwUmEI=,tag:K33OQUShvGTsED5Wh3gtxg==,type:str] + password: ENC[AES256_GCM,data:IOZi7h7UYHyBTzQ=,iv:t15/MbCQzFdajQsA3fgKn7jdktOCoWs2rvFEQj6ieEI=,tag:zWb9z8x4326uJvVwqh02WQ==,type:str] matrix: client-id: ENC[AES256_GCM,data:mMpc+BsS9YYCXRrTNaCQcMMVdxw98uQdvywauYGjVV+ASalZA3PbBA==,iv:5Qzgny+6HkKFAYLckkVYsHVlhp0GuI96PPMjVx6RRZI=,tag:5LlLg3nnyHy9ak2VT1+hMQ==,type:str] client-secret: ENC[AES256_GCM,data:mH83GAgAziN0CMy/GuSGCTrm0wyopzvrxw1xkA7aBDSdP7N0ZYkfJ5et7daB+5jew+bbVA/Gy8aO1U2/rJ4FhRr5C0XhayHs1fT1sZBel904OHboTGRpy+eg4H+RSaA6WYWk5HRKH2ZcAfMa1jOqnbqol3+P96KpIPiMotDGL/c=,iv:mg8XbHu4ZkYICDjK2Q88SXt1Gl9IdbehFZyKES8OU50=,tag:UBnysN2qgIg53GRzbog7+A==,type:str] @@ -241,8 +243,8 @@ sops: L0gwQm5takNjMkVGNzVlSStJYlUwWDAKP8QA3rRUHYbyyhPC/k0Eq2EIKfjyc7Co 7BkHH3msC6h9g42BB5iIYe6KQ+UGxMQBFvp+qSB27jaIfajN5MP0BA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-03-24T14:56:54Z" - mac: ENC[AES256_GCM,data:vT6sECqPM0XBCfsuGqeUe4szq6dqT8Hp/yDxWBKyLca7/rROc8qTdEll/aq060TJb98HrPcteKBIy3UxAUiRJOEfpn8XyUkiEGZuLzbesrq7qOkGOCVHKQk0Us8PM3kL9s5VIj9BWaCHS8GfdyboIP+1+Wn6ZxO+nDyzWXnSRKE=,iv:r+wQTeYtIydw3T0QDhcen7hQF0g93eBjfPY7DZuHU3o=,tag:xpM6YZxDoH8ZLYBesH4m1w==,type:str] + lastmodified: "2026-03-24T19:34:00Z" + mac: ENC[AES256_GCM,data:vuAtq87oIAIbWw7d/PwNPfVIlWJx3C1sT5tlpgLvHTgCAp8ZtSSBhqTJNCV7za+kjrF/SrZ0Asc1Ui6W3spgU6QAu2TR7JtAseXDyN7c8hLbBzZdZ5RtrqYdr8bj+dohbC9ZTHBt1YH7iXb1rm3CmwkCkzSY6ux3afnWubK2TIg=,iv:UAnH3J0Lnyxk1XPk3Zlm/VpCSA7omv9czc50C9qrfsM=,tag:Xh+iUD9S2rSlpmUoa5uQ8A==,type:str] pgp: - created_at: "2026-02-06T15:34:30Z" enc: |- diff --git a/systems/x86_64-linux/jallen-nas/apps.nix b/systems/x86_64-linux/jallen-nas/apps.nix index dff63ef..c21d249 100755 --- a/systems/x86_64-linux/jallen-nas/apps.nix +++ b/systems/x86_64-linux/jallen-nas/apps.nix @@ -87,6 +87,10 @@ in enable = true; port = 8181; apiKey = config.sops.secrets."jallen-nas/crowdsec-capi".path; + ntfy = { + enable = true; + envFile = config.sops.templates."ntfy.env".path; + }; }; dispatcharr = { enable = false; @@ -208,7 +212,7 @@ in smtpPort = 1025; imapPort = 1143; }; - restic = { + restic-server = { enable = true; port = 8008; }; diff --git a/systems/x86_64-linux/jallen-nas/default.nix b/systems/x86_64-linux/jallen-nas/default.nix index c201e6a..38aa4e6 100755 --- a/systems/x86_64-linux/jallen-nas/default.nix +++ b/systems/x86_64-linux/jallen-nas/default.nix @@ -192,7 +192,13 @@ in # # Power # # # ################################################### - power.ups = enabled; + power.ups = { + enable = true; + ntfy = { + enable = true; + envFile = config.sops.templates."ntfy.env".path; + }; + }; # ################################################### # # Samba # # @@ -304,31 +310,34 @@ in # Configure environment environment = { - systemPackages = with pkgs; [ - attic-client - bcachefs-tools - cryptsetup - clevis - deconz - duperemove - efibootmgr - ffmpeg - ipset - keyutils - nut - packagekit - pass - protonmail-bridge - protonvpn-gui - qrencode - sbctl - systemctl-tui - tigervnc - tpm2-tools - tpm2-tss - ] ++ (with pkgs.${namespace}; [ - nebula-sign-cert - ]); + systemPackages = + with pkgs; + [ + attic-client + bcachefs-tools + cryptsetup + clevis + deconz + duperemove + efibootmgr + ffmpeg + ipset + keyutils + nut + packagekit + pass + protonmail-bridge + protonvpn-gui + qrencode + sbctl + systemctl-tui + tigervnc + tpm2-tools + tpm2-tss + ] + ++ (with pkgs.${namespace}; [ + nebula-sign-cert + ]); persistence."/media/nas/main/persist" = { hideMounts = true; directories = [ diff --git a/systems/x86_64-linux/jallen-nas/disabled.nix b/systems/x86_64-linux/jallen-nas/disabled.nix index 88bc3ca..2a75ebb 100644 --- a/systems/x86_64-linux/jallen-nas/disabled.nix +++ b/systems/x86_64-linux/jallen-nas/disabled.nix @@ -55,7 +55,7 @@ in paperless = mkForce disabled; paperless-ai = mkForce disabled; protonmail-bridge = mkForce disabled; - restic = mkForce disabled; + restic-server = mkForce disabled; sunshine = mkForce disabled; tdarr = mkForce disabled; unmanic = mkForce disabled; diff --git a/systems/x86_64-linux/jallen-nas/nas-defaults.nix b/systems/x86_64-linux/jallen-nas/nas-defaults.nix index 0d204c0..f348525 100644 --- a/systems/x86_64-linux/jallen-nas/nas-defaults.nix +++ b/systems/x86_64-linux/jallen-nas/nas-defaults.nix @@ -72,7 +72,7 @@ in "paperless" "paperless-ai" "protonmail-bridge" - "restic" + "restic-server" "sparky-fitness" "sparky-fitness-server" "sunshine" diff --git a/systems/x86_64-linux/jallen-nas/sops.nix b/systems/x86_64-linux/jallen-nas/sops.nix index 08af039..e2f55dd 100755 --- a/systems/x86_64-linux/jallen-nas/sops.nix +++ b/systems/x86_64-linux/jallen-nas/sops.nix @@ -259,6 +259,28 @@ in "jallen-nas/ntfy/auth-users" = { sopsFile = defaultSops; }; + "jallen-nas/ntfy/user" = { + sopsFile = defaultSops; + mode = "0440"; + owner = "grafana"; + group = "keys"; + restartUnits = [ + "grafana.service" + "crowdsec.service" + "upsmon.service" + ]; + }; + "jallen-nas/ntfy/password" = { + sopsFile = defaultSops; + mode = "0440"; + owner = "grafana"; + group = "keys"; + restartUnits = [ + "grafana.service" + "crowdsec.service" + "upsmon.service" + ]; + }; # ------------------------------ # sparky-fitness @@ -330,6 +352,20 @@ in restartUnits = [ "podman-authenticRac.service" ]; }; + "ntfy.env" = { + content = '' + NTFY_USER=${config.sops.placeholder."jallen-nas/ntfy/user"} + NTFY_PASSWORD=${config.sops.placeholder."jallen-nas/ntfy/password"} + ''; + mode = "0600"; + restartUnits = [ + "crowdsec.service" + "upsmon.service" + "nix-rebuild-cache.service" + "update-qwen-model.service" + ]; + }; + "paperless.env" = { content = '' PAPERLESS_ADMIN_USER = "mjallen"