prometheus

modules/nixos/services/grafana/default.nix

@@ -9,6 +9,10 @@ let
   name = "grafana";
   cfg = config.${namespace}.services.${name};
 
+  giteaPort = config.${namespace}.services.gitea.port;
+  resticPort = config.${namespace}.services.restic.port;
+  nextcloudPort = config.${namespace}.services.nextcloud.port;
+
   grafanaConfig = lib.${namespace}.mkModule {
     inherit config name;
     description = "grafana";
@@ -25,30 +29,61 @@ let
                 "diskstats"
                 "meminfo"
                 "cpu"
-                "systemd" # Ensures systemd collector is enabled
+                "systemd"
                 "processes"
               ];
               extraFlags = [
                 "--collector.filesystem.mount-points-exclude=^/(dev|proc|sys|run)($|/)"
               ];
             };
+
             libvirt = {
               enable = false;
               openFirewall = true;
             };
+
             nut = {
               enable = true;
               openFirewall = true;
               passwordPath = config.sops.secrets."jallen-nas/ups_password".path;
               nutUser = upsUser;
             };
-            # restic = {
+
-            #   enable = true;
+            # PostgreSQL — runs as the local postgres superuser via peer auth
-            #   openFirewall = true;
+            # (Unix socket, no password required).
-            #   resticPort = 8008;
+            postgres = {
-            # };
+              enable = true;
+              runAsLocalSuperUser = true;
+            };
+
+            # Redis — single exporter instance covering all four Redis servers
+            # via the multi-target scrape pattern (/scrape?target=<addr>).
+            # The exporter needs AF_INET to reach TCP Redis instances.
+            redis = {
+              enable = true;
+              # No fixed --redis.addr: multi-target mode uses ?target= param.
+            };
+
+            # MariaDB — runs as the mysql OS user so it can connect via Unix
+            # socket without a password.
+            mysqld = {
+              enable = true;
+              runAsLocalSuperUser = true;
+            };
+
+            # Nextcloud — authenticates with the admin account.
+            # passwordFile must be readable by the prometheus-nextcloud-exporter
+            # user; sops mode 0440 + group keys covers that.
+            nextcloud = {
+              enable = true;
+              url = "http://localhost:${toString nextcloudPort}";
+              username = "mjallen";
+              passwordFile = config.sops.secrets."jallen-nas/nextcloud/adminpassword".path;
+            };
           };
+
           scrapeConfigs = [
+            # ── System ──────────────────────────────────────────────────────────
             {
               job_name = "node";
               static_configs = [
@@ -57,11 +92,102 @@ let
                 }
               ];
             }
+
+            # ── UPS (NUT) ────────────────────────────────────────────────────────
             {
-              job_name = "traefik";
+              job_name = "nut";
               static_configs = [
                 {
-                  targets = [ "localhost:8082" ];
+                  targets = [ "localhost:${toString config.services.prometheus.exporters.nut.port}" ];
+                }
+              ];
+            }
+
+            # ── Databases ────────────────────────────────────────────────────────
+            {
+              job_name = "postgres";
+              static_configs = [
+                {
+                  targets = [ "localhost:${toString config.services.prometheus.exporters.postgres.port}" ];
+                }
+              ];
+            }
+            {
+              # Redis multi-target: one exporter, four Redis instances.
+              # The redis_exporter's /scrape?target= endpoint proxies each target
+              # so a single exporter process covers all servers.
+              job_name = "redis";
+              metrics_path = "/scrape";
+              static_configs = [
+                {
+                  targets = [
+                    "redis://localhost:6379" # authentik
+                    "redis://localhost:6363" # ccache
+                    "redis://localhost:6380" # manyfold
+                    "redis://localhost:6381" # onlyoffice
+                  ];
+                }
+              ];
+              relabel_configs = [
+                {
+                  source_labels = [ "__address__" ];
+                  target_label = "__param_target";
+                }
+                {
+                  source_labels = [ "__param_target" ];
+                  target_label = "instance";
+                }
+                {
+                  target_label = "__address__";
+                  replacement = "localhost:${toString config.services.prometheus.exporters.redis.port}";
+                }
+              ];
+            }
+            {
+              job_name = "mysqld";
+              static_configs = [
+                {
+                  targets = [ "localhost:${toString config.services.prometheus.exporters.mysqld.port}" ];
+                }
+              ];
+            }
+
+            # ── Application services ─────────────────────────────────────────────
+            {
+              # Caddy exposes its built-in Prometheus endpoint on port 2019.
+              job_name = "caddy";
+              static_configs = [
+                {
+                  targets = [ "localhost:2019" ];
+                }
+              ];
+            }
+            {
+              # Gitea's /metrics endpoint is protected by a Bearer token.
+              job_name = "gitea";
+              metrics_path = "/metrics";
+              bearer_token_file = config.sops.secrets."jallen-nas/gitea/metrics-key".path;
+              static_configs = [
+                {
+                  targets = [ "localhost:${toString giteaPort}" ];
+                }
+              ];
+            }
+            {
+              # restic REST server exposes Prometheus metrics at /metrics.
+              job_name = "restic";
+              metrics_path = "/metrics";
+              static_configs = [
+                {
+                  targets = [ "localhost:${toString resticPort}" ];
+                }
+              ];
+            }
+            {
+              job_name = "nextcloud";
+              static_configs = [
+                {
+                  targets = [ "localhost:${toString config.services.prometheus.exporters.nextcloud.port}" ];
                 }
               ];
             }
@@ -92,6 +218,14 @@ let
           };
         };
       };
+
+      # The redis exporter needs AF_INET to reach TCP Redis instances.
+      # The default systemd hardening only allows AF_UNIX.
+      systemd.services.prometheus-redis-exporter.serviceConfig.RestrictAddressFamilies = [
+        "AF_UNIX"
+        "AF_INET"
+        "AF_INET6"
+      ];
     };
   };
 

systems/x86_64-linux/jallen-nas/sops.nix

@@ -135,10 +135,13 @@ in
       };
       "jallen-nas/nextcloud/adminpassword" = {
         sopsFile = defaultSops;
-        mode = "0650";
+        mode = "0440";
         owner = config.users.users."${user}".name;
-        group = config.users.users."${user}".group;
+        group = "keys";
-        restartUnits = [ "nextcloud.service" ];
+        restartUnits = [
+          "nextcloud.service"
+          "prometheus-nextcloud-exporter.service"
+        ];
       };
       "jallen-nas/nextcloud/smtp_settings" = {
         sopsFile = defaultSops;

systems/x86_64-linux/jallen-nas/users.nix

@@ -43,6 +43,15 @@ in
         hashedPasswordFile = passwordFile;
       };
       root.shell = pkgs.zsh;
+
+      # Allow the Prometheus Nextcloud exporter to read its password secret.
+      prometheus-nextcloud-exporter = {
+        isSystemUser = true;
+        group = "prometheus-nextcloud-exporter";
+        extraGroups = [ "keys" ];
+      };
     };
+
+    groups.prometheus-nextcloud-exporter = { };
   };
 }