refactor: make Prometheus & Loki config reusable

4 files changed, 125 insertions, 321 deletions

diff --git a/nixos/hosts/tente/default.nix b/nixos/hosts/tente/default.nix
index e63b9bc..b9ac389 100644
--- a/nixos/hosts/tente/default.nix
+++ b/nixos/hosts/tente/default.nix
@@ -18,11 +18,13 @@ rec {
     <top/profiles/server>
     <top/shared/postgresql.nix>
     <top/shared/tailscale.nix>
+    <top/shared/monitoring.nix>
     ./web-personal.nix
     ./git.nix
     ./headscale.nix
     ./matrix.nix
-    ./monitoring.nix
+    ./exporters.nix
+    ./grafana.nix
     "${sources.my-lex-surf}/service.nix"
     "${sources.my-osm-proposals}/service.nix"
     "${sources.my-geopos-link}/service.nix"
@@ -115,17 +117,16 @@ rec {
       };
     };
 
-  monitoring.grafanaUiPort = 3000;
+  grafana.port = 3000;
+  grafana.matrixForwarderPort = 3002;
+  grafana.matrixServerUrl = "http://localhost:${toString matrix.port}";
   monitoring.alloyUiPort = 3001;
-  monitoring.grafanaMatrixForwarderPort = 3002;
   monitoring.lokiPort = 3030;
   monitoring.prometheusNodeExporterPort = 9002;
-  monitoring.prometheusSqlExporterPort = 9003;
+  exporters.sqlExporterPort = 9003;
   headscale.port = 8080;
   matrix.port = 8008;
 
-  monitoring.matrixServerUrl = "http://localhost:${toString matrix.port}";
-
   # Use the GRUB 2 boot loader.
   boot.loader.grub.enable = true;
   # boot.loader.grub.efiSupport = true;
diff --git a/nixos/hosts/tente/exporters.nix b/nixos/hosts/tente/exporters.nix
new file mode 100644
index 0000000..1ed53e5
--- /dev/null
+++ b/nixos/hosts/tente/exporters.nix
@@ -0,0 +1,55 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.exporters;
+in
+{
+  options.exporters = {
+    sqlExporterPort = lib.mkOption {
+      type = lib.types.int;
+    };
+  };
+
+  imports = [
+    <top/shared/prometheus-sql-exporter/service.nix>
+  ];
+
+  config = {
+    services.prometheus-sql-exporter = {
+      enable = true;
+      port = cfg.sqlExporterPort;
+      config = {
+        target = {
+          # This URL should be postgresql:///postgres?host=/run/postgresql
+          # but sql_exporter uses xo/dburl which isn't spec-compliant: https://github.com/xo/dburl/issues/46
+          data_source_name = "postgresql:/run/postgresql:/postgres";
+          collectors = ["db-sizes"];
+        };
+        collectors = [
+          {
+            collector_name = "db-sizes";
+            metrics = [
+              {
+                metric_name = "pg_db_size_bytes";
+                help = "disk space used by the database";
+                type = "gauge";
+                key_labels = ["database_name"];
+                values = ["size"];
+                query = "SELECT datname AS database_name, pg_database_size(datname) as size from pg_database";
+              }
+            ];
+          }
+        ];
+      };
+    };
+
+    monitoring.prometheusScrapeConfigs = [
+      {
+        job_name = "sql";
+        static_configs = [{
+          targets = [ "localhost:${toString cfg.sqlExporterPort}" ];
+        }];
+      }
+    ];
+  };
+}
diff --git a/nixos/hosts/tente/grafana.nix b/nixos/hosts/tente/grafana.nix
new file mode 100644
index 0000000..e6fb7af
--- /dev/null
+++ b/nixos/hosts/tente/grafana.nix
@@ -0,0 +1,63 @@
+{ config, lib, pkgs, ... }:
+
+let
+  cfg = config.grafana;
+in
+{
+  options.grafana = {
+    port = lib.mkOption {
+      type = lib.types.int;
+    };
+    matrixForwarderPort = lib.mkOption {
+      type = lib.types.int;
+    };
+    prometheusSqlExporterPort = lib.mkOption {
+      type = lib.types.int;
+    };
+    matrixServerUrl = lib.mkOption {
+      type = lib.types.str;
+    };
+  };
+
+  imports = [
+    <top/shared/prometheus-sql-exporter/service.nix>
+    <top/shared/grafana-matrix-forwarder/service.nix>
+  ];
+
+  config = {
+    age.secrets.grafana-matrix-forwarder-env.file = <top/secrets/grafana-matrix-forwarder-env.age>;
+
+    services.grafana = {
+      enable = true;
+      settings = {
+        server = {
+          http_addr = "0.0.0.0";
+          http_port = cfg.port;
+        };
+      };
+
+      provision = {
+        enable = true;
+        datasources.settings.datasources = [
+          {
+            name = "Prometheus Tente";
+            type = "prometheus";
+            url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
+          }
+          {
+            name = "Loki Tente";
+            type = "loki";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
+          }
+        ];
+      };
+    };
+    services.grafana-matrix-forwarder = {
+      enable = true;
+      port = cfg.matrixForwarderPort;
+      homeserver = cfg.matrixServerUrl;
+      environmentFile = config.age.secrets.grafana-matrix-forwarder-env.path;
+    };
+  };
+}
diff --git a/nixos/hosts/tente/monitoring.nix b/nixos/hosts/tente/monitoring.nix
deleted file mode 100644
index f6ed7cf..0000000
--- a/nixos/hosts/tente/monitoring.nix
+++ /dev/null
@@ -1,315 +0,0 @@
-{ config, lib, pkgs, ... }:
-
-let
-  cfg = config.monitoring;
-  helpers = import <top/helpers.nix> { inherit config lib pkgs; };
-in
-{
-  options.monitoring = {
-    grafanaUiPort = lib.mkOption {
-      type = lib.types.int;
-    };
-    grafanaMatrixForwarderPort = lib.mkOption {
-      type = lib.types.int;
-    };
-    lokiPort = lib.mkOption {
-      type = lib.types.int;
-    };
-    alloyUiPort = lib.mkOption {
-      type = lib.types.int;
-    };
-    prometheusNodeExporterPort = lib.mkOption {
-      type = lib.types.int;
-    };
-    prometheusSqlExporterPort = lib.mkOption {
-      type = lib.types.int;
-    };
-    matrixServerUrl = lib.mkOption {
-      type = lib.types.str;
-    };
-  };
-
-  imports = [
-    <top/shared/prometheus-sql-exporter/service.nix>
-    <top/shared/grafana-matrix-forwarder/service.nix>
-  ];
-
-  config = {
-    age.secrets.grafana-matrix-forwarder-env.file = <top/secrets/grafana-matrix-forwarder-env.age>;
-
-    services.grafana = {
-      enable = true;
-      settings = {
-        server = {
-          http_addr = "0.0.0.0";
-          http_port = cfg.grafanaUiPort;
-        };
-      };
-
-      provision = {
-        enable = true;
-        datasources.settings.datasources = [
-          {
-            name = "Prometheus";
-            type = "prometheus";
-            url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
-          }
-          {
-            name = "Loki";
-            type = "loki";
-            access = "proxy";
-            url = "http://127.0.0.1:${toString cfg.lokiPort}";
-          }
-        ];
-      };
-    };
-    services.grafana-matrix-forwarder = {
-      enable = true;
-      port = cfg.grafanaMatrixForwarderPort;
-      homeserver = cfg.matrixServerUrl;
-      environmentFile = config.age.secrets.grafana-matrix-forwarder-env.path;
-    };
-
-    services.prometheus = {
-      enable = true;
-
-      retentionTime = "1y";
-
-      scrapeConfigs = [
-        {
-          job_name = "node";
-          static_configs = [{
-            targets = [ "localhost:${toString cfg.prometheusNodeExporterPort}" ];
-          }];
-        }
-        {
-          job_name = "sql";
-          static_configs = [{
-            targets = [ "localhost:${toString cfg.prometheusSqlExporterPort}" ];
-          }];
-        }
-      ];
-
-      exporters.node = {
-        enable = true;
-        enabledCollectors = [ "systemd" ];
-        port = cfg.prometheusNodeExporterPort;
-      };
-    };
-
-    services.prometheus-sql-exporter = {
-      enable = true;
-      port = cfg.prometheusSqlExporterPort;
-      config = {
-        target = {
-          # This URL should be postgresql:///postgres?host=/run/postgresql
-          # but sql_exporter uses xo/dburl which isn't spec-compliant: https://github.com/xo/dburl/issues/46
-          data_source_name = "postgresql:/run/postgresql:/postgres";
-          collectors = ["db-sizes"];
-        };
-        collectors = [
-          {
-            collector_name = "db-sizes";
-            metrics = [
-              {
-                metric_name = "pg_db_size_bytes";
-                help = "disk space used by the database";
-                type = "gauge";
-                key_labels = ["database_name"];
-                values = ["size"];
-                query = "SELECT datname AS database_name, pg_database_size(datname) as size from pg_database";
-              }
-            ];
-          }
-        ];
-      };
-    };
-
-    services.loki = {
-      enable = true;
-      configuration = {
-        server.http_listen_port = cfg.lokiPort;
-        auth_enabled = false;
-
-        ingester = {
-          lifecycler = {
-            address = "127.0.0.1";
-            ring = {
-              kvstore.store = "inmemory";
-              replication_factor = 1;
-            };
-          };
-        };
-
-        schema_config = {
-          configs = [{
-            store = "tsdb";
-            object_store = "filesystem";
-            schema = "v13";
-            index = {
-              prefix = "index_";
-              period = "24h";
-            };
-          }];
-        };
-
-        storage_config = {
-          tsdb_shipper = {
-            active_index_directory = "/var/lib/loki/tsdb-active";
-            cache_location = "/var/lib/loki/tsdb-cache";
-          };
-        };
-
-        compactor = {
-          working_directory = "/var/lib/loki";
-        };
-
-        limits_config = {
-          allow_structured_metadata = true;
-        };
-      };
-    };
-
-    systemd.services.alloy = {
-      serviceConfig = {
-        SupplementaryGroups = [
-          "systemd-journal"
-          "www-data"
-        ];
-      };
-    };
-
-    services.alloy = {
-      enable = true;
-      extraFlags = ["--server.http.listen-addr=0.0.0.0:${toString cfg.alloyUiPort}"];
-      configPath =
-        let
-          ref = helpers.alloyConfigRef;
-        in
-        helpers.writeAlloyConfig {
-          "loki.source.journal".journal = {
-            max_age = "12h0m0s";
-            relabel_rules = ref "discovery.relabel.journal.rules";
-            forward_to = [(ref "loki.process.journal.receiver")];
-            labels = {
-              host = "tente";
-              job = "systemd-journal";
-            };
-          };
-          "loki.process".journal = {
-            forward_to = [(ref "loki.write.default.receiver")];
-            blocks = [
-              {
-                name = "stage.match";
-                # Select messages from systemd services that have LogExtraFields=LOG_FORMAT=logfmt.
-                selector = ''{__journal_LOG_FORMAT="logfmt"}'';
-                blocks = [
-                  { name = "stage.logfmt"; mapping = { time = ""; level = ""; }; }
-                  { name = "stage.timestamp"; source = "time"; format = "RFC3339"; }
-                  {
-                    # The slog package of the Go standard library prints levels as uppercase.
-                    name = "stage.template";
-                    source = "level";
-                    template = "{{ ToLower .Value }}";
-                  }
-                  { name = "stage.structured_metadata"; values = { level = ""; }; }
-                ];
-              }
-            ];
-          };
-          "discovery.relabel".journal = {
-            targets = [];
-            blocks = [
-              {
-                name = "rule";
-                source_labels = ["__journal__systemd_unit"];
-                target_label = "unit";
-              }
-            ];
-          };
-
-          "loki.source.file".nginx_access = {
-            targets = ref "local.file_match.nginx_access.targets";
-            forward_to = [(ref "loki.process.nginx_access.receiver")];
-          };
-          "local.file_match".nginx_access = {
-            path_targets = [{
-              __path__ = "/var/log/nginx/*.access.log";
-            }];
-          };
-          "loki.process".nginx_access = {
-            forward_to = [(ref "loki.write.default.receiver")];
-            blocks = [
-              { name = "stage.static_labels"; values = { job = "nginx"; }; }
-
-              {
-                # Extracting the log file name as vhost because it's more convenient
-                # to query for than the full filename. We could also use server_name
-                # but there could be wildcard server_names and Loki labels should have
-                # a low cardinality for performance reasons.
-                name = "stage.regex";
-                source = "filename";
-                expression = "(?P<vhost>[^/]+)\\.access\\.log$";
-              }
-
-              { name = "stage.labels"; values = { vhost = ""; }; }
-              { name = "stage.json"; expressions = { msec = ""; path = ""; }; }
-              { name = "stage.timestamp"; source = "msec"; format = "Unix"; }
-              {
-                # Setting level=info to prevent Loki's log level detection from wrongly
-                # detecting messages with paths containing "error" as errors.
-                # Creating the filetype entry via stage.template because there's no
-                # static_structured_metadata stage yet. (https://github.com/grafana/loki/issues/16703)
-                name = "stage.template";
-                source = "level";
-                template = "info";
-              }
-              { name = "stage.structured_metadata"; values = { level = ""; }; }
-
-              # Temporarily adding path as a label so that we can use it in the match selectors.
-              { name = "stage.labels"; values = { path = ""; }; }
-              {
-                name = "stage.match";
-                selector = "{path=~\"/\\\\.well-known/.*\"}";
-                # Creating the filetype entry via stage.template because there's no
-                # static_structured_metadata stage yet. (https://github.com/grafana/loki/issues/16703)
-                blocks = [
-                  { name = "stage.template"; source = "filetype"; template = "well-known"; }
-                ];
-              }
-              {
-                name = "stage.match";
-                selector = "{path=\"/robots.txt\"}";
-                blocks = [
-                  { name = "stage.template"; source = "filetype"; template = "robots.txt"; }
-                ];
-              }
-              {
-                name = "stage.match";
-                selector = "{path=~\".*\\\\.atom$\"}";
-                blocks = [
-                  { name = "stage.template"; source = "filetype"; template = "feed"; }
-                ];
-              }
-              {
-                name = "stage.structured_metadata";
-                values = { filetype = ""; };
-              }
-
-              # Dropping path again because it has a too high cardinality for a label.
-              { name = "stage.label_drop"; values = ["path"]; }
-            ];
-          };
-          "loki.write".default = {
-            blocks = [
-              {
-                name = "endpoint";
-                url = "http://127.0.0.1:${toString cfg.lokiPort}/loki/api/v1/push";
-              }
-            ];
-            external_labels = {};
-          };
-        };
-    };
-  };
-}