diff options
Diffstat (limited to 'nixos/shared/monitoring.nix')
| -rw-r--r-- | nixos/shared/monitoring.nix | 231 |
1 files changed, 231 insertions, 0 deletions
diff --git a/nixos/shared/monitoring.nix b/nixos/shared/monitoring.nix new file mode 100644 index 0000000..8711630 --- /dev/null +++ b/nixos/shared/monitoring.nix @@ -0,0 +1,231 @@ +{ config, lib, pkgs, ... }: + +let + cfg = config.monitoring; + helpers = import <top/helpers.nix> { inherit config lib pkgs; }; +in +{ + options.monitoring = { + lokiPort = lib.mkOption { + type = lib.types.int; + }; + alloyUiPort = lib.mkOption { + type = lib.types.int; + }; + prometheusNodeExporterPort = lib.mkOption { + type = lib.types.int; + }; + prometheusScrapeConfigs = lib.mkOption { + type = lib.types.listOf lib.types.attrs; + default = []; + }; + }; + config = { + services.prometheus = { + enable = true; + + retentionTime = "1y"; + + scrapeConfigs = [ + { + job_name = "node"; + static_configs = [{ + targets = [ "localhost:${toString cfg.prometheusNodeExporterPort}" ]; + }]; + } + ] ++ cfg.prometheusScrapeConfigs; + + exporters.node = { + enable = true; + enabledCollectors = [ "systemd" ]; + port = cfg.prometheusNodeExporterPort; + }; + }; + + services.loki = { + enable = true; + configuration = { + server.http_listen_port = cfg.lokiPort; + auth_enabled = false; + + ingester = { + lifecycler = { + address = "127.0.0.1"; + ring = { + kvstore.store = "inmemory"; + replication_factor = 1; + }; + }; + }; + + schema_config = { + configs = [{ + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + }]; + }; + + storage_config = { + tsdb_shipper = { + active_index_directory = "/var/lib/loki/tsdb-active"; + cache_location = "/var/lib/loki/tsdb-cache"; + }; + }; + + compactor = { + working_directory = "/var/lib/loki"; + }; + + limits_config = { + allow_structured_metadata = true; + }; + }; + }; + + systemd.services.alloy = { + serviceConfig = { + SupplementaryGroups = [ + "systemd-journal" + ] ++ lib.optional config.services.nginx.enable config.services.nginx.group; + }; + }; + + services.alloy = { + enable = true; + extraFlags = ["--server.http.listen-addr=0.0.0.0:${toString cfg.alloyUiPort}"]; + configPath = + let + ref = helpers.alloyConfigRef; + in + helpers.writeAlloyConfig { + "loki.source.journal".journal = { + max_age = "12h0m0s"; + relabel_rules = ref "discovery.relabel.journal.rules"; + forward_to = [(ref "loki.process.journal.receiver")]; + labels = { + host = "tente"; + job = "systemd-journal"; + }; + }; + "loki.process".journal = { + forward_to = [(ref "loki.write.default.receiver")]; + blocks = [ + { + name = "stage.match"; + # Select messages from systemd services that have LogExtraFields=LOG_FORMAT=logfmt. + selector = ''{__journal_LOG_FORMAT="logfmt"}''; + blocks = [ + { name = "stage.logfmt"; mapping = { time = ""; level = ""; }; } + { name = "stage.timestamp"; source = "time"; format = "RFC3339"; } + { + # The slog package of the Go standard library prints levels as uppercase. + name = "stage.template"; + source = "level"; + template = "{{ ToLower .Value }}"; + } + { name = "stage.structured_metadata"; values = { level = ""; }; } + ]; + } + ]; + }; + "discovery.relabel".journal = { + targets = []; + blocks = [ + { + name = "rule"; + source_labels = ["__journal__systemd_unit"]; + target_label = "unit"; + } + ]; + }; + + "loki.source.file".nginx_access = { + targets = ref "local.file_match.nginx_access.targets"; + forward_to = [(ref "loki.process.nginx_access.receiver")]; + }; + "local.file_match".nginx_access = { + path_targets = [{ + __path__ = "/var/log/nginx/*.access.log"; + }]; + }; + "loki.process".nginx_access = { + forward_to = [(ref "loki.write.default.receiver")]; + blocks = [ + { name = "stage.static_labels"; values = { job = "nginx"; }; } + + { + # Extracting the log file name as vhost because it's more convenient + # to query for than the full filename. We could also use server_name + # but there could be wildcard server_names and Loki labels should have + # a low cardinality for performance reasons. + name = "stage.regex"; + source = "filename"; + expression = "(?P<vhost>[^/]+)\\.access\\.log$"; + } + + { name = "stage.labels"; values = { vhost = ""; }; } + { name = "stage.json"; expressions = { msec = ""; path = ""; }; } + { name = "stage.timestamp"; source = "msec"; format = "Unix"; } + { + # Setting level=info to prevent Loki's log level detection from wrongly + # detecting messages with paths containing "error" as errors. + # Creating the filetype entry via stage.template because there's no + # static_structured_metadata stage yet. (https://github.com/grafana/loki/issues/16703) + name = "stage.template"; + source = "level"; + template = "info"; + } + { name = "stage.structured_metadata"; values = { level = ""; }; } + + # Temporarily adding path as a label so that we can use it in the match selectors. + { name = "stage.labels"; values = { path = ""; }; } + { + name = "stage.match"; + selector = "{path=~\"/\\\\.well-known/.*\"}"; + # Creating the filetype entry via stage.template because there's no + # static_structured_metadata stage yet. (https://github.com/grafana/loki/issues/16703) + blocks = [ + { name = "stage.template"; source = "filetype"; template = "well-known"; } + ]; + } + { + name = "stage.match"; + selector = "{path=\"/robots.txt\"}"; + blocks = [ + { name = "stage.template"; source = "filetype"; template = "robots.txt"; } + ]; + } + { + name = "stage.match"; + selector = "{path=~\".*\\\\.atom$\"}"; + blocks = [ + { name = "stage.template"; source = "filetype"; template = "feed"; } + ]; + } + { + name = "stage.structured_metadata"; + values = { filetype = ""; }; + } + + # Dropping path again because it has a too high cardinality for a label. + { name = "stage.label_drop"; values = ["path"]; } + ]; + }; + "loki.write".default = { + blocks = [ + { + name = "endpoint"; + url = "http://127.0.0.1:${toString cfg.lokiPort}/loki/api/v1/push"; + } + ]; + external_labels = {}; + }; + }; + }; + }; +} |