{ config, lib, pkgs, ... }: let cfg = config.monitoring; helpers = import { inherit config lib pkgs; }; in { options.monitoring = { lokiPort = lib.mkOption { type = lib.types.int; }; alloyUiPort = lib.mkOption { type = lib.types.int; }; prometheusNodeExporterPort = lib.mkOption { type = lib.types.int; }; prometheusScrapeConfigs = lib.mkOption { type = lib.types.listOf lib.types.attrs; default = []; }; }; config = { services.prometheus = { enable = true; retentionTime = "1y"; scrapeConfigs = [ { job_name = "node"; static_configs = [{ targets = [ "localhost:${toString cfg.prometheusNodeExporterPort}" ]; }]; } ] ++ cfg.prometheusScrapeConfigs; exporters.node = { enable = true; enabledCollectors = [ "systemd" ]; port = cfg.prometheusNodeExporterPort; }; }; services.loki = { enable = true; configuration = { server.http_listen_port = cfg.lokiPort; auth_enabled = false; ingester = { lifecycler = { address = "127.0.0.1"; ring = { kvstore.store = "inmemory"; replication_factor = 1; }; }; }; schema_config = { configs = [{ store = "tsdb"; object_store = "filesystem"; schema = "v13"; index = { prefix = "index_"; period = "24h"; }; }]; }; storage_config = { tsdb_shipper = { active_index_directory = "/var/lib/loki/tsdb-active"; cache_location = "/var/lib/loki/tsdb-cache"; }; }; compactor = { working_directory = "/var/lib/loki"; }; limits_config = { allow_structured_metadata = true; }; }; }; systemd.services.alloy = { serviceConfig = { SupplementaryGroups = [ "systemd-journal" ] ++ lib.optional config.services.nginx.enable config.services.nginx.group; }; }; services.alloy = { enable = true; extraFlags = ["--server.http.listen-addr=0.0.0.0:${toString cfg.alloyUiPort}"]; configPath = let ref = helpers.alloyConfigRef; in helpers.writeAlloyConfig { "loki.source.journal".journal = { max_age = "12h0m0s"; relabel_rules = ref "discovery.relabel.journal.rules"; forward_to = [(ref "loki.process.journal.receiver")]; labels = { host = "tente"; job = "systemd-journal"; }; }; "loki.process".journal = { forward_to = [(ref "loki.write.default.receiver")]; blocks = [ { name = "stage.match"; # Select messages from systemd services that have LogExtraFields=LOG_FORMAT=logfmt. selector = ''{__journal_LOG_FORMAT="logfmt"}''; blocks = [ { name = "stage.logfmt"; mapping = { time = ""; level = ""; }; } { name = "stage.timestamp"; source = "time"; format = "RFC3339"; } { # The slog package of the Go standard library prints levels as uppercase. name = "stage.template"; source = "level"; template = "{{ ToLower .Value }}"; } { name = "stage.structured_metadata"; values = { level = ""; }; } ]; } ]; }; "discovery.relabel".journal = { targets = []; blocks = [ { name = "rule"; source_labels = ["__journal__systemd_unit"]; target_label = "unit"; } ]; }; "loki.source.file".nginx_access = { targets = ref "local.file_match.nginx_access.targets"; forward_to = [(ref "loki.process.nginx_access.receiver")]; }; "local.file_match".nginx_access = { path_targets = [{ __path__ = "/var/log/nginx/*.access.log"; }]; }; "loki.process".nginx_access = { forward_to = [(ref "loki.write.default.receiver")]; blocks = [ { name = "stage.static_labels"; values = { job = "nginx"; }; } { # Extracting the log file name as vhost because it's more convenient # to query for than the full filename. We could also use server_name # but there could be wildcard server_names and Loki labels should have # a low cardinality for performance reasons. name = "stage.regex"; source = "filename"; expression = "(?P[^/]+)\\.access\\.log$"; } { name = "stage.labels"; values = { vhost = ""; }; } { name = "stage.json"; expressions = { msec = ""; path = ""; }; } { name = "stage.timestamp"; source = "msec"; format = "Unix"; } { # Setting level=info to prevent Loki's log level detection from wrongly # detecting messages with paths containing "error" as errors. # Creating the filetype entry via stage.template because there's no # static_structured_metadata stage yet. (https://github.com/grafana/loki/issues/16703) name = "stage.template"; source = "level"; template = "info"; } { name = "stage.structured_metadata"; values = { level = ""; }; } # Temporarily adding path as a label so that we can use it in the match selectors. { name = "stage.labels"; values = { path = ""; }; } { name = "stage.match"; selector = "{path=~\"/\\\\.well-known/.*\"}"; # Creating the filetype entry via stage.template because there's no # static_structured_metadata stage yet. (https://github.com/grafana/loki/issues/16703) blocks = [ { name = "stage.template"; source = "filetype"; template = "well-known"; } ]; } { name = "stage.match"; selector = "{path=\"/robots.txt\"}"; blocks = [ { name = "stage.template"; source = "filetype"; template = "robots.txt"; } ]; } { name = "stage.match"; selector = "{path=~\".*\\\\.atom$\"}"; blocks = [ { name = "stage.template"; source = "filetype"; template = "feed"; } ]; } { name = "stage.structured_metadata"; values = { filetype = ""; }; } # Dropping path again because it has a too high cardinality for a label. { name = "stage.label_drop"; values = ["path"]; } ]; }; "loki.write".default = { blocks = [ { name = "endpoint"; url = "http://127.0.0.1:${toString cfg.lokiPort}/loki/api/v1/push"; } ]; external_labels = {}; }; }; }; }; }