From 2631aab59a748a9f630a68226048131aa9fbd932 Mon Sep 17 00:00:00 2001
From: Martin Fischer <martin@push-f.com>
Date: Sun, 9 Mar 2025 22:28:32 +0100
Subject: feat(tente): report nginx access logs to Loki
---
nixos/helpers.nix | 42 +++++++++++++++++++++++-
nixos/hosts/tente/default.nix | 2 ++
nixos/hosts/tente/monitoring.nix | 70 ++++++++++++++++++++++++++++++++++++++++
3 files changed, 113 insertions(+), 1 deletion(-)
(limited to 'nixos')
diff --git a/nixos/helpers.nix b/nixos/helpers.nix
index 0588cd6..e7e1877 100644
--- a/nixos/helpers.nix
+++ b/nixos/helpers.nix
@@ -3,10 +3,50 @@ let
in
{
mkNginxConfig = name: ''
- access_log /var/log/nginx/${name}.access.log;
+ access_log /var/log/nginx/${name}.access.log json;
error_log /var/log/nginx/${name}.error.log;
'';
+ commonHttpConfig = ''
+ map $request_uri $request_uri_path {
+ "~^(?P<path>[^?]*)(\?.*)?$" $path;
+ }
+ log_format json escape=json '{'
+ '"msec": "$msec", ' # request unixtime in seconds with a milliseconds resolution
+ '"remote_addr": "$remote_addr", ' # client IP
+ '"remote_user": "$remote_user", ' # client HTTP username
+ '"method": "$request_method", '
+ '"path": "$request_uri_path", '
+ '"args": "$args", '
+ '"status": "$status", ' # response status code
+ '"user_agent": "$http_user_agent", '
+ '"body_bytes_sent": "$body_bytes_sent", ' # the number of body bytes exclude headers sent to a client
+ '"request_time": "$request_time", ' # request processing time in seconds with msec resolution
+ '"http_host": "$http_host", ' # the request Host: header
+ '"http_referer": "$http_referer", ' # HTTP referer
+ '"http_x_forwarded_for": "$http_x_forwarded_for", ' # http_x_forwarded_for
+ '"connection": "$connection", ' # connection serial number
+ '"connection_requests": "$connection_requests", ' # number of requests made in connection
+ '"request_id": "$request_id", ' # the unique request id
+ '"request_length": "$request_length", ' # request length (including headers and body)
+ '"time_iso8601": "$time_iso8601", ' # local time in the ISO 8601 standard format
+ '"bytes_sent": "$bytes_sent", ' # the number of bytes sent to a client
+ '"server_name": "$server_name", ' # the name of the vhost serving the request
+ '"upstream": "$upstream_addr", ' # upstream backend server for proxied requests
+ '"upstream_connect_time": "$upstream_connect_time", ' # upstream handshake time incl. TLS
+ '"upstream_header_time": "$upstream_header_time", ' # time spent receiving upstream headers
+ '"upstream_response_time": "$upstream_response_time", ' # time spend receiving upstream body
+ '"upstream_response_length": "$upstream_response_length", ' # upstream response length
+ '"upstream_cache_status": "$upstream_cache_status", ' # cache HIT/MISS where applicable
+ '"ssl_protocol": "$ssl_protocol", ' # TLS protocol
+ '"ssl_cipher": "$ssl_cipher", ' # TLS cipher
+ '"scheme": "$scheme", ' # http or https
+ '"server_protocol": "$server_protocol", ' # request protocol, like HTTP/1.1 or HTTP/2.0
+ '"pipe": "$pipe", ' # "p" if request was pipelined, "." otherwise
+ '"gzip_ratio": "$gzip_ratio"'
+ '}';
+ '';
+
joinWgNamespace = ns: cfg:
nixpkgs.lib.attrsets.recursiveUpdate cfg {
bindsTo = ["netns@${ns}.service"];
diff --git a/nixos/hosts/tente/default.nix b/nixos/hosts/tente/default.nix
index 07b70a4..f910876 100644
--- a/nixos/hosts/tente/default.nix
+++ b/nixos/hosts/tente/default.nix
@@ -136,6 +136,8 @@ in
return 444;
}
'';
+
+ commonHttpConfig = helpers.commonHttpConfig;
};
};
diff --git a/nixos/hosts/tente/monitoring.nix b/nixos/hosts/tente/monitoring.nix
index bbf4439..c761018 100644
--- a/nixos/hosts/tente/monitoring.nix
+++ b/nixos/hosts/tente/monitoring.nix
@@ -104,6 +104,19 @@ in
compactor = {
working_directory = "/var/lib/loki";
};
+
+ limits_config = {
+ allow_structured_metadata = true;
+ };
+ };
+ };
+
+ systemd.services.alloy = {
+ serviceConfig = {
+ SupplementaryGroups = [
+ "systemd-journal"
+ "www-data"
+ ];
};
};
@@ -131,6 +144,63 @@ in
}
}
+ loki.source.file "nginx_access" {
+ targets = local.file_match.nginx_access.targets
+ forward_to = [loki.process.nginx_access.receiver]
+ }
+
+ local.file_match "nginx_access" {
+ path_targets = [{
+ __path__ = "/var/log/nginx/*.access.log",
+ }]
+ }
+
+ loki.process "nginx_access" {
+ forward_to = [loki.write.default.receiver]
+
+ stage.static_labels {
+ values = {
+ job = "nginx",
+ }
+ }
+
+ // Extracting the log file name as vhost because it's more convenient
+ // to query for than the full filename. We could also use server_name
+ // but there could be wildcard server_names and Loki labels should have
+ // a low cardinality for performance reasons.
+ stage.regex {
+ source = "filename"
+ expression = "(?P<vhost>[^/]+)\\.access\\.log$"
+ }
+
+ stage.labels {
+ values = {
+ vhost = "",
+ }
+ }
+
+ stage.json {
+ expressions = { "msec" = "" }
+ }
+
+ stage.timestamp {
+ source = "msec"
+ format = "Unix"
+ }
+
+ // Setting level=info to prevent Loki's log level detection from wrongly
+ // detecting messages with paths containing "error" as errors.
+ // Creating the filetype entry via stage.template because there's no
+ // static_structured_metadata stage yet. (https://github.com/grafana/loki/issues/16703)
+ stage.template {
+ source = "level"
+ template = "info"
+ }
+ stage.structured_metadata {
+ values = { level = "" }
+ }
+ }
+
loki.write "default" {
endpoint {
url = "http://127.0.0.1:${toString cfg.lokiPort}/loki/api/v1/push"
--
cgit v1.2.3