feat(tente): report nginx access logs to Loki

3 files changed, 113 insertions, 1 deletions

diff --git a/nixos/helpers.nix b/nixos/helpers.nix
index 0588cd6..e7e1877 100644
--- a/nixos/helpers.nix
+++ b/nixos/helpers.nix
@@ -3,10 +3,50 @@ let
 in
 {
   mkNginxConfig = name: ''
-    access_log /var/log/nginx/${name}.access.log;
+    access_log /var/log/nginx/${name}.access.log json;
     error_log /var/log/nginx/${name}.error.log;
   '';
 
+  commonHttpConfig = ''
+    map $request_uri $request_uri_path {
+      "~^(?P<path>[^?]*)(\?.*)?$"  $path;
+    }
+    log_format json escape=json '{'
+      '"msec": "$msec", ' # request unixtime in seconds with a milliseconds resolution
+      '"remote_addr": "$remote_addr", ' # client IP
+      '"remote_user": "$remote_user", ' # client HTTP username
+      '"method": "$request_method", '
+      '"path": "$request_uri_path", '
+      '"args": "$args", '
+      '"status": "$status", ' # response status code
+      '"user_agent": "$http_user_agent", '
+      '"body_bytes_sent": "$body_bytes_sent", ' # the number of body bytes exclude headers sent to a client
+      '"request_time": "$request_time", ' # request processing time in seconds with msec resolution
+      '"http_host": "$http_host", ' # the request Host: header
+      '"http_referer": "$http_referer", ' # HTTP referer
+      '"http_x_forwarded_for": "$http_x_forwarded_for", ' # http_x_forwarded_for
+      '"connection": "$connection", ' # connection serial number
+      '"connection_requests": "$connection_requests", ' # number of requests made in connection
+      '"request_id": "$request_id", ' # the unique request id
+      '"request_length": "$request_length", ' # request length (including headers and body)
+      '"time_iso8601": "$time_iso8601", ' # local time in the ISO 8601 standard format
+      '"bytes_sent": "$bytes_sent", ' # the number of bytes sent to a client
+      '"server_name": "$server_name", ' # the name of the vhost serving the request
+      '"upstream": "$upstream_addr", ' # upstream backend server for proxied requests
+      '"upstream_connect_time": "$upstream_connect_time", ' # upstream handshake time incl. TLS
+      '"upstream_header_time": "$upstream_header_time", ' # time spent receiving upstream headers
+      '"upstream_response_time": "$upstream_response_time", ' # time spend receiving upstream body
+      '"upstream_response_length": "$upstream_response_length", ' # upstream response length
+      '"upstream_cache_status": "$upstream_cache_status", ' # cache HIT/MISS where applicable
+      '"ssl_protocol": "$ssl_protocol", ' # TLS protocol
+      '"ssl_cipher": "$ssl_cipher", ' # TLS cipher
+      '"scheme": "$scheme", ' # http or https
+      '"server_protocol": "$server_protocol", ' # request protocol, like HTTP/1.1 or HTTP/2.0
+      '"pipe": "$pipe", ' # "p" if request was pipelined, "." otherwise
+      '"gzip_ratio": "$gzip_ratio"'
+    '}';
+  '';
+
   joinWgNamespace = ns: cfg:
    nixpkgs.lib.attrsets.recursiveUpdate cfg {
     bindsTo = ["netns@${ns}.service"];
diff --git a/nixos/hosts/tente/default.nix b/nixos/hosts/tente/default.nix
index 07b70a4..f910876 100644
--- a/nixos/hosts/tente/default.nix
+++ b/nixos/hosts/tente/default.nix
@@ -136,6 +136,8 @@ in
           return 444;
         }
       '';
+
+      commonHttpConfig = helpers.commonHttpConfig;
     };
   };
 
diff --git a/nixos/hosts/tente/monitoring.nix b/nixos/hosts/tente/monitoring.nix
index bbf4439..c761018 100644
--- a/nixos/hosts/tente/monitoring.nix
+++ b/nixos/hosts/tente/monitoring.nix
@@ -104,6 +104,19 @@ in
         compactor = {
           working_directory = "/var/lib/loki";
         };
+
+        limits_config = {
+          allow_structured_metadata = true;
+        };
+      };
+    };
+
+    systemd.services.alloy = {
+      serviceConfig = {
+        SupplementaryGroups = [
+          "systemd-journal"
+          "www-data"
+        ];
       };
     };
 
@@ -131,6 +144,63 @@ in
           }
         }
 
+        loki.source.file "nginx_access" {
+          targets = local.file_match.nginx_access.targets
+          forward_to = [loki.process.nginx_access.receiver]
+        }
+
+        local.file_match "nginx_access" {
+          path_targets = [{
+            __path__ = "/var/log/nginx/*.access.log",
+          }]
+        }
+
+        loki.process "nginx_access" {
+          forward_to = [loki.write.default.receiver]
+
+          stage.static_labels {
+            values = {
+              job = "nginx",
+            }
+          }
+
+          // Extracting the log file name as vhost because it's more convenient
+          // to query for than the full filename. We could also use server_name
+          // but there could be wildcard server_names and Loki labels should have
+          // a low cardinality for performance reasons.
+          stage.regex {
+            source = "filename"
+            expression = "(?P<vhost>[^/]+)\\.access\\.log$"
+          }
+
+          stage.labels {
+            values = {
+              vhost = "",
+            }
+          }
+
+          stage.json {
+            expressions = { "msec" = "" }
+          }
+
+          stage.timestamp {
+            source = "msec"
+            format = "Unix"
+          }
+
+          // Setting level=info to prevent Loki's log level detection from wrongly
+          // detecting messages with paths containing "error" as errors.
+          // Creating the filetype entry via stage.template because there's no
+          // static_structured_metadata stage yet. (https://github.com/grafana/loki/issues/16703)
+          stage.template {
+            source = "level"
+            template = "info"
+          }
+          stage.structured_metadata {
+            values = { level = "" }
+          }
+        }
+
         loki.write "default" {
           endpoint {
             url = "http://127.0.0.1:${toString cfg.lokiPort}/loki/api/v1/push"