feat: improve logging

5 files changed, 39 insertions, 9 deletions

diff --git a/default.nix b/default.nix
index 897ac2e..d70e409 100644
--- a/default.nix
+++ b/default.nix
@@ -1,5 +1,5 @@
 { pkgs ? import <nixpkgs> {} }:
-with pkgs.python310Packages;
+with pkgs.python313Packages;
 
 let
   sources = import ./npins;
@@ -12,6 +12,7 @@ let
       requests
     ];
   };
+  logformat = import "${sources.logformat}" {};
 in
 buildPythonApplication rec {
   pname = "osm-proposals";
@@ -27,6 +28,7 @@ buildPythonApplication rec {
     requests
     mwparserfromhell
     pywikiapi
+    logformat
   ];
 
   postInstall = ''
diff --git a/npins/sources.json b/npins/sources.json
index 3b57f31..65b2764 100644
--- a/npins/sources.json
+++ b/npins/sources.json
@@ -1,5 +1,19 @@
 {
   "pins": {
+    "logformat": {
+      "type": "GitRelease",
+      "repository": {
+        "type": "Git",
+        "url": "https://git.push-f.com/logformat"
+      },
+      "pre_releases": false,
+      "version_upper_bound": null,
+      "release_prefix": "v",
+      "version": "v0.1.0",
+      "revision": "bded757ac8b71df61bef068c320c639ab64f2f06",
+      "url": null,
+      "hash": "16x5fnzvkxw52j288a791zvy66rj5cq7i5dvi9wpnn795l82jz9h"
+    },
     "pywikiapi": {
       "type": "PyPi",
       "name": "pywikiapi",
diff --git a/osm_proposals/proposals.py b/osm_proposals/proposals.py
index 1bb1f14..edcb453 100755
--- a/osm_proposals/proposals.py
+++ b/osm_proposals/proposals.py
@@ -4,8 +4,10 @@ import argparse
 import html
 import json
 import sys
+import logging
 from collections.abc import Container
 
+import logformat
 import pywikiapi
 import mwparserfromhell
 import requests
@@ -14,7 +16,13 @@ OSMWIKI_ENDPOINT = 'https://wiki.openstreetmap.org/w/api.php'
 
 # https://wiki.openstreetmap.org/w/index.php?title=Template:Proposal_page&action=edit
 
+logfmt_handler = logging.StreamHandler()
+logfmt_handler.setFormatter(logformat.LogfmtFormatter())
+logging.basicConfig(handlers=[logfmt_handler], level=logging.INFO)
+logger = logformat.get_logger()
 
+
+@logger.log_uncaught
 def run():
     arg_parser = argparse.ArgumentParser(description=__doc__)
     arg_parser.add_argument("out_file")
@@ -29,11 +37,17 @@ def run():
             text='{{#invoke:languages/table|json}}',
         ),
     )
-    langs: dict[str, dict] = json.loads(res.json()['expandtemplates']['wikitext'])
+    if not res.ok:
+        logger.error("expandtemplates request failed", status=res.status_code)
+        sys.exit(1)
+
+    data = res.json()
+    langs: dict[str, dict] = json.loads(data['expandtemplates']['wikitext'])
 
     osmwiki = pywikiapi.Site(OSMWIKI_ENDPOINT)
 
     proposals = []
+    # TODO: catch exception raised if HTTP request fails
     for page in osmwiki.query_pages(
         generator='embeddedin',
         geititle='Template:Proposal page',
@@ -55,6 +69,8 @@ def run():
     with open(args.out_file, 'w') as f:
         json.dump([{k: v for k, v in p.items() if v is not None} for p in proposals], f)
 
+    logger.info(f"updated {args.out_file}")
+
 
 def get_template_val(tpl, name):
     param = tpl.get(name, None)
@@ -65,10 +81,6 @@ def get_template_val(tpl, name):
             return value
 
 
-def eprint(*args):
-    print(*args, file=sys.stderr)
-
-
 def is_stub(doc):
     if any(
         doc.ifilter_templates(matches=lambda t: t.name.matches('Archived proposal'))
@@ -103,7 +115,7 @@ def parse_proposal(page_title: str, text: str, langs: Container[str]) -> dict |
     )
 
     if not proposal_page_templates:
-        eprint('{{Proposal Page}} not found in', page_title)
+        logger.info('{{Proposal Page}} not found', page=page_title)
         return None
 
     for comment in doc.ifilter_comments():
@@ -118,9 +130,9 @@ def parse_proposal(page_title: str, text: str, langs: Container[str]) -> dict |
 
     if is_stub(doc):
         if status in ('approved', 'rejected'):
-            eprint(f'WARNING {status} proposal is a stub', page_title)
+            logger.info(f'{status} proposal is a stub', page=page_title)
         else:
-            eprint('skipping stub', page_title)
+            logger.info('skipping stub', page=page_title)
             return None
 
     name = get_template_val(tpl, 'name')
diff --git a/requirements.txt b/requirements.txt
index d58a4d3..ac39062 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
 mwparserfromhell==0.6.4
 pywikiapi==4.3.0
 requests==2.25.1
+logformat==0.1.0
diff --git a/service.nix b/service.nix
index fcd665e..32af356 100644
--- a/service.nix
+++ b/service.nix
@@ -45,6 +45,7 @@ in
         ExecStart = "${osm_proposals}/bin/osm-proposals /var/lib/osm-proposals/proposals.json";
         # Not using DynamicUser because then the StateDirectory becomes unreadable
         # by other users, even when setting StateDirectoryMode for some reason.
+        LogExtraFields = "log_format=logfmt";
       };
       startAt = "hourly";
     };