From 2ee6afdf3356dea2aebdd48d20dfe9eb07037ba3 Mon Sep 17 00:00:00 2001
From: Martin Fischer <martin@push-f.com>
Date: Sat, 22 Mar 2025 22:17:48 +0100
Subject: feat: improve logging
---
default.nix | 4 +++-
npins/sources.json | 14 ++++++++++++++
osm_proposals/proposals.py | 28 ++++++++++++++++++++--------
requirements.txt | 1 +
service.nix | 1 +
5 files changed, 39 insertions(+), 9 deletions(-)
diff --git a/default.nix b/default.nix
index 897ac2e..d70e409 100644
--- a/default.nix
+++ b/default.nix
@@ -1,5 +1,5 @@
{ pkgs ? import <nixpkgs> {} }:
-with pkgs.python310Packages;
+with pkgs.python313Packages;
let
sources = import ./npins;
@@ -12,6 +12,7 @@ let
requests
];
};
+ logformat = import "${sources.logformat}" {};
in
buildPythonApplication rec {
pname = "osm-proposals";
@@ -27,6 +28,7 @@ buildPythonApplication rec {
requests
mwparserfromhell
pywikiapi
+ logformat
];
postInstall = ''
diff --git a/npins/sources.json b/npins/sources.json
index 3b57f31..65b2764 100644
--- a/npins/sources.json
+++ b/npins/sources.json
@@ -1,5 +1,19 @@
{
"pins": {
+ "logformat": {
+ "type": "GitRelease",
+ "repository": {
+ "type": "Git",
+ "url": "https://git.push-f.com/logformat"
+ },
+ "pre_releases": false,
+ "version_upper_bound": null,
+ "release_prefix": "v",
+ "version": "v0.1.0",
+ "revision": "bded757ac8b71df61bef068c320c639ab64f2f06",
+ "url": null,
+ "hash": "16x5fnzvkxw52j288a791zvy66rj5cq7i5dvi9wpnn795l82jz9h"
+ },
"pywikiapi": {
"type": "PyPi",
"name": "pywikiapi",
diff --git a/osm_proposals/proposals.py b/osm_proposals/proposals.py
index 1bb1f14..edcb453 100755
--- a/osm_proposals/proposals.py
+++ b/osm_proposals/proposals.py
@@ -4,8 +4,10 @@ import argparse
import html
import json
import sys
+import logging
from collections.abc import Container
+import logformat
import pywikiapi
import mwparserfromhell
import requests
@@ -14,7 +16,13 @@ OSMWIKI_ENDPOINT = 'https://wiki.openstreetmap.org/w/api.php'
# https://wiki.openstreetmap.org/w/index.php?title=Template:Proposal_page&action=edit
+logfmt_handler = logging.StreamHandler()
+logfmt_handler.setFormatter(logformat.LogfmtFormatter())
+logging.basicConfig(handlers=[logfmt_handler], level=logging.INFO)
+logger = logformat.get_logger()
+
+@logger.log_uncaught
def run():
arg_parser = argparse.ArgumentParser(description=__doc__)
arg_parser.add_argument("out_file")
@@ -29,11 +37,17 @@ def run():
text='{{#invoke:languages/table|json}}',
),
)
- langs: dict[str, dict] = json.loads(res.json()['expandtemplates']['wikitext'])
+ if not res.ok:
+ logger.error("expandtemplates request failed", status=res.status_code)
+ sys.exit(1)
+
+ data = res.json()
+ langs: dict[str, dict] = json.loads(data['expandtemplates']['wikitext'])
osmwiki = pywikiapi.Site(OSMWIKI_ENDPOINT)
proposals = []
+ # TODO: catch exception raised if HTTP request fails
for page in osmwiki.query_pages(
generator='embeddedin',
geititle='Template:Proposal page',
@@ -55,6 +69,8 @@ def run():
with open(args.out_file, 'w') as f:
json.dump([{k: v for k, v in p.items() if v is not None} for p in proposals], f)
+ logger.info(f"updated {args.out_file}")
+
def get_template_val(tpl, name):
param = tpl.get(name, None)
@@ -65,10 +81,6 @@ def get_template_val(tpl, name):
return value
-def eprint(*args):
- print(*args, file=sys.stderr)
-
-
def is_stub(doc):
if any(
doc.ifilter_templates(matches=lambda t: t.name.matches('Archived proposal'))
@@ -103,7 +115,7 @@ def parse_proposal(page_title: str, text: str, langs: Container[str]) -> dict |
)
if not proposal_page_templates:
- eprint('{{Proposal Page}} not found in', page_title)
+ logger.info('{{Proposal Page}} not found', page=page_title)
return None
for comment in doc.ifilter_comments():
@@ -118,9 +130,9 @@ def parse_proposal(page_title: str, text: str, langs: Container[str]) -> dict |
if is_stub(doc):
if status in ('approved', 'rejected'):
- eprint(f'WARNING {status} proposal is a stub', page_title)
+ logger.info(f'{status} proposal is a stub', page=page_title)
else:
- eprint('skipping stub', page_title)
+ logger.info('skipping stub', page=page_title)
return None
name = get_template_val(tpl, 'name')
diff --git a/requirements.txt b/requirements.txt
index d58a4d3..ac39062 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
mwparserfromhell==0.6.4
pywikiapi==4.3.0
requests==2.25.1
+logformat==0.1.0
diff --git a/service.nix b/service.nix
index fcd665e..32af356 100644
--- a/service.nix
+++ b/service.nix
@@ -45,6 +45,7 @@ in
ExecStart = "${osm_proposals}/bin/osm-proposals /var/lib/osm-proposals/proposals.json";
# Not using DynamicUser because then the StateDirectory becomes unreadable
# by other users, even when setting StateDirectoryMode for some reason.
+ LogExtraFields = "log_format=logfmt";
};
startAt = "hourly";
};
--
cgit v1.2.3