diff options
| -rw-r--r-- | .gitignore | 12 | ||||
| -rw-r--r-- | LICENSE | 19 | ||||
| -rw-r--r-- | Makefile | 2 | ||||
| -rw-r--r-- | README.md | 16 | ||||
| -rw-r--r-- | default.nix | 45 | ||||
| -rwxr-xr-x | find_archived_proposals_without_template.py | 23 | ||||
| -rw-r--r-- | npins/default.nix | 80 | ||||
| -rw-r--r-- | npins/sources.json | 27 | ||||
| -rwxr-xr-x | osm_proposals/archived_without_template.py | 34 | ||||
| -rwxr-xr-x | osm_proposals/proposals.py (renamed from proposals.py) | 126 | ||||
| -rw-r--r-- | pyproject.toml | 14 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | service.nix | 53 | ||||
| -rw-r--r-- | static/index.html (renamed from index.html) | 0 | ||||
| -rw-r--r-- | static/script.js (renamed from script.js) | 0 | ||||
| -rw-r--r-- | static/style.css (renamed from style.css) | 0 | ||||
| -rw-r--r-- | tests/test_proposals.py | 56 |
17 files changed, 434 insertions, 74 deletions
@@ -1 +1,11 @@ -proposals.json +*.pyc +*.egg-info/ + +# created by `python -m venv venv` (listed here for hatchling) +venv/ + +# created by `python -m build` +dist/ + +# created by `nix-build` +result @@ -0,0 +1,19 @@ +Copyright (c) 2025 Martin Fischer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile deleted file mode 100644 index 31de770..0000000 --- a/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -all: - ./proposals.py > proposals.json diff --git a/README.md b/README.md new file mode 100644 index 0000000..eb4eab7 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# osm-proposals + +An autogenerated web page listing [OpenStreetMap proposals]. + +Hosted under https://osm-proposals.push-f.com/. + +# Local development + + python -m venv venv + . venv/bin/activate + pip install -r requirements.txt + pip install -e . + osm-proposals static/proposals.json + python -m http.server --directory static + +[OpenStreetMap proposals]: https://wiki.openstreetmap.org/wiki/Proposal diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..a828aa9 --- /dev/null +++ b/default.nix @@ -0,0 +1,45 @@ +{ pkgs ? import <nixpkgs> {} }: +with pkgs.python313Packages; + +let + sources = import ./npins; + pywikiapi = buildPythonPackage rec { + pname = "pywikiapi"; + version = "pinned"; + src = sources.pywikiapi; + + dependencies = [ + requests + ]; + }; + logformat = import "${sources.logformat}" {}; +in +buildPythonApplication rec { + pname = "osm-proposals"; + version = "git"; + src = ./.; + pyproject = true; + + build-system = [ + hatchling + ]; + + dependencies = [ + requests + mwparserfromhell + pywikiapi + logformat + ]; + + # smoke test + preCheck = "$out/bin/osm-proposals --help >/dev/null"; + + postInstall = '' + mkdir -p $out/share/osm-proposals + cp -r ${./static}/. $out/share/osm-proposals/ + ''; + + nativeCheckInputs = [ + pytestCheckHook + ]; +} diff --git a/find_archived_proposals_without_template.py b/find_archived_proposals_without_template.py deleted file mode 100755 index 0acb4bf..0000000 --- a/find_archived_proposals_without_template.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 -""" -Sometimes when archiving a page people accidentally also replace the -{{Proposal page}} template, which however means that proposal.py -won't find the page anymore. This script lists such pages so that the -template can be manually restored. -""" -import pywikiapi -import mwparserfromhell - -OSMWIKI_ENDPOINT = 'https://wiki.openstreetmap.org/w/api.php' - -osmwiki = pywikiapi.Site(OSMWIKI_ENDPOINT) - -for page in osmwiki.query_pages( - generator='categorymembers', - gcmtitle='Category:Archived proposals', - gcmlimit='max', - prop='templates', - tltemplates='Template:Proposal page' -): - if not 'templates' in page: - print(page['title']) diff --git a/npins/default.nix b/npins/default.nix new file mode 100644 index 0000000..5e7d086 --- /dev/null +++ b/npins/default.nix @@ -0,0 +1,80 @@ +# Generated by npins. Do not modify; will be overwritten regularly +let + data = builtins.fromJSON (builtins.readFile ./sources.json); + version = data.version; + + mkSource = + spec: + assert spec ? type; + let + path = + if spec.type == "Git" then + mkGitSource spec + else if spec.type == "GitRelease" then + mkGitSource spec + else if spec.type == "PyPi" then + mkPyPiSource spec + else if spec.type == "Channel" then + mkChannelSource spec + else + builtins.throw "Unknown source type ${spec.type}"; + in + spec // { outPath = path; }; + + mkGitSource = + { + repository, + revision, + url ? null, + hash, + branch ? null, + ... + }: + assert repository ? type; + # At the moment, either it is a plain git repository (which has an url), or it is a GitHub/GitLab repository + # In the latter case, there we will always be an url to the tarball + if url != null then + (builtins.fetchTarball { + inherit url; + sha256 = hash; # FIXME: check nix version & use SRI hashes + }) + else + assert repository.type == "Git"; + let + urlToName = + url: rev: + let + matched = builtins.match "^.*/([^/]*)(\\.git)?$" repository.url; + + short = builtins.substring 0 7 rev; + + appendShort = if (builtins.match "[a-f0-9]*" rev) != null then "-${short}" else ""; + in + "${if matched == null then "source" else builtins.head matched}${appendShort}"; + name = urlToName repository.url revision; + in + builtins.fetchGit { + url = repository.url; + rev = revision; + inherit name; + # hash = hash; + }; + + mkPyPiSource = + { url, hash, ... }: + builtins.fetchurl { + inherit url; + sha256 = hash; + }; + + mkChannelSource = + { url, hash, ... }: + builtins.fetchTarball { + inherit url; + sha256 = hash; + }; +in +if version == 3 then + builtins.mapAttrs (_: mkSource) data.pins +else + throw "Unsupported format version ${toString version} in sources.json. Try running `npins upgrade`" diff --git a/npins/sources.json b/npins/sources.json new file mode 100644 index 0000000..65b2764 --- /dev/null +++ b/npins/sources.json @@ -0,0 +1,27 @@ +{ + "pins": { + "logformat": { + "type": "GitRelease", + "repository": { + "type": "Git", + "url": "https://git.push-f.com/logformat" + }, + "pre_releases": false, + "version_upper_bound": null, + "release_prefix": "v", + "version": "v0.1.0", + "revision": "bded757ac8b71df61bef068c320c639ab64f2f06", + "url": null, + "hash": "16x5fnzvkxw52j288a791zvy66rj5cq7i5dvi9wpnn795l82jz9h" + }, + "pywikiapi": { + "type": "PyPi", + "name": "pywikiapi", + "version_upper_bound": null, + "version": "4.3.0", + "url": "https://files.pythonhosted.org/packages/9e/f0/09741c61d3069c99195690321cf0ad17c0a63d6ed33f130f45d9d85c2cb6/pywikiapi-4.3.0.tar.gz", + "hash": "8329e4c7df32f15c7f3778bac6843bc7d3204d6eaa002a18703c32ef51436081" + } + }, + "version": 3 +}
\ No newline at end of file diff --git a/osm_proposals/archived_without_template.py b/osm_proposals/archived_without_template.py new file mode 100755 index 0000000..ea026d3 --- /dev/null +++ b/osm_proposals/archived_without_template.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +""" +Queries wiki.openstreetmap.org for archived proposal pages without the {{Proposal page}} template. + +Sometimes when archiving a page people accidentally also replace the +{{Proposal page}} template, which however means that proposal.py +won't find the page anymore. This script lists such pages so that the +template can be manually restored. +""" +import argparse + +import pywikiapi +import mwparserfromhell + +OSMWIKI_ENDPOINT = 'https://wiki.openstreetmap.org/w/api.php' + + +def run(): + arg_parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + arg_parser.parse_args() + + osmwiki = pywikiapi.Site(OSMWIKI_ENDPOINT) + + for page in osmwiki.query_pages( + generator='categorymembers', + gcmtitle='Category:Archived proposals', + gcmlimit='max', + prop='templates', + tltemplates='Template:Proposal page', + ): + if not 'templates' in page: + print(page['title']) diff --git a/proposals.py b/osm_proposals/proposals.py index dc770b4..edcb453 100755 --- a/proposals.py +++ b/osm_proposals/proposals.py @@ -1,22 +1,75 @@ #!/usr/bin/env python3 +"""Queries wiki.openstreetmap.org for proposals and writes a JSON list of them to the given file.""" +import argparse import html import json import sys +import logging +from collections.abc import Container +import logformat import pywikiapi import mwparserfromhell import requests OSMWIKI_ENDPOINT = 'https://wiki.openstreetmap.org/w/api.php' -osmwiki = pywikiapi.Site(OSMWIKI_ENDPOINT) - # https://wiki.openstreetmap.org/w/index.php?title=Template:Proposal_page&action=edit -res = requests.get( - 'https://wiki.openstreetmap.org/w/api.php?action=expandtemplates&prop=wikitext&format=json&text={%7B%23invoke:languages/table%7Cjson}}' -) -langs = json.loads(res.json()['expandtemplates']['wikitext']) +logfmt_handler = logging.StreamHandler() +logfmt_handler.setFormatter(logformat.LogfmtFormatter()) +logging.basicConfig(handlers=[logfmt_handler], level=logging.INFO) +logger = logformat.get_logger() + + +@logger.log_uncaught +def run(): + arg_parser = argparse.ArgumentParser(description=__doc__) + arg_parser.add_argument("out_file") + args = arg_parser.parse_args() + + res = requests.get( + OSMWIKI_ENDPOINT, + params=dict( + action='expandtemplates', + prop='wikitext', + format='json', + text='{{#invoke:languages/table|json}}', + ), + ) + if not res.ok: + logger.error("expandtemplates request failed", status=res.status_code) + sys.exit(1) + + data = res.json() + langs: dict[str, dict] = json.loads(data['expandtemplates']['wikitext']) + + osmwiki = pywikiapi.Site(OSMWIKI_ENDPOINT) + + proposals = [] + # TODO: catch exception raised if HTTP request fails + for page in osmwiki.query_pages( + generator='embeddedin', + geititle='Template:Proposal page', + geilimit='max', + prop='revisions', + rvprop='content', + rvslots='main', + ): + proposal = parse_proposal( + page_title=page['title'], + text=page['revisions'][0]['slots']['main']['content'], + langs=langs, + ) + if proposal: + proposals.append(proposal) + + proposals.sort(key=sort_key, reverse=True) + + with open(args.out_file, 'w') as f: + json.dump([{k: v for k, v in p.items() if v is not None} for p in proposals], f) + + logger.info(f"updated {args.out_file}") def get_template_val(tpl, name): @@ -28,13 +81,6 @@ def get_template_val(tpl, name): return value -proposals = [] - - -def eprint(*args): - print(*args, file=sys.stderr) - - def is_stub(doc): if any( doc.ifilter_templates(matches=lambda t: t.name.matches('Archived proposal')) @@ -61,24 +107,16 @@ def is_stub(doc): return False -for page in osmwiki.query_pages( - generator='embeddedin', - geititle='Template:Proposal Page', - geilimit='max', - prop='revisions', - rvprop='content', - rvslots='main', -): - page_title = page['title'] - text = page['revisions'][0]['slots']['main']['content'] +def parse_proposal(page_title: str, text: str, langs: Container[str]) -> dict | None: doc = mwparserfromhell.parse(text) proposal_page_templates = doc.filter_templates( - matches=lambda t: t.name.matches('Proposal Page') + matches=lambda t: t.name.matches('Proposal page') + or t.name.matches('Proposal Page') ) if not proposal_page_templates: - eprint('{{Proposal Page}} not found in', page_title) - continue + logger.info('{{Proposal Page}} not found', page=page_title) + return None for comment in doc.ifilter_comments(): # remove comments like <!-- Date the RFC email is sent to the Tagging list: YYYY-MM-DD --> @@ -92,10 +130,10 @@ for page in osmwiki.query_pages( if is_stub(doc): if status in ('approved', 'rejected'): - eprint(f'WARNING {status} proposal is a stub', page['title']) + logger.info(f'{status} proposal is a stub', page=page_title) else: - eprint('skipping stub', page['title']) - continue + logger.info('skipping stub', page=page_title) + return None name = get_template_val(tpl, 'name') if name: @@ -123,20 +161,19 @@ for page in osmwiki.query_pages( if parts[0] in langs: lang = parts[0] - proposals.append( - dict( - page_title=page_title, - lang=lang, - name=name, - status=status, - definition=definition, - draft_start=draft_start, - rfc_start=rfc_start, - vote_start=vote_start, - authors=users, - ) + return dict( + page_title=page_title, + lang=lang, + name=name, + status=status, + definition=definition, + draft_start=draft_start, + rfc_start=rfc_start, + vote_start=vote_start, + authors=users, ) + STATUSES = { 'voting': 0, 'post-vote': 1, @@ -162,10 +199,3 @@ def sort_key(proposal): date = proposal['draft_start'] or '' return (-STATUSES.get(proposal['status'], 10), date) - - -proposals.sort(key=sort_key, reverse=True) - -json.dump( - [{k: v for k, v in p.items() if v is not None} for p in proposals], sys.stdout -) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2a6d936 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,14 @@ +[project] +name = "osm-proposals" +version = "0.0.0" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["osm_proposals"] + +[project.scripts] +osm-proposals = "osm_proposals.proposals:run" +osm-proposals-archived-without-template = "osm_proposals.archived_without_template:run" diff --git a/requirements.txt b/requirements.txt index d58a4d3..ac39062 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ mwparserfromhell==0.6.4 pywikiapi==4.3.0 requests==2.25.1 +logformat==0.1.0 diff --git a/service.nix b/service.nix new file mode 100644 index 0000000..7e2fabc --- /dev/null +++ b/service.nix @@ -0,0 +1,53 @@ +{ config, lib, pkgs, ... }: + +let + osm_proposals = pkgs.callPackage ./default.nix {}; + cfg = config.services.osm_proposals; +in +{ + options.services.osm_proposals = { + enable = lib.mkEnableOption "osm_proposals"; + + virtualHost = lib.mkOption { + type = lib.types.str; + description = "Name of the nginx virtualhost to set up."; + }; + + nginx = lib.mkOption { + type = lib.types.submodule (import <nixpkgs/nixos/modules/services/web-servers/nginx/vhost-options.nix>); + default = {}; + }; + }; + + config = lib.mkIf cfg.enable { + services.nginx = { + enable = true; + + virtualHosts.${cfg.virtualHost} = lib.mkMerge [ + cfg.nginx + { + locations."/" = { + root = "${osm_proposals}/share/osm-proposals"; + }; + locations."=/proposals.json" = { + extraConfig = '' + alias /var/lib/osm-proposals/proposals.json; + ''; + }; + } + ]; + }; + + systemd.services.osm-proposals = { + serviceConfig = { + Type = "oneshot"; + StateDirectory = "osm-proposals"; # creates /var/lib/osm-proposals + ExecStart = "${osm_proposals}/bin/osm-proposals /var/lib/osm-proposals/proposals.json"; + # Not using DynamicUser because then the StateDirectory becomes unreadable + # by other users, even when setting StateDirectoryMode for some reason. + LogExtraFields = "LOG_FORMAT=logfmt"; + }; + startAt = "hourly"; + }; + }; +} diff --git a/index.html b/static/index.html index 8d7bd7a..8d7bd7a 100644 --- a/index.html +++ b/static/index.html diff --git a/script.js b/static/script.js index 35aa43e..35aa43e 100644 --- a/script.js +++ b/static/script.js diff --git a/style.css b/static/style.css index 66fe28f..66fe28f 100644 --- a/style.css +++ b/static/style.css diff --git a/tests/test_proposals.py b/tests/test_proposals.py new file mode 100644 index 0000000..e11c1be --- /dev/null +++ b/tests/test_proposals.py @@ -0,0 +1,56 @@ +import textwrap + +import pytest + +from osm_proposals import proposals + + +@pytest.mark.parametrize("has_heading", (True, False)) +@pytest.mark.parametrize("has_text", (True, False)) +def test_parse_proposal_with_template(has_heading: bool, has_text: bool): + proposal = proposals.parse_proposal( + "some title", + textwrap.dedent( + ''' + {{Proposal page + | name = some name + | status = draft + | user = SomeUser + | key = <!-- The key of the proposed new tag, if relevant --> + | value = <!-- The value of the proposed new tag, if relevant --> + | tagging = <!-- If your proposal is about multiple tags, you may link them here like: {{tag|foo|bar}}, {{tag|bar}} --> + | type = <!-- node, way, area, relation ({{IconNode}} / {{IconWay}} / {{IconArea}} / {{IconRelation}}) --> + | definition = <!-- A short, clear definition of the feature or property which the new tag represents --> + | taginfo = yes <!-- yes / no: to show taginfo statistics box --> + | appearance = <!-- A possible rendering, if relevant – optional --> + | draftStartDate = 2022-12-07 + | rfcStartDate = <!-- Date the RFC email is sent to the Tagging list: YYYY-MM-DD --> + | voteStartDate = <!-- YYYY-MM-DD 00:00:00 (UTC) – date voting starts: at least 2 weeks after RFC --> + | voteEndDate = <!-- YYYY-MM-DD 23:59:59 (UTC) – date voting will end: at least 2 weeks after start of voting --> + }} + ''' + f''' + {"== Heading ==" * has_heading} + {"Some text." * has_text} + ''' + ), + (), + ) + if not has_heading or not has_text: + assert proposal is None + else: + assert proposal == { + 'page_title': 'some title', + 'lang': None, + 'name': 'some name', + 'status': 'draft', + 'authors': 'SomeUser', + 'definition': None, + 'draft_start': '2022-12-07', + 'rfc_start': None, + 'vote_start': None, + } + + +def test_parse_proposal_without_template(): + assert proposals.parse_proposal("test", "nothing", ()) is None |