#!/usr/bin/env python3 import html import json import sys from collections.abc import Container import pywikiapi import mwparserfromhell import requests OSMWIKI_ENDPOINT = 'https://wiki.openstreetmap.org/w/api.php' # https://wiki.openstreetmap.org/w/index.php?title=Template:Proposal_page&action=edit def run(): res = requests.get( OSMWIKI_ENDPOINT, params=dict( action='expandtemplates', prop='wikitext', format='json', text='{{#invoke:languages/table|json}}', ), ) langs: dict[str, dict] = json.loads(res.json()['expandtemplates']['wikitext']) osmwiki = pywikiapi.Site(OSMWIKI_ENDPOINT) proposals = [] for page in osmwiki.query_pages( generator='embeddedin', geititle='Template:Proposal page', geilimit='max', prop='revisions', rvprop='content', rvslots='main', ): proposal = parse_proposal(page, langs) if proposal: proposals.append(proposal) proposals.sort(key=sort_key, reverse=True) json.dump( [{k: v for k, v in p.items() if v is not None} for p in proposals], sys.stdout ) def get_template_val(tpl, name): param = tpl.get(name, None) if param: value = param.value.strip() if value: # turn empty strings into None return value def eprint(*args): print(*args, file=sys.stderr) def is_stub(doc): if any( doc.ifilter_templates(matches=lambda t: t.name.matches('Archived proposal')) ): return False if not any(doc.ifilter_headings()): # detect proposals without headings as stubs return True if not any( n for n in doc.nodes # any text if isinstance(n, mwparserfromhell.nodes.text.Text) # other than newlines and n.strip() # and "Please comment on the [[{{TALKPAGENAME}}|discussion page]]." and n.strip() not in ('Please comment on the', '.') ): # detect proposals without text as stubs return True return False def parse_proposal(page: dict, langs: Container[str]) -> dict | None: page_title = page['title'] text = page['revisions'][0]['slots']['main']['content'] doc = mwparserfromhell.parse(text) proposal_page_templates = doc.filter_templates( matches=lambda t: t.name.matches('Proposal page') or t.name.matches('Proposal Page') ) if not proposal_page_templates: eprint('{{Proposal Page}} not found in', page_title) return None for comment in doc.ifilter_comments(): # remove comments like doc.remove(comment) tpl = proposal_page_templates[0] status = get_template_val(tpl, 'status') if status: status = status.lower() if is_stub(doc): if status in ('approved', 'rejected'): eprint(f'WARNING {status} proposal is a stub', page['title']) else: eprint('skipping stub', page['title']) return None name = get_template_val(tpl, 'name') if name: name = html.unescape(name) draft_start = get_template_val(tpl, 'draftStartDate') if draft_start in ('*', '-'): draft_start = None rfc_start = get_template_val(tpl, 'rfcStartDate') if rfc_start in ('*', '-'): rfc_start = None vote_start = get_template_val(tpl, 'voteStartDate') if vote_start in ('*', '-'): vote_start = None definition = get_template_val(tpl, 'definition') users = get_template_val(tpl, 'users') or get_template_val(tpl, 'user') parts = page_title.split(':', maxsplit=1) parts[0] = parts[0].lower() lang = None if parts[0] in langs: lang = parts[0] return dict( page_title=page_title, lang=lang, name=name, status=status, definition=definition, draft_start=draft_start, rfc_start=rfc_start, vote_start=vote_start, authors=users, ) STATUSES = { 'voting': 0, 'post-vote': 1, 'proposed': 2, 'draft': 3, 'approved': 4, 'inactive': 5, 'rejected': 6, 'abandoned': 7, 'canceled': 8, 'obsoleted': 9, } def sort_key(proposal): status = proposal['status'] if status in ('voting', 'approved', 'rejected'): date = proposal['vote_start'] or '' elif status == 'proposed': date = proposal['rfc_start'] or '' else: date = proposal['draft_start'] or '' return (-STATUSES.get(proposal['status'], 10), date) if __name__ == "__main__": run()