path: root/check_checkers.py

#!/usr/bin/env python3
import hashlib
import html
import json
import os
import re
import shutil
from subprocess import PIPE, Popen, check_output
import tempfile
import time
from typing import Dict, List, NamedTuple, TextIO, Tuple, TypedDict
from pathlib import Path

import mypy.api
from pygments import highlight
from pygments.formatters import HtmlFormatter
from pygments.lexers import PythonLexer
import tomli


Error = Tuple[str, int, str]
"""The filename, line number and the error message."""


class Checker:
    name: str
    url: str
    extra_args: Tuple[str, ...] = ()

    def run(self, path: str, typeshed_path: str) -> List[Error]:
        """
        Type checks the given path with the given options.
        """
        raise NotImplementedError()

    def version(self) -> str:
        """Returns the version of the checker."""
        raise NotImplementedError()


class Mypy(Checker):
    name = 'Mypy'
    url = 'https://github.com/python/mypy'

    # mypy cannot output JSON (https://github.com/python/mypy/issues/10816)
    # though there is a PR (https://github.com/python/mypy/pull/11396)
    # so we just use the regex from https://github.com/matangover/mypy-vscode/blob/48162f345c7f14b96f29976660100ae1dd49cc0a/src/mypy.ts
    _pattern = re.compile(
        r'^(?P<file>[^\n]+?):((?P<line>\d+):)?((?P<column>\d+):)? (?P<type>\w+): (?P<message>.*)$',
        re.MULTILINE,
    )

    @classmethod
    def run(cls, path: str, typeshed_path: str):
        cachedir = tempfile.mkdtemp(prefix='mypy-cache-')
        stdout, stderr, retcode = mypy.api.run(
            [
                # fmt: off
                '--cache-dir', cachedir,
                '--custom-typeshed-dir', typeshed_path,
                *cls.extra_args,
                # fmt: on
                '--',
                path,
            ]
        )
        shutil.rmtree(cachedir)
        return [
            (m.group('file'), m.group('line'), m.group('message'))
            for m in cls._pattern.finditer(stdout)
        ]

    @staticmethod
    def version():
        return mypy.api.run(['--version'])[0].split()[1].strip()


class MypyStrict(Mypy):
    extra_args = ('--strict',)


class Pytype(Checker):
    name = 'Pytype'
    url = 'https://github.com/google/pytype'

    # pytype supports CSV output only for pytype-single which however doesn't support multiple modules
    # (https://github.com/google/pytype/issues/92)
    _pattern = re.compile(
        r'^File "(?P<file>[^"]+?)", line (?P<line>\d+), in (?P<module>[^ ]+): (?P<message>.*) \[(?P<id>[^]]+)\]$',
        re.MULTILINE,
    )

    @classmethod
    def run(cls, path: str, typeshed_path: str):
        env = {'TYPESHED_HOME': typeshed_path, 'PATH': os.environ['PATH']}
        proc = Popen(
            ['pytype', *cls.extra_args, '--', path],
            stdout=PIPE,
            stderr=PIPE,
            encoding='utf-8',
            env=env,
        )
        stdout, stderr = proc.communicate()
        return [
            (m.group('file'), m.group('line'), m.group('message'))
            for m in cls._pattern.finditer(stdout)
        ]

    @staticmethod
    def version():
        return check_output(['pytype', '--version'], encoding='utf-8').strip()


class Pyright(Checker):
    name = 'Pyright'
    url = 'https://github.com/microsoft/pyright'

    @classmethod
    def run(cls, path: str, typeshed_path: str):
        proc = Popen(
            [
                'pyright',
                # fmt: off
                '--typeshed-path', typeshed_path,
                '--outputjson',
                *cls.extra_args,
                # fmt: on
                # pyright does not support --
                path,
            ],
            stdout=PIPE,
            stderr=PIPE,
            encoding='utf-8',
        )
        stdout, stderr = proc.communicate()
        return [
            (d['file'], d['range']['start']['line'] + 1, d['message'])
            for d in json.loads(stdout)['generalDiagnostics']
        ]

    @staticmethod
    def version():
        return (
            check_output(['pyright', '--version'], encoding='utf-8').split()[1].strip()
        )


class PyrightStrict(Pyright):
    extra_args = ('-p', 'pyrightstrict.json')


# We don't check pyre because it has a very slow startup time (5s) since it parses the whole typeshed.
# (see https://github.com/facebook/pyre-check/issues/592)


class Puzzle(TypedDict):
    checker_results: Dict[str, List[Error]]
    puzzle_hash: str


def run_checkers(checkers: List[Checker], puzzle: str, typeshed_path: str):
    results = {}
    for checker in checkers:
        start = time.time()
        results[checker.__class__.__name__] = checker.run(puzzle, typeshed_path)
        duration = time.time() - start
        print(checker, time.time() - start)
    return results


def run(
    checkers: List[Checker],
    puzzles: List[Path],
    default_typeshed: str,
    out: TextIO,
    cache: Dict[str, Puzzle],
    issues: Dict[str, Dict[str, str]],
):
    python_lexer = PythonLexer()
    html_formatter = HtmlFormatter(noclasses=True, linenos='table')
    out.write(
        "<meta charset=utf-8><title>Comparison of static type checkers for Python</title>"
    )
    out.write(
        '''<p>This page compares three static type checkers for Python.
    The <span class=unexpected>red</span> background indicates that the checker
    fails to detect a type error or reports a false positive.
    <a href=/>Back to start page</a>.</p>'''
    )

    out.write(
        '''<style>
    .unexpected {background: #ffd2d0}
    th {position: sticky; top: 0; background: #e8e8e8;}
    table {border-spacing: 0;}
    td:target {background: #ffffc3;}

    html {
        /* prevent sticky table header from overlapping table cell targeted via #anchor */
        scroll-padding-top: 60px;
    }
    </style>
    '''
    )

    out.write('<table border=1>')
    out.write('<tr><th>Input')
    for checker in checkers:
        out.write('<th>')
        out.write('<a href="{}">'.format(html.escape(checker.url)))
        out.write(checker.name)
        out.write('</a>')
        if checker.extra_args:
            out.write(' with ')
            out.write('<code>')
            for arg in checker.extra_args:
                if not arg.startswith('-'):
                    out.write('<a href="configs/{}">'.format(html.escape(arg)))
                out.write(html.escape(arg) + ' ')
                if not arg.startswith('-'):
                    out.write('</a>')
                    os.makedirs('dist/configs', exist_ok=True)
                    shutil.copyfile(arg, 'dist/configs/' + arg)
            out.write('</code>')
        out.write('<br>({})'.format(html.escape(checker.version())))
    out.write('</tr>')

    for puzzle in puzzles:
        print(puzzle)

        codes: Dict[str, bytes] = {}

        if puzzle.is_dir():
            for file in sorted(puzzle.iterdir()):
                with file.open('rb') as f:
                    codes[file.name] = f.read()
        else:
            with puzzle.open('rb') as f:
                codes[puzzle.name] = f.read()

        puzzle_hash = '-'.join(
            hashlib.sha256(code).hexdigest() for code in codes.values()
        )
        if str(puzzle) in cache and puzzle_hash == cache[str(puzzle)]['puzzle_hash']:
            checker_results = cache[str(puzzle)]['checker_results']
        else:
            checker_results = run_checkers(checkers, str(puzzle), default_typeshed)
            cache[str(puzzle)] = {
                'puzzle_hash': puzzle_hash,
                'checker_results': checker_results,
            }

        anchor = html.escape(puzzle.name).replace('.py', '').replace('_', '-')
        out.write('<tr><td id="{0}"><a href="#{0}">{0}</a><br>'.format(anchor))

        error_ok = False
        no_error_ok = True

        for fname, code in codes.items():
            code = code.decode('utf-8')
            if len(codes) > 1:
                out.write(html.escape(fname))
            out.write(highlight(code, python_lexer, html_formatter))

            if '# error' in code:
                error_ok = True
                no_error_ok = False
            elif '# maybe error' in code:
                error_ok = True

        results = [
            (checker, checker_results[checker.__class__.__name__])
            for checker in checkers
        ]

        while results:
            checker, errors = results.pop(0)
            expected = (errors and error_ok) or (not errors and no_error_ok)
            colspan = 1
            if results and (checker.name, errors) == (
                results[0][0].name,
                results[0][1],
            ):
                results.pop(0)
                colspan = 2
            out.write(
                '<td class="{}" colspan={}>'.format(
                    'ok' if expected else 'unexpected', colspan
                )
            )
            if errors:
                out.write('<ul>')
                for filename, line, message in errors:
                    out.write('<li>')
                    out.write(f'{line}: ' + html.escape(message).replace('\n', '<br>'))
                    out.write('</li>')
                out.write('</ul>')
            else:
                out.write('<center>no errors found')
            if not expected:
                checker_issues = issues.get(
                    checker.__class__.__name__.lower(),
                    issues.get(
                        checker.__class__.__name__.lower().replace('strict', ''), {}
                    ),
                )
                issue = checker_issues.get(
                    os.path.splitext(os.path.basename(puzzle))[0]
                )
                if issue:
                    out.write('<br>(')
                    out.write(issue)
                    out.write(')')
    out.write('</table>')


if __name__ == '__main__':
    # TODO: git clone typeshed if missing
    typeshed = os.path.abspath('typeshed')  # pytype requries an absolute path

    try:
        with open('cache.json') as f:
            cache = json.load(f)
    except FileNotFoundError:
        cache = {}

    with open('issues.toml', 'rb') as f:
        issues = tomli.load(f)

    with open('dist/checkers.html', 'w') as f:
        run(
            [Mypy(), MypyStrict(), Pytype(), Pyright(), PyrightStrict()],
            list(sorted(Path('puzzles/').iterdir())),
            typeshed,
            f,
            cache,
            issues,
        )

    with open('cache.json', 'w') as f:
        json.dump(cache, f)