diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | README.md | 6 | ||||
| -rwxr-xr-x | deploy.sh | 1 | ||||
| -rw-r--r-- | packages.txt | 7 | ||||
| -rwxr-xr-x | pydoc.py | 144 |
5 files changed, 161 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..418173c --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +versions.json +sources/ +dist/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..987c23d --- /dev/null +++ b/README.md @@ -0,0 +1,6 @@ +# pydoc.dev - docs.rs for Python + +* powered by [pydoctor] + + +[pydoctor]: https://github.com/twisted/pydoctor diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..0f98bbd --- /dev/null +++ b/deploy.sh @@ -0,0 +1 @@ +tar cf - -C dist . | ssh pydoc.dev 'sh -c "set -x; cd /var/www/pydoc.dev && rm -r * && pwd && tar xvf - && echo done"' diff --git a/packages.txt b/packages.txt new file mode 100644 index 0000000..8e0150b --- /dev/null +++ b/packages.txt @@ -0,0 +1,7 @@ +beautifulsoup4 +flask +httpx +lxml +pydoctor +requests +setuptools diff --git a/pydoc.py b/pydoc.py new file mode 100755 index 0000000..75777a3 --- /dev/null +++ b/pydoc.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +import json +import shutil +import tarfile +import tempfile +from pathlib import Path +from typing import Dict, List + +import pkg_resources +import pydoctor.driver +import requests + +# TODO: set USER_AGENT + + +def fetch_package_info(package_name: str): + return requests.get(f'https://pypi.org/pypi/{package_name}/json').json() + + +def find_packages(path: Path, package_name: str) -> List[Path]: + package_name = package_name.lower() + + if (path / package_name / '__init__.py').exists(): + return [path / package_name] + + if (path / 'src' / package_name / '__init__.py').exists(): + return [path / 'src' / package_name] + + packages = [] + + for subpath in path.iterdir(): + if subpath.is_dir(): + if (subpath / '__init__.py').exists(): + packages.append(subpath) + return packages + + +if __name__ == '__main__': + sources = Path('sources') + sources.mkdir(exist_ok=True) + + try: + with open('versions.json') as f: + versions = json.load(f) + except FileNotFoundError: + versions: Dict[str, str] = {} + + download_dir = Path(tempfile.mkdtemp(prefix='pydoc-')) + + # 1. fetch sources + + for package_name in ( + pkg_resources.resource_string(__name__, 'packages.txt').decode().splitlines() + ): + package = fetch_package_info(package_name) + version = package['info']['version'] + + sourceid = f'{package_name}-{version}' + + if ( + package_name not in versions + or versions[package_name] != version + or not (sources / sourceid).exists() + ): + print('downloading', sourceid) + + source_packages = [ + p for p in package['releases'][version] if p['packagetype'] == 'sdist' + ] + assert len(source_packages) > 0 + + if len(source_packages) > 1: + print( + f"[warning] {package_name} returned multiple source distributions, we're just using the first one" + ) + + source_package = source_packages[0] + + filename = source_package['filename'] + assert '/' not in filename # for security + + archive_path = download_dir / filename + + with requests.get(source_package['url'], stream=True) as r: + with open(archive_path, 'wb') as f: + shutil.copyfileobj(r.raw, f) + + tf = tarfile.open(archive_path) + for member in tf.getmembers(): + if '/' in member.name: + member.name = member.name.split('/', maxsplit=1)[1] + else: + member.name = '.' + tf.extractall(sources / sourceid) + versions[package_name] = version + + with open('versions.json', 'w') as f: + json.dump(versions, f) + + shutil.rmtree(download_dir) + + # 2. generate docs with pydoctor + + dist = Path('dist') + dist.mkdir(exist_ok=True) + + for path in sources.iterdir(): + sourceid = path.name + package_name, version = sourceid.rsplit('-', maxsplit=1) + out_dir = dist / package_name / version + + if out_dir.exists(): + continue + + print('generating', sourceid) + + packages = list(find_packages(sources / sourceid, package_name)) + + if len(packages) == 0: + print( + '[error] failed to determine package directory for', sources / sourceid + ) + continue + + if len(packages) > 1: + print( + f"[warning] found multiple packages for {package_name} ({packages}), we're just using the first one" + ) + + out_dir.mkdir(parents=True) + pydoctor.driver.main( + # fmt: off + [ + str(packages[0]), + '--html-output', out_dir, + ] + # fmt: on + ) + + # 3. create latest symlinks + for package_name, version in versions.items(): + latest = dist / package_name / 'latest' + latest.unlink(missing_ok=True) + latest.symlink_to(version) |