Add Python API documentation generation with pdoc3

This commit adds automated Python API documentation generation for all workspace packages using pdoc3: - Add contrib/api/generate-python-docs.py script to generate docs - Add Makefile targets: python-docs and python-docs-clean - Add GitHub Actions workflow for nightly documentation generation - Documents 5 packages: pyln.client, pyln.proto, pyln.grpc, pyln.testing, pyln.spec.bolt7 - Creates beautiful index page with cards linking to each package - Stores generated docs as artifacts with 90-day retention - Add pdoc3 and markdown to dev dependencies Bug fix: - Fix pyln-client version.py: __all__ must contain strings, not class objects This was causing "TypeError: attribute name must be string, not 'type'" in pdoc3 Documentation is generated to docs/python/ which is excluded from version control. Run 'make python-docs' to generate locally, or download from nightly workflow artifacts. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
2025-11-29 19:21:20 +01:00
parent 4b9cffe183
commit 5c9d3884bb
7 changed files with 294 additions and 2 deletions
--- a/.github/workflows/python-docs-nightly.yaml
+++ b/.github/workflows/python-docs-nightly.yaml
@@ -0,0 +1,58 @@
+name: Python API Docs (Nightly)
+
+on:
+  schedule:
+    # Run at 3 AM UTC every day
+    - cron: '0 3 * * *'
+  # Allow manual triggers for testing
+  workflow_dispatch:
+
+concurrency:
+  group: python-docs-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  generate-docs:
+    name: Generate Python API Documentation
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install dependencies
+        run: |
+          uv sync --all-extras
+
+      - name: Generate documentation
+        run: |
+          make python-docs
+
+      - name: Upload documentation artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: python-api-docs
+          path: docs/python
+          retention-days: 90
+
+      - name: Add summary to job
+        run: |
+          echo "## Python API Documentation Generated" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "📚 Documentation has been generated for the following packages:" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "- pyln.client - Client library and plugin library" >> $GITHUB_STEP_SUMMARY
+          echo "- pyln.proto - Lightning Network protocol implementation" >> $GITHUB_STEP_SUMMARY
+          echo "- pyln.grpc - gRPC protocol definitions" >> $GITHUB_STEP_SUMMARY
+          echo "- pyln.testing - Testing utilities" >> $GITHUB_STEP_SUMMARY
+          echo "- pyln.spec.bolt7 - BOLT #7 specification implementation" >> $GITHUB_STEP_SUMMARY
+          echo "" >> $GITHUB_STEP_SUMMARY
+          echo "Download the artifact to view the complete API documentation." >> $GITHUB_STEP_SUMMARY
--- a/.gitignore
+++ b/.gitignore
@@ -27,6 +27,8 @@ coverage
 # Coverage profiling data files
 *.profraw
 *.profdata
+# Generated Python API documentation
+docs/python
 ccan/config.h
 __pycache__
 config.vars
--- a/9
+++ b/9
@@ -689,6 +689,15 @@ coverage-clang-clean:

 .PHONY: coverage-clang-collect coverage-clang-report coverage-clang coverage-clang-clean

+# Python API documentation targets
+python-docs:
+	@./contrib/api/generate-python-docs.py
+
+python-docs-clean:
+	rm -rf docs/python
+
+.PHONY: python-docs python-docs-clean
+
 # We make libwallycore.la a dependency, so that it gets built normally, without ncc.
 # Ncc can't handle the libwally source code (yet).
 ncc: ${TARGET_DIR}/libwally-core-build/src/libwallycore.la
--- a/contrib/api/generate-python-docs.py
+++ b/contrib/api/generate-python-docs.py
@@ -0,0 +1,179 @@
+#!/usr/bin/env python3
+"""
+Generate Python API documentation for all workspace packages using pdoc3.
+
+This script generates HTML documentation for all Python packages in the
+Core Lightning workspace and creates an index page linking to all of them.
+"""
+
+import os
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+
+# Define packages to document (module name -> source directory)
+# Only includes packages that are in the workspace and can be imported
+PACKAGES = {
+    "pyln.client": "contrib/pyln-client",
+    "pyln.proto": "contrib/pyln-proto",
+    "pyln.grpc": "contrib/pyln-grpc-proto",
+    "pyln.testing": "contrib/pyln-testing",
+    "pyln.spec.bolt7": "contrib/pyln-spec/bolt7",
+}
+
+INDEX_HTML_TEMPLATE = """<!DOCTYPE html>
+<html>
+<head>
+    <title>Core Lightning Python Packages Documentation</title>
+    <style>
+        body {{
+            font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif;
+            max-width: 1200px;
+            margin: 0 auto;
+            padding: 40px 20px;
+            line-height: 1.6;
+        }}
+        h1 {{
+            border-bottom: 2px solid #eaecef;
+            padding-bottom: 0.3em;
+        }}
+        .package-grid {{
+            display: grid;
+            grid-template-columns: repeat(auto-fill, minmax(300px, 1fr));
+            gap: 20px;
+            margin-top: 30px;
+        }}
+        .package-card {{
+            border: 1px solid #e1e4e8;
+            border-radius: 6px;
+            padding: 20px;
+            transition: box-shadow 0.2s;
+        }}
+        .package-card:hover {{
+            box-shadow: 0 4px 8px rgba(0,0,0,0.1);
+        }}
+        .package-card h2 {{
+            margin-top: 0;
+            font-size: 1.3em;
+        }}
+        .package-card a {{
+            color: #0366d6;
+            text-decoration: none;
+        }}
+        .package-card a:hover {{
+            text-decoration: underline;
+        }}
+        .package-description {{
+            color: #586069;
+            font-size: 0.9em;
+            margin-top: 8px;
+        }}
+        .timestamp {{
+            color: #586069;
+            font-size: 0.9em;
+            margin-top: 30px;
+            text-align: center;
+        }}
+    </style>
+</head>
+<body>
+    <h1>Core Lightning Python Packages Documentation</h1>
+    <p>This page provides links to the API documentation for all Python packages in the Core Lightning workspace.</p>
+
+    <div class="package-grid">
+        <div class="package-card">
+            <h2><a href="pyln/client/index.html">pyln.client</a></h2>
+            <p class="package-description">Client library and plugin library for Core Lightning</p>
+        </div>
+
+        <div class="package-card">
+            <h2><a href="pyln/proto/index.html">pyln.proto</a></h2>
+            <p class="package-description">Lightning Network protocol implementation</p>
+        </div>
+
+        <div class="package-card">
+            <h2><a href="pyln/grpc/index.html">pyln.grpc</a></h2>
+            <p class="package-description">gRPC protocol definitions for Core Lightning</p>
+        </div>
+
+        <div class="package-card">
+            <h2><a href="pyln/testing/index.html">pyln.testing</a></h2>
+            <p class="package-description">Testing utilities for Core Lightning</p>
+        </div>
+
+        <div class="package-card">
+            <h2><a href="pyln/spec/bolt7/index.html">pyln.spec.bolt7</a></h2>
+            <p class="package-description">BOLT #7 specification implementation</p>
+        </div>
+    </div>
+
+    <p class="timestamp">Generated on {timestamp}</p>
+</body>
+</html>
+"""
+
+
+def generate_docs(output_dir: Path, repo_root: Path):
+    """Generate documentation for all packages."""
+    print(f"Generating Python documentation for all workspace packages...")
+    print(f"Output directory: {output_dir}")
+
+    # Clean and create output directory
+    if output_dir.exists():
+        import shutil
+        shutil.rmtree(output_dir)
+    output_dir.mkdir(parents=True)
+
+    # Change to repo root for imports to work correctly
+    os.chdir(repo_root)
+
+    # Generate documentation for each package
+    for package, source_dir in PACKAGES.items():
+        print(f"Generating docs for {package} (from {source_dir})...")
+
+        try:
+            # Use pdoc3 to generate HTML documentation
+            subprocess.run(
+                [
+                    "uv", "run", "pdoc3",
+                    "--html",
+                    "--output-dir", str(output_dir),
+                    "--force",
+                    package
+                ],
+                check=True,
+                cwd=repo_root,
+            )
+        except subprocess.CalledProcessError as e:
+            print(f"Warning: Failed to generate docs for {package}, skipping...")
+            print(f"Error: {e}")
+            continue
+
+    # Create index.html
+    index_path = output_dir / "index.html"
+    timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
+    index_path.write_text(INDEX_HTML_TEMPLATE.format(timestamp=timestamp))
+
+    print("\nDocumentation generated successfully!")
+    print(f"Open {output_dir}/index.html in your browser to view the documentation.")
+
+
+def main():
+    """Main entry point."""
+    # Determine paths
+    script_dir = Path(__file__).parent.resolve()
+    repo_root = script_dir.parent.parent
+
+    # Default output directory
+    output_dir = repo_root / "docs" / "python"
+
+    # Allow override via command line argument
+    if len(sys.argv) > 1:
+        output_dir = Path(sys.argv[1])
+
+    generate_docs(output_dir, repo_root)
+
+
+if __name__ == "__main__":
+    main()
--- a/contrib/pyln-client/pyln/client/version.py
+++ b/contrib/pyln-client/pyln/client/version.py
@@ -75,4 +75,4 @@ class NodeVersion:
        return False


-__all__ = [NodeVersion]
+__all__ = ["NodeVersion"]
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,7 +24,7 @@ package-mode = false
 [dependency-groups]
 dev = [
  # Test dependencies and inherited dependencies belong here
-  "crc32c>=2.2.post0",              # Belongs to lnprototest
+  "crc32c>=2.2.post0", # Belongs to lnprototest
  "pytest>=8.0.0",
  "pytest-xdist>=3.6.0",
  "pytest-test-groups>=1.2.0",
@@ -35,6 +35,7 @@ dev = [
  "flask-socketio>=5",
  "tqdm",
  "pytest-benchmark",
+  "pdoc3>=0.11.6",
 ]

 [project.optional-dependencies]
--- a/uv.lock
+++ b/uv.lock
@@ -457,6 +457,7 @@ dev = [
    { name = "crc32c" },
    { name = "flake8" },
    { name = "flask-socketio" },
+    { name = "pdoc3" },
    { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
    { name = "pytest", version = "9.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
    { name = "pytest-benchmark" },
@@ -491,6 +492,7 @@ dev = [
    { name = "crc32c", specifier = ">=2.2.post0" },
    { name = "flake8", specifier = ">=7.0" },
    { name = "flask-socketio", specifier = ">=5" },
+    { name = "pdoc3", specifier = ">=0.11.6" },
    { name = "pytest", specifier = ">=8.0.0" },
    { name = "pytest-benchmark" },
    { name = "pytest-custom-exit-code", specifier = "==0.3.0" },
@@ -1061,6 +1063,33 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/87/fb/99f81ac72ae23375f22b7afdb7642aba97c00a713c217124420147681a2f/mako-1.3.10-py3-none-any.whl", hash = "sha256:baef24a52fc4fc514a0887ac600f9f1cff3d82c61d4d700a1fa84d597b88db59", size = 78509, upload-time = "2025-04-10T12:50:53.297Z" },
 ]

+[[package]]
+name = "markdown"
+version = "3.9"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version < '3.10'",
+]
+dependencies = [
+    { name = "importlib-metadata", marker = "python_full_version < '3.10'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/8d/37/02347f6d6d8279247a5837082ebc26fc0d5aaeaf75aa013fcbb433c777ab/markdown-3.9.tar.gz", hash = "sha256:d2900fe1782bd33bdbbd56859defef70c2e78fc46668f8eb9df3128138f2cb6a", size = 364585, upload-time = "2025-09-04T20:25:22.885Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/70/ae/44c4a6a4cbb496d93c6257954260fe3a6e91b7bed2240e5dad2a717f5111/markdown-3.9-py3-none-any.whl", hash = "sha256:9f4d91ed810864ea88a6f32c07ba8bee1346c0cc1f6b1f9f6c822f2a9667d280", size = 107441, upload-time = "2025-09-04T20:25:21.784Z" },
+]
+
+[[package]]
+name = "markdown"
+version = "3.10"
+source = { registry = "https://pypi.org/simple" }
+resolution-markers = [
+    "python_full_version >= '3.10'",
+]
+sdist = { url = "https://files.pythonhosted.org/packages/7d/ab/7dd27d9d863b3376fcf23a5a13cb5d024aed1db46f963f1b5735ae43b3be/markdown-3.10.tar.gz", hash = "sha256:37062d4f2aa4b2b6b32aefb80faa300f82cc790cb949a35b8caede34f2b68c0e", size = 364931, upload-time = "2025-11-03T19:51:15.007Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/70/81/54e3ce63502cd085a0c556652a4e1b919c45a446bd1e5300e10c44c8c521/markdown-3.10-py3-none-any.whl", hash = "sha256:b5b99d6951e2e4948d939255596523444c0e677c669700b1d17aa4a8a464cb7c", size = 107678, upload-time = "2025-11-03T19:51:13.887Z" },
+]
+
 [[package]]
 name = "markupsafe"
 version = "3.0.3"
@@ -1197,6 +1226,20 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" },
 ]

+[[package]]
+name = "pdoc3"
+version = "0.11.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "mako" },
+    { name = "markdown", version = "3.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
+    { name = "markdown", version = "3.10", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ed/f0/07d8b771b99c16a06741cd7b2639494a15357df819ecf899c33b87db6257/pdoc3-0.11.6.tar.gz", hash = "sha256:1ea5e84b87a754d191fb64bf5e517ca6c50d0d84a614c1efecf6b46d290ae387", size = 177107, upload-time = "2025-03-20T22:53:53.099Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/df/98/629f269c2bd91bdcac147aad5cf51ceb645c0196e23a41ee3c051125190f/pdoc3-0.11.6-py3-none-any.whl", hash = "sha256:8b72723767bd48d899812d2aec8375fc1c3476e179455db0b4575e6dccb44b93", size = 255188, upload-time = "2025-03-20T22:53:51.671Z" },
+]
+
 [[package]]
 name = "pluggy"
 version = "1.6.0"