diff --git a/contrib/pytest-trackflaky/README.md b/contrib/pytest-trackflaky/README.md new file mode 100644 index 000000000..fb4ee939f --- /dev/null +++ b/contrib/pytest-trackflaky/README.md @@ -0,0 +1,63 @@ +# pytest-trackflaky + +A pytest plugin to track and report test flakiness to a central server. + +## Features + +- Automatically tracks test execution times and outcomes +- Collects GitHub Actions metadata (commit SHA, branch, run ID, etc.) +- Reports test results to a configurable server endpoint +- Zero configuration needed when running in CI environments + +## Installation + +Install the plugin using pip or uv: + +```bash +pip install -e contrib/pytest-trackflaky +``` + +Or with uv: + +```bash +uv pip install -e contrib/pytest-trackflaky +``` + +## Usage + +Once installed, the plugin is automatically activated when running pytest. No additional configuration is needed. + +### Configuration + +The plugin is controlled via environment variables: + +- `CI_SERVER_URL`: The base URL of the server to report results to (required for reporting) + - Test results will be POSTed to `{CI_SERVER_URL}/hook/test` +- `GITHUB_*`: Standard GitHub Actions environment variables are automatically collected + +### Example + +```bash +export CI_SERVER_URL="https://your-flaky-tracker.example.com" +pytest +``` + +## Data Collected + +For each test, the plugin collects: + +- Test name +- Outcome (success/skip/fail) +- Start and end times +- GitHub repository information +- Git commit SHA and branch +- GitHub Actions run metadata + +## Development + +To work on the plugin locally: + +```bash +cd contrib/pytest-trackflaky +pip install -e . +``` diff --git a/contrib/pytest-trackflaky/pyproject.toml b/contrib/pytest-trackflaky/pyproject.toml new file mode 100644 index 000000000..8f4cca05b --- /dev/null +++ b/contrib/pytest-trackflaky/pyproject.toml @@ -0,0 +1,19 @@ +[project] +name = "pytest-trackflaky" +version = "0.1.0" +description = "A pytest plugin to track and report test flakiness" +authors = [ + {name = "Lightning Development Team"} +] +readme = "README.md" +requires-python = ">=3.8" +dependencies = [ + "pytest>=7.0.0", +] + +[project.entry-points.pytest11] +trackflaky = "pytest_trackflaky.plugin" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/contrib/pytest-trackflaky/pytest_trackflaky/__init__.py b/contrib/pytest-trackflaky/pytest_trackflaky/__init__.py new file mode 100644 index 000000000..494b29959 --- /dev/null +++ b/contrib/pytest-trackflaky/pytest_trackflaky/__init__.py @@ -0,0 +1,3 @@ +"""pytest-trackflaky: A pytest plugin to track and report test flakiness.""" + +__version__ = "0.1.0" diff --git a/contrib/pytest-trackflaky/pytest_trackflaky/plugin.py b/contrib/pytest-trackflaky/pytest_trackflaky/plugin.py new file mode 100644 index 000000000..1162661a2 --- /dev/null +++ b/contrib/pytest-trackflaky/pytest_trackflaky/plugin.py @@ -0,0 +1,296 @@ +"""pytest-trackflaky plugin implementation.""" + +import pytest +import subprocess +from urllib import request +import os +import json +from time import time +import unittest +import threading + + +# Global state for run tracking +_run_id = None +_run_id_lock = threading.Lock() +server = os.environ.get("CI_SERVER_URL", None) + + +class SnowflakeGenerator: + """ + Generates Twitter-style Snowflake IDs. + + Format (64 bits): + - 41 bits: timestamp in milliseconds since custom epoch + - 10 bits: worker/machine ID + - 12 bits: sequence number + """ + + # Custom epoch (2024-01-01 00:00:00 UTC in milliseconds) + EPOCH = 1704067200000 + + # Bit allocation + TIMESTAMP_BITS = 41 + WORKER_BITS = 10 + SEQUENCE_BITS = 12 + + # Max values + MAX_WORKER_ID = (1 << WORKER_BITS) - 1 + MAX_SEQUENCE = (1 << SEQUENCE_BITS) - 1 + + # Bit shifts + TIMESTAMP_SHIFT = WORKER_BITS + SEQUENCE_BITS + WORKER_SHIFT = SEQUENCE_BITS + + def __init__(self, worker_id=None): + """Initialize the snowflake generator.""" + if worker_id is None: + # Try to get worker ID from environment or use process ID + worker_id = os.getpid() & self.MAX_WORKER_ID + + if worker_id > self.MAX_WORKER_ID or worker_id < 0: + raise ValueError(f"Worker ID must be between 0 and {self.MAX_WORKER_ID}") + + self.worker_id = worker_id + self.sequence = 0 + self.last_timestamp = -1 + self.lock = threading.Lock() + + def _current_timestamp(self): + """Get current timestamp in milliseconds since epoch.""" + return int(time() * 1000) + + def generate(self): + """Generate a new Snowflake ID.""" + with self.lock: + timestamp = self._current_timestamp() - self.EPOCH + + if timestamp < self.last_timestamp: + raise Exception("Clock moved backwards. Refusing to generate ID.") + + if timestamp == self.last_timestamp: + self.sequence = (self.sequence + 1) & self.MAX_SEQUENCE + if self.sequence == 0: + # Sequence exhausted, wait for next millisecond + while timestamp <= self.last_timestamp: + timestamp = self._current_timestamp() - self.EPOCH + else: + self.sequence = 0 + + self.last_timestamp = timestamp + + # Combine all parts + snowflake_id = ( + (timestamp << self.TIMESTAMP_SHIFT) + | (self.worker_id << self.WORKER_SHIFT) + | self.sequence + ) + + return snowflake_id + + +# Global snowflake generator +_snowflake_gen = SnowflakeGenerator() + + +def get_git_sha(): + """Get the current git commit SHA.""" + try: + return ( + subprocess.check_output(["git", "rev-parse", "HEAD"]) + .decode("ASCII") + .strip() + ) + except subprocess.CalledProcessError: + return None + + +def get_git_branch(): + """Get the current git branch name.""" + try: + return ( + subprocess.check_output(["git", "rev-parse", "--abbrev-ref", "HEAD"]) + .decode("ASCII") + .strip() + ) + except subprocess.CalledProcessError: + return None + + +def get_git_repository(): + """ + Detect the git repository from remotes. + + Checks refs in order: master@upstream, main@upstream, master@origin, main@origin. + Returns repository in "owner/repo" format (e.g., "ElementsProject/lightning"). + """ + # Check these refs in order + refs_to_check = [ + "master@upstream", + "main@upstream", + "master@origin", + "main@origin" + ] + + for ref in refs_to_check: + try: + # Try to get the URL for this ref + remote_url = ( + subprocess.check_output( + ["git", "config", "--get", f"remote.{ref.split('@')[1]}.url"], + stderr=subprocess.DEVNULL + ) + .decode("ASCII") + .strip() + ) + + # Parse the URL to extract owner/repo + # Handle various formats: + # - https://github.com/owner/repo.git + # - git@github.com:owner/repo.git + # - https://github.com/owner/repo + + if remote_url.startswith("git@"): + # SSH format: git@github.com:owner/repo.git + path = remote_url.split(":", 1)[1] + elif "://" in remote_url: + # HTTPS format: https://github.com/owner/repo.git + path = remote_url.split("://", 1)[1] + # Remove the domain part + if "/" in path: + path = "/".join(path.split("/")[1:]) + else: + # Unknown format, try next ref + continue + + # Remove .git suffix if present + if path.endswith(".git"): + path = path[:-4] + + # Ensure we have owner/repo format + parts = path.split("/") + if len(parts) >= 2: + return f"{parts[-2]}/{parts[-1]}" + + except subprocess.CalledProcessError: + # This ref doesn't exist, try the next one + continue + + return None + + +def get_run_id(): + """Get or generate the run ID for this test session.""" + global _run_id + with _run_id_lock: + if _run_id is None: + _run_id = _snowflake_gen.generate() + return _run_id + + +def set_run_id(run_id): + """Set the run ID (used by workers to inherit from main process).""" + global _run_id + with _run_id_lock: + _run_id = run_id + + +def get_base_result(): + """Collect base result information from environment and git.""" + github_sha = get_git_sha() + github_ref_name = get_git_branch() + github_run_id = os.environ.get("GITHUB_RUN_ID", None) + run_number = os.environ.get("GITHUB_RUN_NUMBER", None) + + # Auto-detect repository from git remotes if not in environment + github_repository = os.environ.get("GITHUB_REPOSITORY", None) + if not github_repository: + github_repository = get_git_repository() + + return { + "run_id": get_run_id(), + "github_repository": github_repository, + "github_sha": os.environ.get("GITHUB_SHA", github_sha), + "github_ref": os.environ.get("GITHUB_REF", None), + "github_ref_name": github_ref_name, + "github_run_id": int(github_run_id) if github_run_id else None, + "github_head_ref": os.environ.get("GITHUB_HEAD_REF", None), + "github_run_number": int(run_number) if run_number else None, + "github_base_ref": os.environ.get("GITHUB_BASE_REF", None), + "github_run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT", None), + } + + +def pytest_configure(config): + """Generate a unique run ID when pytest starts.""" + # Generate the run ID early so it's available for all tests + get_run_id() + + +def pytest_report_header(config): + """Add run ID to pytest header.""" + run_id = get_run_id() + return f"Run ID: {run_id}, server: {server}" + + +def pytest_configure_node(node): + """ + Configure worker nodes to inherit the run ID from the main process. + + This hook is called by pytest-xdist to configure worker nodes. + """ + node.workerinput["trackflaky_run_id"] = get_run_id() + + +@pytest.hookimpl(tryfirst=True) +def pytest_sessionstart(session): + """ + Initialize run ID from worker input if this is a worker process. + + This runs on worker nodes to receive the run ID from the main process. + """ + if hasattr(session.config, "workerinput"): + # We're in a worker process + workerinput = session.config.workerinput + if "trackflaky_run_id" in workerinput: + set_run_id(workerinput["trackflaky_run_id"]) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_pyfunc_call(pyfuncitem): + """Hook into pytest test execution to track test outcomes.""" + + result = get_base_result() + result["testname"] = pyfuncitem.name + result["start_time"] = int(time()) + + outcome = yield + + result["end_time"] = int(time()) + + if outcome.excinfo is None: + result["outcome"] = "success" + elif outcome.excinfo[0] == unittest.case.SkipTest: + result["outcome"] = "skip" + else: + result["outcome"] = "fail" + + print(result) + + if not server: + return + + try: + req = request.Request(f"{server}/hook/test", method="POST") + req.add_header("Content-Type", "application/json") + + request.urlopen( + req, + data=json.dumps(result).encode("ASCII"), + ) + except ConnectionError as e: + print(f"Could not report testrun: {e}") + except Exception as e: + import warnings + + warnings.warn(f"Error reporting testrun: {e}") diff --git a/pyproject.toml b/pyproject.toml index 400393761..61bc72719 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,8 @@ dependencies = [ "pyln-client", "pyln-proto", "pyln-grpc-proto", + "pytest-trackflaky", + "pyln-testing", ] package-mode = false [dependency-groups] @@ -54,6 +56,7 @@ members = [ "contrib/pyln-spec/bolt2", "contrib/pyln-spec/bolt4", "contrib/pyln-spec/bolt7", + "contrib/pytest-trackflaky", ] [tool.uv.sources] @@ -61,8 +64,9 @@ pyln-client = { workspace = true } pyln-proto = { workspace = true } pyln-grpc-proto = { workspace = true } wss-proxy = { workspace = true } -pyln-testing = { workspace = true } pyln-bolt1 = { workspace = true } pyln-bolt2 = { workspace = true } pyln-bolt4 = { workspace = true } pyln-bolt7 = { workspace = true } +pytest-trackflaky = { workspace = true } +pyln-testing = { workspace = true } diff --git a/uv.lock b/uv.lock index 6b57a1079..d770d1287 100644 --- a/uv.lock +++ b/uv.lock @@ -17,6 +17,7 @@ members = [ "pyln-grpc-proto", "pyln-proto", "pyln-testing", + "pytest-trackflaky", ] [[package]] @@ -400,6 +401,8 @@ dependencies = [ { name = "pyln-client" }, { name = "pyln-grpc-proto" }, { name = "pyln-proto" }, + { name = "pyln-testing" }, + { name = "pytest-trackflaky" }, { name = "websocket-client" }, ] @@ -434,7 +437,9 @@ requires-dist = [ { name = "pyln-client", editable = "contrib/pyln-client" }, { name = "pyln-grpc-proto", editable = "contrib/pyln-grpc-proto" }, { name = "pyln-proto", editable = "contrib/pyln-proto" }, + { name = "pyln-testing", editable = "contrib/pyln-testing" }, { name = "pyln-testing", marker = "extra == 'grpc'", editable = "contrib/pyln-testing" }, + { name = "pytest-trackflaky", editable = "contrib/pytest-trackflaky" }, { name = "websocket-client", specifier = ">=1.2.3" }, ] provides-extras = ["grpc"] @@ -1568,6 +1573,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/b6/3127540ecdf1464a00e5a01ee60a1b09175f6913f0644ac748494d9c4b21/pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2", size = 14382, upload-time = "2025-05-05T19:44:33.502Z" }, ] +[[package]] +name = "pytest-trackflaky" +version = "0.1.0" +source = { editable = "contrib/pytest-trackflaky" } +dependencies = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [{ name = "pytest", specifier = ">=7.0.0" }] + [[package]] name = "pytest-xdist" version = "3.8.0"