rag-from-scratch/chunks/parser.py

from __future__ import annotations
from markdown_it import MarkdownIt
from markdown_it.token import Token

try:
    from mdit_py_plugins.dollarmath import dollarmath_plugin as _dollarmath
    _HAS_DOLLARMATH = True
except ImportError:
    _HAS_DOLLARMATH = False


def parse(source: str) -> tuple[list[Token], list[str]]:
    """Parsa Markdown in token stream con source map.

    Returns:
        tokens: lista Token con .map = [start_line, end_line] (0-indexed, end escluso)
        lines:  righe sorgente (0-indexed) per ricostruzione testo esatto
    """
    md = MarkdownIt().enable("table")
    if _HAS_DOLLARMATH:
        md = md.use(_dollarmath, allow_labels=False, allow_space=False)
    tokens = md.parse(source)
    lines = source.splitlines()
    return tokens, lines