Files
rag-from-scratch/chunks/parser.py
T
2026-06-08 16:22:58 +02:00

17 lines
527 B
Python

from __future__ import annotations
from markdown_it import MarkdownIt
from markdown_it.token import Token
def parse(source: str) -> tuple[list[Token], list[str]]:
"""Parsa Markdown in token stream con source map.
Returns:
tokens: lista Token con .map = [start_line, end_line] (0-indexed, end escluso)
lines: righe sorgente (0-indexed) per ricostruzione testo esatto
"""
md = MarkdownIt().enable("table")
tokens = md.parse(source)
lines = source.splitlines()
return tokens, lines