diff --git a/chunks/parser.py b/chunks/parser.py new file mode 100644 index 0000000..0095bc0 --- /dev/null +++ b/chunks/parser.py @@ -0,0 +1,16 @@ +from __future__ import annotations +from markdown_it import MarkdownIt +from markdown_it.token import Token + + +def parse(source: str) -> tuple[list[Token], list[str]]: + """Parsa Markdown in token stream con source map. + + Returns: + tokens: lista Token con .map = [start_line, end_line] (0-indexed, end escluso) + lines: righe sorgente (0-indexed) per ricostruzione testo esatto + """ + md = MarkdownIt().enable("table") + tokens = md.parse(source) + lines = source.splitlines() + return tokens, lines diff --git a/tests/chunks/test_parser.py b/tests/chunks/test_parser.py new file mode 100644 index 0000000..1b16fb6 --- /dev/null +++ b/tests/chunks/test_parser.py @@ -0,0 +1,46 @@ +from chunks.parser import parse + + +def test_parse_returns_tokens_and_lines(): + md = "# Titolo\n\nParagrafo.\n" + tokens, lines = parse(md) + assert len(tokens) > 0 + assert lines[0] == "# Titolo" + assert lines[2] == "Paragrafo." + + +def test_tokens_have_source_map(): + md = "# Titolo\n\nParagrafo.\n" + tokens, lines = parse(md) + heading = next(t for t in tokens if t.type == "heading_open") + assert heading.map is not None + assert heading.map[0] == 0 + + +def test_parse_code_fence(): + md = "# T\n\n```python\ncodice\n```\n" + tokens, lines = parse(md) + fence = next((t for t in tokens if t.type == "fence"), None) + assert fence is not None + assert "codice" in fence.content + + +def test_parse_table(): + md = "| A | B |\n|---|---|\n| 1 | 2 |\n" + tokens, lines = parse(md) + types = [t.type for t in tokens] + assert "table_open" in types + + +def test_parse_list(): + md = "- item 1\n- item 2\n" + tokens, lines = parse(md) + types = [t.type for t in tokens] + assert "bullet_list_open" in types + + +def test_lines_preserves_source(): + md = "Riga 1\nRiga 2\nRiga 3\n" + _, lines = parse(md) + assert lines[0] == "Riga 1" + assert lines[1] == "Riga 2"