rag-from-scratch/tests/chunks/test_parser.py

from chunks.parser import parse


def test_parse_returns_tokens_and_lines():
    md = "# Titolo\n\nParagrafo.\n"
    tokens, lines = parse(md)
    assert len(tokens) > 0
    assert lines[0] == "# Titolo"
    assert lines[2] == "Paragrafo."


def test_tokens_have_source_map():
    md = "# Titolo\n\nParagrafo.\n"
    tokens, lines = parse(md)
    heading = next(t for t in tokens if t.type == "heading_open")
    assert heading.map is not None
    assert heading.map[0] == 0


def test_parse_code_fence():
    md = "# T\n\n```python\ncodice\n```\n"
    tokens, lines = parse(md)
    fence = next((t for t in tokens if t.type == "fence"), None)
    assert fence is not None
    assert "codice" in fence.content


def test_parse_table():
    md = "| A | B |\n|---|---|\n| 1 | 2 |\n"
    tokens, lines = parse(md)
    types = [t.type for t in tokens]
    assert "table_open" in types


def test_parse_list():
    md = "- item 1\n- item 2\n"
    tokens, lines = parse(md)
    types = [t.type for t in tokens]
    assert "bullet_list_open" in types


def test_lines_preserves_source():
    md = "Riga 1\nRiga 2\nRiga 3\n"
    _, lines = parse(md)
    assert lines[0] == "Riga 1"
    assert lines[1] == "Riga 2"


def test_parse_math_block():
    md = "# T\n\n$$\nE=mc^2\n$$\n"
    tokens, lines = parse(md)
    types = [t.type for t in tokens]
    assert "math_block" in types


def test_math_block_has_source_map():
    md = "# T\n\n$$\nE=mc^2\n$$\n"
    tokens, lines = parse(md)
    mb = next(t for t in tokens if t.type == "math_block")
    assert mb.map is not None
    assert "E=mc^2" in mb.content


def test_parse_math_inline_stays_in_paragraph():
    md = "Testo con $x^2$ inline.\n"
    tokens, lines = parse(md)
    types = [t.type for t in tokens]
    assert "paragraph_open" in types
    assert "math_block" not in types