diff --git a/chunks/packer.py b/chunks/packer.py index 0537657..cdb9dd5 100644 --- a/chunks/packer.py +++ b/chunks/packer.py @@ -41,9 +41,18 @@ def _build_chunk(blocks: list[Block], index: int, config: ChunkerConfig, header_path=header_path, block_ids=[b.id for b in blocks], flags={ - "has_code": any(b.kind == "code" for b in blocks), - "has_table": any(b.kind == "table" for b in blocks), - "has_math": any(b.kind == "math" for b in blocks), + "has_code": any(b.kind == "code" for b in blocks), + "has_table": any( + b.kind == "table" + or (b.kind == "html" and " config.max_chars, "is_sparse": total_chars < config.min_chars, }, diff --git a/chunks/parser.py b/chunks/parser.py index 0095bc0..1bd5844 100644 --- a/chunks/parser.py +++ b/chunks/parser.py @@ -2,6 +2,12 @@ from __future__ import annotations from markdown_it import MarkdownIt from markdown_it.token import Token +try: + from mdit_py_plugins.dollarmath import dollarmath_plugin as _dollarmath + _HAS_DOLLARMATH = True +except ImportError: + _HAS_DOLLARMATH = False + def parse(source: str) -> tuple[list[Token], list[str]]: """Parsa Markdown in token stream con source map. @@ -11,6 +17,8 @@ def parse(source: str) -> tuple[list[Token], list[str]]: lines: righe sorgente (0-indexed) per ricostruzione testo esatto """ md = MarkdownIt().enable("table") + if _HAS_DOLLARMATH: + md = md.use(_dollarmath, allow_labels=False, allow_space=False) tokens = md.parse(source) lines = source.splitlines() return tokens, lines diff --git a/chunks/segmenter.py b/chunks/segmenter.py index e95ab8f..a3e3d3d 100644 --- a/chunks/segmenter.py +++ b/chunks/segmenter.py @@ -191,6 +191,25 @@ def segment(tokens: list[Token], lines: list[str], config: ChunkerConfig) -> lis i += 1 continue + # ── Math block (dollarmath plugin) ─────────────────────────────────── + if tok.type == "math_block": + start = tok.map[0] if tok.map else 0 + end = tok.map[1] if tok.map else start + 1 + content = _lines_content(lines, start, end) + blocks.append(Block( + id=_make_id(), + kind="math", + content=content, + plain_text="[formula]", + atomic=True, + start_line=start, + end_line=end, + header_path=header_path, + chars=len(content), + )) + i += 1 + continue + # ── HTML block ──────────────────────────────────────────────────────── if tok.type == "html_block": start = tok.map[0] if tok.map else 0 diff --git a/tests/chunks/test_packer.py b/tests/chunks/test_packer.py index 7dfd2e3..72cf2dd 100644 --- a/tests/chunks/test_packer.py +++ b/tests/chunks/test_packer.py @@ -116,3 +116,35 @@ def test_assets_empty(cfg): blocks = [make_block(1, "Testo.")] chunks = pack(blocks, cfg, "test") assert chunks[0].assets == [] + + +def test_flags_has_math_via_kind(cfg): + blocks = [make_block(1, "$$\nE=mc^2\n$$", kind="math", atomic=True)] + chunks = pack(blocks, cfg, "test") + assert chunks[0].flags["has_math"] is True + + +def test_flags_has_math_via_content_dollars(cfg): + # Paragrafo con $$ inline (senza dollarmath plugin attivo) + blocks = [make_block(1, "Vedi la formula:\n$$\nx^2\n$$")] + chunks = pack(blocks, cfg, "test") + assert chunks[0].flags["has_math"] is True + + +def test_flags_has_math_via_content_begin(cfg): + blocks = [make_block(1, r"Equazione: \begin{align} x = 1 \end{align}")] + chunks = pack(blocks, cfg, "test") + assert chunks[0].flags["has_math"] is True + + +def test_flags_has_table_via_html(cfg): + html_content = "
A
" + blocks = [make_block(1, html_content, kind="html", atomic=True)] + chunks = pack(blocks, cfg, "test") + assert chunks[0].flags["has_table"] is True + + +def test_flags_no_math_plain_text(cfg): + blocks = [make_block(1, "Testo normale senza formule.")] + chunks = pack(blocks, cfg, "test") + assert chunks[0].flags["has_math"] is False diff --git a/tests/chunks/test_parser.py b/tests/chunks/test_parser.py index 1b16fb6..eb49285 100644 --- a/tests/chunks/test_parser.py +++ b/tests/chunks/test_parser.py @@ -44,3 +44,26 @@ def test_lines_preserves_source(): _, lines = parse(md) assert lines[0] == "Riga 1" assert lines[1] == "Riga 2" + + +def test_parse_math_block(): + md = "# T\n\n$$\nE=mc^2\n$$\n" + tokens, lines = parse(md) + types = [t.type for t in tokens] + assert "math_block" in types + + +def test_math_block_has_source_map(): + md = "# T\n\n$$\nE=mc^2\n$$\n" + tokens, lines = parse(md) + mb = next(t for t in tokens if t.type == "math_block") + assert mb.map is not None + assert "E=mc^2" in mb.content + + +def test_parse_math_inline_stays_in_paragraph(): + md = "Testo con $x^2$ inline.\n" + tokens, lines = parse(md) + types = [t.type for t in tokens] + assert "paragraph_open" in types + assert "math_block" not in types diff --git a/tests/chunks/test_segmenter.py b/tests/chunks/test_segmenter.py index 81b87fa..9cf1a9e 100644 --- a/tests/chunks/test_segmenter.py +++ b/tests/chunks/test_segmenter.py @@ -126,3 +126,30 @@ def test_source_line_numbers(cfg): para = next(b for b in blocks if b.kind == "paragraph") assert para.start_line == 2 assert para.end_line == 3 + + +def test_math_block_is_atomic(cfg): + md = "# T\n\n$$\nE=mc^2\n$$\n" + tokens, lines = parse(md) + blocks = segment(tokens, lines, cfg) + math = next((b for b in blocks if b.kind == "math"), None) + assert math is not None + assert math.atomic is True + assert "E=mc^2" in math.content + + +def test_math_block_plain_text(cfg): + md = "# T\n\n$$\n\\int_0^1 f(x) dx\n$$\n" + tokens, lines = parse(md) + blocks = segment(tokens, lines, cfg) + math = next(b for b in blocks if b.kind == "math") + assert math.plain_text == "[formula]" + + +def test_html_block_with_table(cfg): + md = "# T\n\n
A
\n" + tokens, lines = parse(md) + blocks = segment(tokens, lines, cfg) + html = next((b for b in blocks if b.kind == "html"), None) + assert html is not None + assert "