feat(chunks): rilevamento universale has_math/has_table + dollarmath plugin
- parser.py: integra dollarmath_plugin (graceful fallback) — blocchi $$…$$ → token math_block atomico
- segmenter.py: gestisce math_block → Block(kind=math, atomic=True, plain_text=[formula])
- packer.py: has_table rileva anche <table> in blocchi html; has_math rileva $$ e \begin{ nel contenuto
- +11 test (parser×3, segmenter×3, packer×5) — totale 65 test
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+12
-3
@@ -41,9 +41,18 @@ def _build_chunk(blocks: list[Block], index: int, config: ChunkerConfig,
|
|||||||
header_path=header_path,
|
header_path=header_path,
|
||||||
block_ids=[b.id for b in blocks],
|
block_ids=[b.id for b in blocks],
|
||||||
flags={
|
flags={
|
||||||
"has_code": any(b.kind == "code" for b in blocks),
|
"has_code": any(b.kind == "code" for b in blocks),
|
||||||
"has_table": any(b.kind == "table" for b in blocks),
|
"has_table": any(
|
||||||
"has_math": any(b.kind == "math" for b in blocks),
|
b.kind == "table"
|
||||||
|
or (b.kind == "html" and "<table" in b.content.lower())
|
||||||
|
for b in blocks
|
||||||
|
),
|
||||||
|
"has_math": any(
|
||||||
|
b.kind == "math"
|
||||||
|
or "$$" in b.content
|
||||||
|
or r"\begin{" in b.content
|
||||||
|
for b in blocks
|
||||||
|
),
|
||||||
"is_overflow": total_chars > config.max_chars,
|
"is_overflow": total_chars > config.max_chars,
|
||||||
"is_sparse": total_chars < config.min_chars,
|
"is_sparse": total_chars < config.min_chars,
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -2,6 +2,12 @@ from __future__ import annotations
|
|||||||
from markdown_it import MarkdownIt
|
from markdown_it import MarkdownIt
|
||||||
from markdown_it.token import Token
|
from markdown_it.token import Token
|
||||||
|
|
||||||
|
try:
|
||||||
|
from mdit_py_plugins.dollarmath import dollarmath_plugin as _dollarmath
|
||||||
|
_HAS_DOLLARMATH = True
|
||||||
|
except ImportError:
|
||||||
|
_HAS_DOLLARMATH = False
|
||||||
|
|
||||||
|
|
||||||
def parse(source: str) -> tuple[list[Token], list[str]]:
|
def parse(source: str) -> tuple[list[Token], list[str]]:
|
||||||
"""Parsa Markdown in token stream con source map.
|
"""Parsa Markdown in token stream con source map.
|
||||||
@@ -11,6 +17,8 @@ def parse(source: str) -> tuple[list[Token], list[str]]:
|
|||||||
lines: righe sorgente (0-indexed) per ricostruzione testo esatto
|
lines: righe sorgente (0-indexed) per ricostruzione testo esatto
|
||||||
"""
|
"""
|
||||||
md = MarkdownIt().enable("table")
|
md = MarkdownIt().enable("table")
|
||||||
|
if _HAS_DOLLARMATH:
|
||||||
|
md = md.use(_dollarmath, allow_labels=False, allow_space=False)
|
||||||
tokens = md.parse(source)
|
tokens = md.parse(source)
|
||||||
lines = source.splitlines()
|
lines = source.splitlines()
|
||||||
return tokens, lines
|
return tokens, lines
|
||||||
|
|||||||
@@ -191,6 +191,25 @@ def segment(tokens: list[Token], lines: list[str], config: ChunkerConfig) -> lis
|
|||||||
i += 1
|
i += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# ── Math block (dollarmath plugin) ───────────────────────────────────
|
||||||
|
if tok.type == "math_block":
|
||||||
|
start = tok.map[0] if tok.map else 0
|
||||||
|
end = tok.map[1] if tok.map else start + 1
|
||||||
|
content = _lines_content(lines, start, end)
|
||||||
|
blocks.append(Block(
|
||||||
|
id=_make_id(),
|
||||||
|
kind="math",
|
||||||
|
content=content,
|
||||||
|
plain_text="[formula]",
|
||||||
|
atomic=True,
|
||||||
|
start_line=start,
|
||||||
|
end_line=end,
|
||||||
|
header_path=header_path,
|
||||||
|
chars=len(content),
|
||||||
|
))
|
||||||
|
i += 1
|
||||||
|
continue
|
||||||
|
|
||||||
# ── HTML block ────────────────────────────────────────────────────────
|
# ── HTML block ────────────────────────────────────────────────────────
|
||||||
if tok.type == "html_block":
|
if tok.type == "html_block":
|
||||||
start = tok.map[0] if tok.map else 0
|
start = tok.map[0] if tok.map else 0
|
||||||
|
|||||||
@@ -116,3 +116,35 @@ def test_assets_empty(cfg):
|
|||||||
blocks = [make_block(1, "Testo.")]
|
blocks = [make_block(1, "Testo.")]
|
||||||
chunks = pack(blocks, cfg, "test")
|
chunks = pack(blocks, cfg, "test")
|
||||||
assert chunks[0].assets == []
|
assert chunks[0].assets == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_flags_has_math_via_kind(cfg):
|
||||||
|
blocks = [make_block(1, "$$\nE=mc^2\n$$", kind="math", atomic=True)]
|
||||||
|
chunks = pack(blocks, cfg, "test")
|
||||||
|
assert chunks[0].flags["has_math"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_flags_has_math_via_content_dollars(cfg):
|
||||||
|
# Paragrafo con $$ inline (senza dollarmath plugin attivo)
|
||||||
|
blocks = [make_block(1, "Vedi la formula:\n$$\nx^2\n$$")]
|
||||||
|
chunks = pack(blocks, cfg, "test")
|
||||||
|
assert chunks[0].flags["has_math"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_flags_has_math_via_content_begin(cfg):
|
||||||
|
blocks = [make_block(1, r"Equazione: \begin{align} x = 1 \end{align}")]
|
||||||
|
chunks = pack(blocks, cfg, "test")
|
||||||
|
assert chunks[0].flags["has_math"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_flags_has_table_via_html(cfg):
|
||||||
|
html_content = "<table><tr><td>A</td></tr></table>"
|
||||||
|
blocks = [make_block(1, html_content, kind="html", atomic=True)]
|
||||||
|
chunks = pack(blocks, cfg, "test")
|
||||||
|
assert chunks[0].flags["has_table"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_flags_no_math_plain_text(cfg):
|
||||||
|
blocks = [make_block(1, "Testo normale senza formule.")]
|
||||||
|
chunks = pack(blocks, cfg, "test")
|
||||||
|
assert chunks[0].flags["has_math"] is False
|
||||||
|
|||||||
@@ -44,3 +44,26 @@ def test_lines_preserves_source():
|
|||||||
_, lines = parse(md)
|
_, lines = parse(md)
|
||||||
assert lines[0] == "Riga 1"
|
assert lines[0] == "Riga 1"
|
||||||
assert lines[1] == "Riga 2"
|
assert lines[1] == "Riga 2"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_math_block():
|
||||||
|
md = "# T\n\n$$\nE=mc^2\n$$\n"
|
||||||
|
tokens, lines = parse(md)
|
||||||
|
types = [t.type for t in tokens]
|
||||||
|
assert "math_block" in types
|
||||||
|
|
||||||
|
|
||||||
|
def test_math_block_has_source_map():
|
||||||
|
md = "# T\n\n$$\nE=mc^2\n$$\n"
|
||||||
|
tokens, lines = parse(md)
|
||||||
|
mb = next(t for t in tokens if t.type == "math_block")
|
||||||
|
assert mb.map is not None
|
||||||
|
assert "E=mc^2" in mb.content
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_math_inline_stays_in_paragraph():
|
||||||
|
md = "Testo con $x^2$ inline.\n"
|
||||||
|
tokens, lines = parse(md)
|
||||||
|
types = [t.type for t in tokens]
|
||||||
|
assert "paragraph_open" in types
|
||||||
|
assert "math_block" not in types
|
||||||
|
|||||||
@@ -126,3 +126,30 @@ def test_source_line_numbers(cfg):
|
|||||||
para = next(b for b in blocks if b.kind == "paragraph")
|
para = next(b for b in blocks if b.kind == "paragraph")
|
||||||
assert para.start_line == 2
|
assert para.start_line == 2
|
||||||
assert para.end_line == 3
|
assert para.end_line == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_math_block_is_atomic(cfg):
|
||||||
|
md = "# T\n\n$$\nE=mc^2\n$$\n"
|
||||||
|
tokens, lines = parse(md)
|
||||||
|
blocks = segment(tokens, lines, cfg)
|
||||||
|
math = next((b for b in blocks if b.kind == "math"), None)
|
||||||
|
assert math is not None
|
||||||
|
assert math.atomic is True
|
||||||
|
assert "E=mc^2" in math.content
|
||||||
|
|
||||||
|
|
||||||
|
def test_math_block_plain_text(cfg):
|
||||||
|
md = "# T\n\n$$\n\\int_0^1 f(x) dx\n$$\n"
|
||||||
|
tokens, lines = parse(md)
|
||||||
|
blocks = segment(tokens, lines, cfg)
|
||||||
|
math = next(b for b in blocks if b.kind == "math")
|
||||||
|
assert math.plain_text == "[formula]"
|
||||||
|
|
||||||
|
|
||||||
|
def test_html_block_with_table(cfg):
|
||||||
|
md = "# T\n\n<table><tr><td>A</td></tr></table>\n"
|
||||||
|
tokens, lines = parse(md)
|
||||||
|
blocks = segment(tokens, lines, cfg)
|
||||||
|
html = next((b for b in blocks if b.kind == "html"), None)
|
||||||
|
assert html is not None
|
||||||
|
assert "<table" in html.content
|
||||||
|
|||||||
Reference in New Issue
Block a user