feat(chunks): rilevamento universale has_math/has_table + dollarmath plugin
- parser.py: integra dollarmath_plugin (graceful fallback) — blocchi $$…$$ → token math_block atomico
- segmenter.py: gestisce math_block → Block(kind=math, atomic=True, plain_text=[formula])
- packer.py: has_table rileva anche <table> in blocchi html; has_math rileva $$ e \begin{ nel contenuto
- +11 test (parser×3, segmenter×3, packer×5) — totale 65 test
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+12
-3
@@ -41,9 +41,18 @@ def _build_chunk(blocks: list[Block], index: int, config: ChunkerConfig,
|
||||
header_path=header_path,
|
||||
block_ids=[b.id for b in blocks],
|
||||
flags={
|
||||
"has_code": any(b.kind == "code" for b in blocks),
|
||||
"has_table": any(b.kind == "table" for b in blocks),
|
||||
"has_math": any(b.kind == "math" for b in blocks),
|
||||
"has_code": any(b.kind == "code" for b in blocks),
|
||||
"has_table": any(
|
||||
b.kind == "table"
|
||||
or (b.kind == "html" and "<table" in b.content.lower())
|
||||
for b in blocks
|
||||
),
|
||||
"has_math": any(
|
||||
b.kind == "math"
|
||||
or "$$" in b.content
|
||||
or r"\begin{" in b.content
|
||||
for b in blocks
|
||||
),
|
||||
"is_overflow": total_chars > config.max_chars,
|
||||
"is_sparse": total_chars < config.min_chars,
|
||||
},
|
||||
|
||||
@@ -2,6 +2,12 @@ from __future__ import annotations
|
||||
from markdown_it import MarkdownIt
|
||||
from markdown_it.token import Token
|
||||
|
||||
try:
|
||||
from mdit_py_plugins.dollarmath import dollarmath_plugin as _dollarmath
|
||||
_HAS_DOLLARMATH = True
|
||||
except ImportError:
|
||||
_HAS_DOLLARMATH = False
|
||||
|
||||
|
||||
def parse(source: str) -> tuple[list[Token], list[str]]:
|
||||
"""Parsa Markdown in token stream con source map.
|
||||
@@ -11,6 +17,8 @@ def parse(source: str) -> tuple[list[Token], list[str]]:
|
||||
lines: righe sorgente (0-indexed) per ricostruzione testo esatto
|
||||
"""
|
||||
md = MarkdownIt().enable("table")
|
||||
if _HAS_DOLLARMATH:
|
||||
md = md.use(_dollarmath, allow_labels=False, allow_space=False)
|
||||
tokens = md.parse(source)
|
||||
lines = source.splitlines()
|
||||
return tokens, lines
|
||||
|
||||
@@ -191,6 +191,25 @@ def segment(tokens: list[Token], lines: list[str], config: ChunkerConfig) -> lis
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# ── Math block (dollarmath plugin) ───────────────────────────────────
|
||||
if tok.type == "math_block":
|
||||
start = tok.map[0] if tok.map else 0
|
||||
end = tok.map[1] if tok.map else start + 1
|
||||
content = _lines_content(lines, start, end)
|
||||
blocks.append(Block(
|
||||
id=_make_id(),
|
||||
kind="math",
|
||||
content=content,
|
||||
plain_text="[formula]",
|
||||
atomic=True,
|
||||
start_line=start,
|
||||
end_line=end,
|
||||
header_path=header_path,
|
||||
chars=len(content),
|
||||
))
|
||||
i += 1
|
||||
continue
|
||||
|
||||
# ── HTML block ────────────────────────────────────────────────────────
|
||||
if tok.type == "html_block":
|
||||
start = tok.map[0] if tok.map else 0
|
||||
|
||||
Reference in New Issue
Block a user