feat(chunks): sentence-boundary flush, math incomplete detection, structure profile export

- chunker: estrai _flush_chunk() con estensione al confine di frase (max 120%)
- verify: rileva chunk matematici incompleti come warning, gestisci hash hex e URL
- conversione: esporta structure_profile.json nell'output dir
This commit is contained in:
2026-04-20 12:27:58 +02:00
parent 240cad9830
commit f340c4c14e
3 changed files with 84 additions and 20 deletions
+39 -10
View File
@@ -44,6 +44,41 @@ def slugify(s: str, max_len: int = 60) -> str:
return s[:max_len] if s else "section"
_SENT_BOUNDARY = re.compile(r"[.!?»)\]'\u2019\"\u201c\u201d/:|\u2026]$")
def _flush_chunk(
current: list[str],
sentences: list[str],
i: int,
prefix: str,
sezione: str,
titolo: str,
sub_index: int,
max_chars: int,
) -> tuple[dict, list[str], int, int]:
"""Emette un chunk, estendendo fino a un confine di frase (max +20%)."""
hard_limit = int(max_chars * 1.2)
current_len = sum(len(s) + 1 for s in current)
while i < len(sentences) and not _SENT_BOUNDARY.search(" ".join(current)):
nxt = sentences[i]
if current_len + len(nxt) + 1 > hard_limit:
break
current.append(nxt)
current_len += len(nxt) + 1
i += 1
chunk_text = prefix + " ".join(current)
chunk = {
"chunk_id": f"{slugify(sezione)}__{slugify(titolo)}__s{sub_index}",
"text": chunk_text,
"sezione": sezione,
"titolo": titolo,
"sub_index": sub_index,
"n_chars": len(chunk_text),
}
return chunk, current, i, sub_index + 1
def make_sub_chunks(
body: str,
prefix: str,
@@ -69,16 +104,10 @@ def make_sub_chunks(
current_len += len(sent) + (1 if len(current) > 1 else 0)
i += 1
else:
chunk_text = prefix + " ".join(current)
chunks.append({
"chunk_id": f"{slugify(sezione)}__{slugify(titolo)}__s{sub_index}",
"text": chunk_text,
"sezione": sezione,
"titolo": titolo,
"sub_index": sub_index,
"n_chars": len(chunk_text),
})
sub_index += 1
chunk, current, i, sub_index = _flush_chunk(
current, sentences, i, prefix, sezione, titolo, sub_index, max_chars
)
chunks.append(chunk)
overlap = current[-overlap_s:] if overlap_s and len(current) > overlap_s else []
current = overlap[:]
current_len = sum(len(s) + 1 for s in current)