feat(chunks): sentence-boundary flush, math incomplete detection, structure profile export
- chunker: estrai _flush_chunk() con estensione al confine di frase (max 120%) - verify: rileva chunk matematici incompleti come warning, gestisci hash hex e URL - conversione: esporta structure_profile.json nell'output dir
This commit is contained in:
+39
-10
@@ -44,6 +44,41 @@ def slugify(s: str, max_len: int = 60) -> str:
|
||||
return s[:max_len] if s else "section"
|
||||
|
||||
|
||||
_SENT_BOUNDARY = re.compile(r"[.!?»)\]'\u2019\"\u201c\u201d/:|\u2026]$")
|
||||
|
||||
|
||||
def _flush_chunk(
|
||||
current: list[str],
|
||||
sentences: list[str],
|
||||
i: int,
|
||||
prefix: str,
|
||||
sezione: str,
|
||||
titolo: str,
|
||||
sub_index: int,
|
||||
max_chars: int,
|
||||
) -> tuple[dict, list[str], int, int]:
|
||||
"""Emette un chunk, estendendo fino a un confine di frase (max +20%)."""
|
||||
hard_limit = int(max_chars * 1.2)
|
||||
current_len = sum(len(s) + 1 for s in current)
|
||||
while i < len(sentences) and not _SENT_BOUNDARY.search(" ".join(current)):
|
||||
nxt = sentences[i]
|
||||
if current_len + len(nxt) + 1 > hard_limit:
|
||||
break
|
||||
current.append(nxt)
|
||||
current_len += len(nxt) + 1
|
||||
i += 1
|
||||
chunk_text = prefix + " ".join(current)
|
||||
chunk = {
|
||||
"chunk_id": f"{slugify(sezione)}__{slugify(titolo)}__s{sub_index}",
|
||||
"text": chunk_text,
|
||||
"sezione": sezione,
|
||||
"titolo": titolo,
|
||||
"sub_index": sub_index,
|
||||
"n_chars": len(chunk_text),
|
||||
}
|
||||
return chunk, current, i, sub_index + 1
|
||||
|
||||
|
||||
def make_sub_chunks(
|
||||
body: str,
|
||||
prefix: str,
|
||||
@@ -69,16 +104,10 @@ def make_sub_chunks(
|
||||
current_len += len(sent) + (1 if len(current) > 1 else 0)
|
||||
i += 1
|
||||
else:
|
||||
chunk_text = prefix + " ".join(current)
|
||||
chunks.append({
|
||||
"chunk_id": f"{slugify(sezione)}__{slugify(titolo)}__s{sub_index}",
|
||||
"text": chunk_text,
|
||||
"sezione": sezione,
|
||||
"titolo": titolo,
|
||||
"sub_index": sub_index,
|
||||
"n_chars": len(chunk_text),
|
||||
})
|
||||
sub_index += 1
|
||||
chunk, current, i, sub_index = _flush_chunk(
|
||||
current, sentences, i, prefix, sezione, titolo, sub_index, max_chars
|
||||
)
|
||||
chunks.append(chunk)
|
||||
overlap = current[-overlap_s:] if overlap_s and len(current) > overlap_s else []
|
||||
current = overlap[:]
|
||||
current_len = sum(len(s) + 1 for s in current)
|
||||
|
||||
Reference in New Issue
Block a user