feat(chunks): pipeline unificata Stage 1+2 con md_optimizer

chunker.py ora esegue in sequenza:
  - Stage 1 (md_optimizer.py): _content_list_v2.json + _model.json → _clean.md
    con pulizia TOC, frontespizio, sommari interni, merge titoli capitolo
  - Stage 2: _clean.md → chunks.json (paragraph-overlap, atomici tabelle/liste)

config.py esteso con CHAPTER_PREFIX_PATTERNS, SOMMARIO_PATTERNS,
MODEL_SKIP_LABELS, MODEL_ABSTRACT_LABELS, MIN_CONTENT_CHARS.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-05-20 16:07:40 +02:00
parent 52e881d09c
commit e96c510e1f
4 changed files with 855 additions and 480 deletions
+4 -4
View File
@@ -34,7 +34,7 @@ if str(_HERE) not in sys.path:
import config as cfg
from verify_chunks import verify_stem as _verify_stem
MAX_CHARS = int(cfg.TARGET_CHARS * (1 + cfg.CHUNK_TOLERANCE))
MAX_CHARS = cfg.MAX_CHARS
def _load_thresholds(stem_dir: Path) -> int:
@@ -314,8 +314,8 @@ def fix_stem(stem: str, project_root: Path, max_chars: int, dry_run: bool,
# Fase 1: risolvi blockers a convergenza (solo merge incomplete)
chunks = _fix_blockers(chunks, report)
_min = int(cfg.TARGET_CHARS * (1 - cfg.CHUNK_TOLERANCE))
_max = int(cfg.TARGET_CHARS * (1 + cfg.CHUNK_TOLERANCE))
_min = cfg.MIN_CHARS
_max = cfg.MAX_CHARS
prev_blockers = sum(len(v) for v in report.get("blockers", {}).values())
for iteration in range(1, max_iter + 1):
@@ -378,7 +378,7 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Fix chunk")
parser.add_argument("--stem", required=True, help="Nome del documento (sottocartella di chunks/)")
_max_def = int(cfg.TARGET_CHARS * (1 + cfg.CHUNK_TOLERANCE))
_max_def = cfg.MAX_CHARS
parser.add_argument(
"--max", type=int, default=_max_def,
help=f"Soglia massima caratteri per lo split (default: TARGET×(1+TOL) = {_max_def})"