feat(chunks): pipeline unificata Stage 1+2 con md_optimizer
chunker.py ora esegue in sequenza:
- Stage 1 (md_optimizer.py): _content_list_v2.json + _model.json → _clean.md
con pulizia TOC, frontespizio, sommari interni, merge titoli capitolo
- Stage 2: _clean.md → chunks.json (paragraph-overlap, atomici tabelle/liste)
config.py esteso con CHAPTER_PREFIX_PATTERNS, SOMMARIO_PATTERNS,
MODEL_SKIP_LABELS, MODEL_ABSTRACT_LABELS, MIN_CONTENT_CHARS.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -34,7 +34,7 @@ if str(_HERE) not in sys.path:
|
||||
import config as cfg
|
||||
from verify_chunks import verify_stem as _verify_stem
|
||||
|
||||
MAX_CHARS = int(cfg.TARGET_CHARS * (1 + cfg.CHUNK_TOLERANCE))
|
||||
MAX_CHARS = cfg.MAX_CHARS
|
||||
|
||||
|
||||
def _load_thresholds(stem_dir: Path) -> int:
|
||||
@@ -314,8 +314,8 @@ def fix_stem(stem: str, project_root: Path, max_chars: int, dry_run: bool,
|
||||
# Fase 1: risolvi blockers a convergenza (solo merge incomplete)
|
||||
chunks = _fix_blockers(chunks, report)
|
||||
|
||||
_min = int(cfg.TARGET_CHARS * (1 - cfg.CHUNK_TOLERANCE))
|
||||
_max = int(cfg.TARGET_CHARS * (1 + cfg.CHUNK_TOLERANCE))
|
||||
_min = cfg.MIN_CHARS
|
||||
_max = cfg.MAX_CHARS
|
||||
prev_blockers = sum(len(v) for v in report.get("blockers", {}).values())
|
||||
|
||||
for iteration in range(1, max_iter + 1):
|
||||
@@ -378,7 +378,7 @@ if __name__ == "__main__":
|
||||
|
||||
parser = argparse.ArgumentParser(description="Fix chunk")
|
||||
parser.add_argument("--stem", required=True, help="Nome del documento (sottocartella di chunks/)")
|
||||
_max_def = int(cfg.TARGET_CHARS * (1 + cfg.CHUNK_TOLERANCE))
|
||||
_max_def = cfg.MAX_CHARS
|
||||
parser.add_argument(
|
||||
"--max", type=int, default=_max_def,
|
||||
help=f"Soglia massima caratteri per lo split (default: TARGET×(1+TOL) = {_max_def})"
|
||||
|
||||
Reference in New Issue
Block a user