feat(chunks): aggiunge ChunkerConfig dataclass
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,24 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ChunkerConfig:
|
||||||
|
max_chars: int = 1200
|
||||||
|
min_chars: int = 80
|
||||||
|
target_chars: int = 800
|
||||||
|
context_depth: int = 3
|
||||||
|
skip_headings: set[str] = field(default_factory=lambda: {
|
||||||
|
"indice",
|
||||||
|
"sommario",
|
||||||
|
"bibliografia",
|
||||||
|
"ringraziamenti",
|
||||||
|
"abbreviazioni",
|
||||||
|
})
|
||||||
|
skip_pre_heading: bool = True
|
||||||
|
merge_short: bool = True
|
||||||
|
atomic_types: set[str] = field(default_factory=lambda: {
|
||||||
|
"table", "code", "list", "math", "html",
|
||||||
|
})
|
||||||
|
fail_on_broken_fence: bool = True
|
||||||
|
fail_on_content_loss: bool = False
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
from chunks.config import ChunkerConfig
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_values():
|
||||||
|
c = ChunkerConfig()
|
||||||
|
assert c.max_chars == 1200
|
||||||
|
assert c.min_chars == 80
|
||||||
|
assert c.target_chars == 800
|
||||||
|
assert c.context_depth == 3
|
||||||
|
assert "indice" in c.skip_headings
|
||||||
|
assert c.skip_pre_heading is True
|
||||||
|
assert c.merge_short is True
|
||||||
|
assert "table" in c.atomic_types
|
||||||
|
assert "code" in c.atomic_types
|
||||||
|
|
||||||
|
|
||||||
|
def test_custom_values():
|
||||||
|
c = ChunkerConfig(max_chars=2000, min_chars=100, context_depth=2)
|
||||||
|
assert c.max_chars == 2000
|
||||||
|
assert c.min_chars == 100
|
||||||
|
assert c.context_depth == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_skip_headings_case_insensitive_check():
|
||||||
|
c = ChunkerConfig()
|
||||||
|
skip_lower = {h.lower() for h in c.skip_headings}
|
||||||
|
assert "indice" in skip_lower
|
||||||
|
assert "sommario" in skip_lower
|
||||||
|
assert "bibliografia" in skip_lower
|
||||||
Reference in New Issue
Block a user