feat(chunks): aggiunge ChunkerConfig dataclass
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkerConfig:
|
||||
max_chars: int = 1200
|
||||
min_chars: int = 80
|
||||
target_chars: int = 800
|
||||
context_depth: int = 3
|
||||
skip_headings: set[str] = field(default_factory=lambda: {
|
||||
"indice",
|
||||
"sommario",
|
||||
"bibliografia",
|
||||
"ringraziamenti",
|
||||
"abbreviazioni",
|
||||
})
|
||||
skip_pre_heading: bool = True
|
||||
merge_short: bool = True
|
||||
atomic_types: set[str] = field(default_factory=lambda: {
|
||||
"table", "code", "list", "math", "html",
|
||||
})
|
||||
fail_on_broken_fence: bool = True
|
||||
fail_on_content_loss: bool = False
|
||||
@@ -0,0 +1,29 @@
|
||||
from chunks.config import ChunkerConfig
|
||||
|
||||
|
||||
def test_default_values():
|
||||
c = ChunkerConfig()
|
||||
assert c.max_chars == 1200
|
||||
assert c.min_chars == 80
|
||||
assert c.target_chars == 800
|
||||
assert c.context_depth == 3
|
||||
assert "indice" in c.skip_headings
|
||||
assert c.skip_pre_heading is True
|
||||
assert c.merge_short is True
|
||||
assert "table" in c.atomic_types
|
||||
assert "code" in c.atomic_types
|
||||
|
||||
|
||||
def test_custom_values():
|
||||
c = ChunkerConfig(max_chars=2000, min_chars=100, context_depth=2)
|
||||
assert c.max_chars == 2000
|
||||
assert c.min_chars == 100
|
||||
assert c.context_depth == 2
|
||||
|
||||
|
||||
def test_skip_headings_case_insensitive_check():
|
||||
c = ChunkerConfig()
|
||||
skip_lower = {h.lower() for h in c.skip_headings}
|
||||
assert "indice" in skip_lower
|
||||
assert "sommario" in skip_lower
|
||||
assert "bibliografia" in skip_lower
|
||||
Reference in New Issue
Block a user