feat(chunks): aggiunge models.py con Block, Chunk, Diagnostics, ChunkingResult
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
from __future__ import annotations
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
|
||||
@dataclass
|
||||
class Block:
|
||||
id: str
|
||||
kind: str # paragraph|heading|table|code|list|math|blockquote|html|thematic_break
|
||||
content: str
|
||||
plain_text: str
|
||||
atomic: bool
|
||||
start_line: int
|
||||
end_line: int
|
||||
header_path: list[dict]
|
||||
chars: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class Chunk:
|
||||
chunk_id: str
|
||||
chunk_index: int
|
||||
content_original: str
|
||||
content_for_embedding: str
|
||||
content_type: str # section_fragment | atomic_block | overflow
|
||||
chars: int
|
||||
start_line: int
|
||||
end_line: int
|
||||
header_path: list[dict]
|
||||
block_ids: list[str]
|
||||
flags: dict
|
||||
neighbors: dict
|
||||
assets: list = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Diagnostics:
|
||||
errors: list[str]
|
||||
warnings: list[str]
|
||||
metrics: dict
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkingResult:
|
||||
stem: str
|
||||
source_path: str
|
||||
chunks: list[Chunk]
|
||||
diagnostics: Diagnostics
|
||||
@@ -0,0 +1,51 @@
|
||||
from chunks.models import Block, Chunk, Diagnostics, ChunkingResult
|
||||
|
||||
|
||||
def test_block_creation():
|
||||
b = Block(
|
||||
id="blk_0001",
|
||||
kind="paragraph",
|
||||
content="Testo di esempio.",
|
||||
plain_text="Testo di esempio.",
|
||||
atomic=False,
|
||||
start_line=0,
|
||||
end_line=1,
|
||||
header_path=[{"level": 1, "text": "Titolo"}],
|
||||
chars=17,
|
||||
)
|
||||
assert b.id == "blk_0001"
|
||||
assert b.kind == "paragraph"
|
||||
assert not b.atomic
|
||||
assert b.chars == 17
|
||||
|
||||
|
||||
def test_chunk_creation():
|
||||
c = Chunk(
|
||||
chunk_id="chk_000001",
|
||||
chunk_index=1,
|
||||
content_original="Testo.",
|
||||
content_for_embedding="Titolo\n\nTesto.",
|
||||
content_type="section_fragment",
|
||||
chars=6,
|
||||
start_line=0,
|
||||
end_line=1,
|
||||
header_path=[{"level": 1, "text": "Titolo"}],
|
||||
block_ids=["blk_0001"],
|
||||
flags={"has_code": False, "has_table": False, "has_math": False,
|
||||
"is_overflow": False, "is_sparse": False},
|
||||
neighbors={"previous_chunk_id": None, "next_chunk_id": None},
|
||||
assets=[],
|
||||
)
|
||||
assert c.chunk_id == "chk_000001"
|
||||
assert c.assets == []
|
||||
|
||||
|
||||
def test_diagnostics_empty():
|
||||
d = Diagnostics(errors=[], warnings=[], metrics={})
|
||||
assert d.errors == []
|
||||
|
||||
|
||||
def test_chunking_result():
|
||||
r = ChunkingResult(stem="doc", source_path="sources/doc.md", chunks=[], diagnostics=Diagnostics([], [], {}))
|
||||
assert r.stem == "doc"
|
||||
assert r.chunks == []
|
||||
Reference in New Issue
Block a user