diff --git a/chunks/__init__.py b/chunks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chunks/models.py b/chunks/models.py new file mode 100644 index 0000000..c7a99b3 --- /dev/null +++ b/chunks/models.py @@ -0,0 +1,47 @@ +from __future__ import annotations +from dataclasses import dataclass, field + + +@dataclass +class Block: + id: str + kind: str # paragraph|heading|table|code|list|math|blockquote|html|thematic_break + content: str + plain_text: str + atomic: bool + start_line: int + end_line: int + header_path: list[dict] + chars: int + + +@dataclass +class Chunk: + chunk_id: str + chunk_index: int + content_original: str + content_for_embedding: str + content_type: str # section_fragment | atomic_block | overflow + chars: int + start_line: int + end_line: int + header_path: list[dict] + block_ids: list[str] + flags: dict + neighbors: dict + assets: list = field(default_factory=list) + + +@dataclass +class Diagnostics: + errors: list[str] + warnings: list[str] + metrics: dict + + +@dataclass +class ChunkingResult: + stem: str + source_path: str + chunks: list[Chunk] + diagnostics: Diagnostics diff --git a/tests/chunks/test_models.py b/tests/chunks/test_models.py new file mode 100644 index 0000000..72ec406 --- /dev/null +++ b/tests/chunks/test_models.py @@ -0,0 +1,51 @@ +from chunks.models import Block, Chunk, Diagnostics, ChunkingResult + + +def test_block_creation(): + b = Block( + id="blk_0001", + kind="paragraph", + content="Testo di esempio.", + plain_text="Testo di esempio.", + atomic=False, + start_line=0, + end_line=1, + header_path=[{"level": 1, "text": "Titolo"}], + chars=17, + ) + assert b.id == "blk_0001" + assert b.kind == "paragraph" + assert not b.atomic + assert b.chars == 17 + + +def test_chunk_creation(): + c = Chunk( + chunk_id="chk_000001", + chunk_index=1, + content_original="Testo.", + content_for_embedding="Titolo\n\nTesto.", + content_type="section_fragment", + chars=6, + start_line=0, + end_line=1, + header_path=[{"level": 1, "text": "Titolo"}], + block_ids=["blk_0001"], + flags={"has_code": False, "has_table": False, "has_math": False, + "is_overflow": False, "is_sparse": False}, + neighbors={"previous_chunk_id": None, "next_chunk_id": None}, + assets=[], + ) + assert c.chunk_id == "chk_000001" + assert c.assets == [] + + +def test_diagnostics_empty(): + d = Diagnostics(errors=[], warnings=[], metrics={}) + assert d.errors == [] + + +def test_chunking_result(): + r = ChunkingResult(stem="doc", source_path="sources/doc.md", chunks=[], diagnostics=Diagnostics([], [], {})) + assert r.stem == "doc" + assert r.chunks == []