48 lines
934 B
Python
48 lines
934 B
Python
|
from __future__ import annotations
|
||
|
|
from dataclasses import dataclass, field
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class Block:
|
||
|
|
id: str
|
||
|
|
kind: str # paragraph|heading|table|code|list|math|blockquote|html|thematic_break
|
||
|
|
content: str
|
||
|
|
plain_text: str
|
||
|
|
atomic: bool
|
||
|
|
start_line: int
|
||
|
|
end_line: int
|
||
|
|
header_path: list[dict]
|
||
|
|
chars: int
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class Chunk:
|
||
|
|
chunk_id: str
|
||
|
|
chunk_index: int
|
||
|
|
content_original: str
|
||
|
|
content_for_embedding: str
|
||
|
|
content_type: str # section_fragment | atomic_block | overflow
|
||
|
|
chars: int
|
||
|
|
start_line: int
|
||
|
|
end_line: int
|
||
|
|
header_path: list[dict]
|
||
|
|
block_ids: list[str]
|
||
|
|
flags: dict
|
||
|
|
neighbors: dict
|
||
|
|
assets: list = field(default_factory=list)
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class Diagnostics:
|
||
|
|
errors: list[str]
|
||
|
|
warnings: list[str]
|
||
|
|
metrics: dict
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass
|
||
|
|
class ChunkingResult:
|
||
|
|
stem: str
|
||
|
|
source_path: str
|
||
|
|
chunks: list[Chunk]
|
||
|
|
diagnostics: Diagnostics
|