davide/rag-from-scratch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(notebook): download ZIP per tutti gli stem in output/

Browse Source 
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Davide Grilli
2026-06-08 15:44:26 +02:00
parent 019a484854
commit ab52d26b99
1 changed files with 32 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mineru.ipynb
+16 -37
View File
@@ -194,22 +194,27 @@
},
"outputs": [],
"source": [
"import os, zipfile, shutil\n",
"import os, zipfile\n",
"\n",
"# Cerca la cartella di output (backend auto o hybrid_auto)\n",
"output_root = './output'\n",
"stems = [d for d in os.listdir(output_root) if os.path.isdir(os.path.join(output_root, d))] if os.path.isdir(output_root) else []\n",
"\n",
"if not stems:\n",
" print(f'Nessuna cartella trovata in {output_root}')\n",
"else:\n",
" for stem in sorted(stems):\n",
" candidates = [\n",
" f'./output/{STEM}/hybrid_auto',\n",
" f'./output/{STEM}/auto',\n",
" f'./output/{STEM}',\n",
" f'{output_root}/{stem}/hybrid_auto',\n",
" f'{output_root}/{stem}/auto',\n",
" f'{output_root}/{stem}',\n",
" ]\n",
" output_dir = next((p for p in candidates if os.path.isdir(p)), None)\n",
"\n",
" if output_dir is None:\n",
" print(f'Cartella output non trovata per \"{STEM}\". Percorsi cercati:')\n",
" for p in candidates:\n",
" print(' ', p)\n",
"else:\n",
" zip_path = f'./{STEM}_output.zip'\n",
" print(f'[{stem}] cartella output non trovata, saltato')\n",
" continue\n",
"\n",
" zip_path = f'./{stem}_output.zip'\n",
" with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:\n",
" for root, dirs, files in os.walk(output_dir):\n",
" for file in files:\n",
@@ -217,7 +222,7 @@
" arcname = os.path.relpath(full_path, start=os.path.dirname(output_dir))\n",
" zf.write(full_path, arcname)\n",
" size_mb = os.path.getsize(zip_path) / (1024 * 1024)\n",
" print(f'ZIP creato: {zip_path} ({size_mb:.1f} MB)')\n",
" print(f'[{stem}] ZIP creato: {zip_path} ({size_mb:.1f} MB)')\n",
"\n",
" try:\n",
" from google.colab import files\n",
@@ -257,32 +262,6 @@
"source": [
"!mineru -p ./ -o ./output -b pipeline"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": "## 3. Chunking\n\nScarica il repo RAG e suddivide il Markdown in chunk pronti per la vettorizzazione."
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": "!git clone https://santantonio.sytes.net/davide/rag-from-scratch.git /content/rag 2>/dev/null || (cd /content/rag && git pull)"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": "# ── Parametri chunking — modifica qui prima di eseguire ──────────────────────\n\nMAX_CHARS = 1200 # lunghezza massima chunk (caratteri)\nMIN_CHARS = 80 # soglia minima (paragrafi più corti vengono fusi)\nCONTEXT_DEPTH = 3 # livelli heading inclusi nel prefisso (1–3)\nMERGE_SHORT_PARAGRAPHS = True\nSKIP_PRE_HEADING = True # salta contenuto prima del primo heading\nSKIP_HEADINGS = {\n \"indice\",\n \"sommario\",\n \"bibliografia\",\n \"ringraziamenti\",\n \"abbreviazioni\",\n}\n\n# ── Applica al config del repo ────────────────────────────────────────────────\n\nconfig_path = \"/content/rag/chunks/config.py\"\nwith open(config_path) as f:\n src = f.read()\n\nimport re\n\ndef _replace(src, name, value):\n if isinstance(value, bool):\n v = \"True\" if value else \"False\"\n return re.sub(rf\"^{name}\\s*=.*$\", f\"{name}: bool = {v}\", src, flags=re.MULTILINE)\n elif isinstance(value, int):\n return re.sub(rf\"^{name}\\s*=.*$\", f\"{name}: int = {value}\", src, flags=re.MULTILINE)\n elif isinstance(value, set):\n items = \",\\n \".join(f'\"{s}\"' for s in sorted(value))\n block = f\"{name}: set[str] = {{\\n {items},\\n}}\"\n return re.sub(rf\"^{name}.*?^\\}}\", block, src, flags=re.MULTILINE | re.DOTALL)\n return src\n\nfor name, val in [\n (\"MAX_CHARS\", MAX_CHARS),\n (\"MIN_CHARS\", MIN_CHARS),\n (\"CONTEXT_DEPTH\", CONTEXT_DEPTH),\n (\"MERGE_SHORT_PARAGRAPHS\", MERGE_SHORT_PARAGRAPHS),\n (\"SKIP_PRE_HEADING\", SKIP_PRE_HEADING),\n (\"SKIP_HEADINGS\", SKIP_HEADINGS),\n]:\n src = _replace(src, name, val)\n\nwith open(config_path, \"w\") as f:\n f.write(src)\n\nprint(\"Config applicato:\", config_path)\n"
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": "import os, sys, shutil\nsys.path.insert(0, \"/content/rag\")\n\n# Copia l'output MinerU dove il chunker si aspetta di trovarlo\nstem_output_src = f\"./output/{STEM}/auto\"\nstem_output_dst = f\"/content/rag/sources/{STEM}_output/auto\"\nif os.path.isdir(stem_output_src):\n os.makedirs(stem_output_dst, exist_ok=True)\n shutil.copytree(stem_output_src, stem_output_dst, dirs_exist_ok=True)\n\n!python /content/rag/chunks/chunker.py --stem {STEM}\n\n# Scarica chunks.json\nchunks_path = f\"/content/rag/chunks/{STEM}/chunks.json\"\nif os.path.exists(chunks_path):\n try:\n from google.colab import files\n files.download(chunks_path)\n except ImportError:\n print(f\"Chunk pronti in: {chunks_path}\")\nelse:\n print(\"chunks.json non trovato — controlla l'output sopra.\")\n"
}
],
"metadata": {