From ab52d26b99ea8a19cfcdeceb99c0e8ceca34d400 Mon Sep 17 00:00:00 2001 From: Davide Grilli Date: Mon, 8 Jun 2026 15:44:26 +0200 Subject: [PATCH] feat(notebook): download ZIP per tutti gli stem in output/ Co-Authored-By: Claude Sonnet 4.6 --- mineru.ipynb | 85 ++++++++++++++++++++-------------------------------- 1 file changed, 32 insertions(+), 53 deletions(-) diff --git a/mineru.ipynb b/mineru.ipynb index af69925..e7da2a2 100644 --- a/mineru.ipynb +++ b/mineru.ipynb @@ -194,36 +194,41 @@ }, "outputs": [], "source": [ - "import os, zipfile, shutil\n", + "import os, zipfile\n", "\n", - "# Cerca la cartella di output (backend auto o hybrid_auto)\n", - "candidates = [\n", - " f'./output/{STEM}/hybrid_auto',\n", - " f'./output/{STEM}/auto',\n", - " f'./output/{STEM}',\n", - "]\n", - "output_dir = next((p for p in candidates if os.path.isdir(p)), None)\n", + "output_root = './output'\n", + "stems = [d for d in os.listdir(output_root) if os.path.isdir(os.path.join(output_root, d))] if os.path.isdir(output_root) else []\n", "\n", - "if output_dir is None:\n", - " print(f'Cartella output non trovata per \"{STEM}\". Percorsi cercati:')\n", - " for p in candidates:\n", - " print(' ', p)\n", + "if not stems:\n", + " print(f'Nessuna cartella trovata in {output_root}')\n", "else:\n", - " zip_path = f'./{STEM}_output.zip'\n", - " with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:\n", - " for root, dirs, files in os.walk(output_dir):\n", - " for file in files:\n", - " full_path = os.path.join(root, file)\n", - " arcname = os.path.relpath(full_path, start=os.path.dirname(output_dir))\n", - " zf.write(full_path, arcname)\n", - " size_mb = os.path.getsize(zip_path) / (1024 * 1024)\n", - " print(f'ZIP creato: {zip_path} ({size_mb:.1f} MB)')\n", + " for stem in sorted(stems):\n", + " candidates = [\n", + " f'{output_root}/{stem}/hybrid_auto',\n", + " f'{output_root}/{stem}/auto',\n", + " f'{output_root}/{stem}',\n", + " ]\n", + " output_dir = next((p for p in candidates if os.path.isdir(p)), None)\n", "\n", - " try:\n", - " from google.colab import files\n", - " files.download(zip_path)\n", - " except ImportError:\n", - " print('(download automatico disponibile solo su Google Colab)')\n" + " if output_dir is None:\n", + " print(f'[{stem}] cartella output non trovata, saltato')\n", + " continue\n", + "\n", + " zip_path = f'./{stem}_output.zip'\n", + " with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zf:\n", + " for root, dirs, files in os.walk(output_dir):\n", + " for file in files:\n", + " full_path = os.path.join(root, file)\n", + " arcname = os.path.relpath(full_path, start=os.path.dirname(output_dir))\n", + " zf.write(full_path, arcname)\n", + " size_mb = os.path.getsize(zip_path) / (1024 * 1024)\n", + " print(f'[{stem}] ZIP creato: {zip_path} ({size_mb:.1f} MB)')\n", + "\n", + " try:\n", + " from google.colab import files\n", + " files.download(zip_path)\n", + " except ImportError:\n", + " print('(download automatico disponibile solo su Google Colab)')\n" ] }, { @@ -257,32 +262,6 @@ "source": [ "!mineru -p ./ -o ./output -b pipeline" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": "## 3. Chunking\n\nScarica il repo RAG e suddivide il Markdown in chunk pronti per la vettorizzazione." - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": "!git clone https://santantonio.sytes.net/davide/rag-from-scratch.git /content/rag 2>/dev/null || (cd /content/rag && git pull)" - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": "# ── Parametri chunking — modifica qui prima di eseguire ──────────────────────\n\nMAX_CHARS = 1200 # lunghezza massima chunk (caratteri)\nMIN_CHARS = 80 # soglia minima (paragrafi più corti vengono fusi)\nCONTEXT_DEPTH = 3 # livelli heading inclusi nel prefisso (1–3)\nMERGE_SHORT_PARAGRAPHS = True\nSKIP_PRE_HEADING = True # salta contenuto prima del primo heading\nSKIP_HEADINGS = {\n \"indice\",\n \"sommario\",\n \"bibliografia\",\n \"ringraziamenti\",\n \"abbreviazioni\",\n}\n\n# ── Applica al config del repo ────────────────────────────────────────────────\n\nconfig_path = \"/content/rag/chunks/config.py\"\nwith open(config_path) as f:\n src = f.read()\n\nimport re\n\ndef _replace(src, name, value):\n if isinstance(value, bool):\n v = \"True\" if value else \"False\"\n return re.sub(rf\"^{name}\\s*=.*$\", f\"{name}: bool = {v}\", src, flags=re.MULTILINE)\n elif isinstance(value, int):\n return re.sub(rf\"^{name}\\s*=.*$\", f\"{name}: int = {value}\", src, flags=re.MULTILINE)\n elif isinstance(value, set):\n items = \",\\n \".join(f'\"{s}\"' for s in sorted(value))\n block = f\"{name}: set[str] = {{\\n {items},\\n}}\"\n return re.sub(rf\"^{name}.*?^\\}}\", block, src, flags=re.MULTILINE | re.DOTALL)\n return src\n\nfor name, val in [\n (\"MAX_CHARS\", MAX_CHARS),\n (\"MIN_CHARS\", MIN_CHARS),\n (\"CONTEXT_DEPTH\", CONTEXT_DEPTH),\n (\"MERGE_SHORT_PARAGRAPHS\", MERGE_SHORT_PARAGRAPHS),\n (\"SKIP_PRE_HEADING\", SKIP_PRE_HEADING),\n (\"SKIP_HEADINGS\", SKIP_HEADINGS),\n]:\n src = _replace(src, name, val)\n\nwith open(config_path, \"w\") as f:\n f.write(src)\n\nprint(\"Config applicato:\", config_path)\n" - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": "import os, sys, shutil\nsys.path.insert(0, \"/content/rag\")\n\n# Copia l'output MinerU dove il chunker si aspetta di trovarlo\nstem_output_src = f\"./output/{STEM}/auto\"\nstem_output_dst = f\"/content/rag/sources/{STEM}_output/auto\"\nif os.path.isdir(stem_output_src):\n os.makedirs(stem_output_dst, exist_ok=True)\n shutil.copytree(stem_output_src, stem_output_dst, dirs_exist_ok=True)\n\n!python /content/rag/chunks/chunker.py --stem {STEM}\n\n# Scarica chunks.json\nchunks_path = f\"/content/rag/chunks/{STEM}/chunks.json\"\nif os.path.exists(chunks_path):\n try:\n from google.colab import files\n files.download(chunks_path)\n except ImportError:\n print(f\"Chunk pronti in: {chunks_path}\")\nelse:\n print(\"chunks.json non trovato — controlla l'output sopra.\")\n" } ], "metadata": { @@ -301,4 +280,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +}