Docs: move legacy FastAPI notes out of CLAUDE.md into docs/legacy-fastapi-notes.md
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
58d1a38ac0
commit
3d1e820564
9 files changed
+525
-488
No files matched your search
@@ -0,0 +1,115 @@
|
||||
# Step 9 - manifest, cost tracker, cleanup, final report
|
||||
|
||||
Moved out of SKILL.md; read it when the pipeline reaches Step 9.
|
||||
|
||||
### Step 9 - Save manifest, update cost tracker, clean up, and report
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
from graphify.detect import save_manifest
|
||||
|
||||
# Save manifest for --update
|
||||
detect = json.loads(Path('graphify-out/.graphify_detect.json').read_text(encoding=\"utf-8\"))
|
||||
extract = json.loads(Path('graphify-out/.graphify_extract.json').read_text(encoding=\"utf-8\"))
|
||||
INPUT_PATH = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip()
|
||||
# In --update mode, 'all_files' carries the full corpus; 'files' is the changed
|
||||
# subset. Full-rebuild mode populates only 'files', so the fallback handles that.
|
||||
# root= relativizes the manifest keys to the scan root (same base as the build),
|
||||
# so the on-disk manifest is portable across clones/machines and a later --update
|
||||
# matches cached files instead of missing every one (#1417).
|
||||
#
|
||||
# Only stamp semantic files (docs/papers/images) that ACTUALLY produced output:
|
||||
# a detected file whose chunk failed or was omitted must stay unstamped so the
|
||||
# next --update re-queues it, otherwise it is marked done and its content is lost
|
||||
# forever (#2015). This mirrors the library extract path exactly
|
||||
# (cli._stamped_manifest_files + clear_semantic + scan_corpus); do not stamp the
|
||||
# raw corpus. Code files are always stamped (AST is deterministic); only semantic
|
||||
# types are gated on output.
|
||||
from graphify.cli import _stamped_manifest_files
|
||||
_corpus = detect.get('all_files') or detect['files']
|
||||
# A file a subagent only partly read DID produce output, so the stamp filter alone would
|
||||
# keep it - but its node set covers just the head of the file, and stamping it means
|
||||
# detect_incremental reads it as done and never re-queues it, leaving the truncated graph
|
||||
# live forever. Name those files so they stay unstamped and are retried next run (#2225).
|
||||
_pf = Path('graphify-out/.graphify_partial.txt')
|
||||
_partial = {l.strip() for l in _pf.read_text(encoding=\"utf-8\").splitlines() if l.strip()} if _pf.exists() else set()
|
||||
_manifest_files = _stamped_manifest_files(_corpus, extract, Path(INPUT_PATH), partial_source_files=_partial)
|
||||
# Files dispatched this run (the changed subset) but NOT stamped above still carry
|
||||
# a stale semantic_hash from a prior run; clear it so detect_incremental re-queues
|
||||
# them instead of reading them as unchanged (#1948).
|
||||
_sem_types = ('document', 'paper', 'image')
|
||||
_dispatched = {f for t, fl in detect['files'].items() if t in _sem_types for f in fl}
|
||||
_stamped = {f for fl in _manifest_files.values() for f in fl}
|
||||
_cleared = _dispatched - _stamped
|
||||
# scan_corpus = the RAW full corpus (not the stamp-filtered subset) so in-root
|
||||
# files newly excluded since last run are dropped rather than masquerading as
|
||||
# deletions; untouched files' prior rows are still preserved (#1908).
|
||||
_scan = {f for fl in _corpus.values() for f in fl}
|
||||
save_manifest(_manifest_files, root=INPUT_PATH, scan_corpus=_scan, clear_semantic=_cleared or None)
|
||||
|
||||
# Update cumulative cost tracker
|
||||
input_tok = extract.get('input_tokens', 0)
|
||||
output_tok = extract.get('output_tokens', 0)
|
||||
|
||||
cost_path = Path('graphify-out/cost.json')
|
||||
if cost_path.exists():
|
||||
cost = json.loads(cost_path.read_text(encoding=\"utf-8\"))
|
||||
else:
|
||||
cost = {'runs': [], 'total_input_tokens': 0, 'total_output_tokens': 0}
|
||||
|
||||
cost['runs'].append({
|
||||
'date': datetime.now(timezone.utc).isoformat(),
|
||||
'input_tokens': input_tok,
|
||||
'output_tokens': output_tok,
|
||||
'files': detect.get('total_files', 0),
|
||||
})
|
||||
cost['total_input_tokens'] += input_tok
|
||||
cost['total_output_tokens'] += output_tok
|
||||
cost_path.write_text(json.dumps(cost, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
|
||||
print(f'This run: {input_tok:,} input tokens, {output_tok:,} output tokens')
|
||||
print(f'All time: {cost[\"total_input_tokens\"]:,} input, {cost[\"total_output_tokens\"]:,} output ({len(cost[\"runs\"])} runs)')
|
||||
"
|
||||
rm -f graphify-out/.graphify_detect.json graphify-out/.graphify_extract.json graphify-out/.graphify_ast.json graphify-out/.graphify_semantic.json graphify-out/.graphify_analysis.json graphify-out/.graphify_partial.txt
|
||||
find graphify-out -maxdepth 1 -name '.graphify_chunk_*.json' -delete 2>/dev/null
|
||||
rm -f graphify-out/.needs_update 2>/dev/null || true
|
||||
```
|
||||
|
||||
INPUT_PATH is read from `graphify-out/.graphify_root` (the scan root Steps 4-5 used), so the manifest is relativized to the scan root.
|
||||
|
||||
Tell the user (omit the obsidian line unless --obsidian was given):
|
||||
```
|
||||
Graph complete. Outputs in PATH_TO_DIR/graphify-out/
|
||||
|
||||
graph.html - interactive graph, open in browser
|
||||
GRAPH_REPORT.md - audit report
|
||||
graph.json - raw graph data
|
||||
obsidian/ - Obsidian vault (only if --obsidian was given)
|
||||
```
|
||||
|
||||
If graphify saved you time, consider supporting it: https://github.com/sponsors/safishamsi
|
||||
|
||||
Replace PATH_TO_DIR with the actual absolute path of the directory that was processed.
|
||||
|
||||
If PATH_TO_DIR is inside a git repository, add one line to the report: `graphify-out/` was written into the working tree, so unless the repository already ignores it, it will show in `git status`; the user can add `graphify-out/` to `.gitignore`, or commit only the outputs their team should share (see https://docs.graphify.com/guides/team-workflows).
|
||||
|
||||
Then paste these sections from GRAPH_REPORT.md directly into the chat:
|
||||
- God Nodes
|
||||
- Surprising Connections
|
||||
- Suggested Questions
|
||||
|
||||
Do NOT paste the full report - just those three sections. Keep it concise.
|
||||
|
||||
Then immediately offer to explore. Pick the single most interesting suggested question from the report - the one that crosses the most community boundaries or has the most surprising bridge node - and ask:
|
||||
|
||||
> "The most interesting question this graph can answer: **[question]**. Want me to trace it?"
|
||||
|
||||
If the user says yes, run `/graphify query "[question]"` on the graph and walk them through the answer using the graph structure - which nodes connect, which community boundaries get crossed, what the path reveals. Keep going as long as they want to explore. Each answer should end with a natural follow-up ("this connects to X - want to go deeper?") so the session feels like navigation, not a one-shot report.
|
||||
|
||||
The graph is the map. Your job after the pipeline is to be the guide.
|
||||
|
||||
---
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
# Semantic extraction: subagent prompt, coverage check and merge (Step B2 prompt + Step B3)
|
||||
|
||||
Moved out of SKILL.md so it is read only when documents, papers or images need extraction.
|
||||
|
||||
Subagent prompt template:
|
||||
|
||||
See `references/extraction-spec.md` for the exact subagent prompt (JSON schema, node-ID rules, confidence rubric, frontmatter, hyperedge, and vision rules). Load it only here, only when at least one chunk holds a doc, paper, or image; a pure-code corpus has skipped Part B and never reads it. Pass each subagent that prompt verbatim with FILE_LIST, CHUNK_NUM, TOTAL_CHUNKS, DEEP_MODE, and CHUNK_PATH substituted, and have it write the result to CHUNK_PATH.
|
||||
|
||||
**Step B3 - Collect, cache, and merge**
|
||||
|
||||
Wait for all subagents. For each result:
|
||||
- Check that `graphify-out/.graphify_chunk_NN.json` exists on disk — this is the success signal
|
||||
- If the file exists and contains valid JSON with `nodes` and `edges`, include it and save to cache
|
||||
- If the file is missing, the subagent was likely dispatched as read-only (Explore type) — print a warning: "chunk N missing from disk — subagent may have been read-only. Re-run with general-purpose agent." Do not silently skip.
|
||||
- If a subagent failed or returned invalid JSON, print a warning and skip that chunk - do not abort
|
||||
|
||||
If more than half the chunks failed or are missing, stop and tell the user to re-run and ensure `subagent_type="general-purpose"` is used.
|
||||
|
||||
**Step B3.5 - Verify each reported file was actually read to its end**
|
||||
|
||||
A subagent that only read the first Read call's worth of a long file still writes a well-formed chunk, so "chunk file exists and parses" is not evidence of coverage. Compare each file's self-reported `coverage` against its real length:
|
||||
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json, glob
|
||||
from pathlib import Path
|
||||
|
||||
INPUT_PATH = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip()
|
||||
covered = {}
|
||||
for c in sorted(glob.glob('graphify-out/.graphify_chunk_*.json')):
|
||||
d = json.loads(Path(c).read_text(encoding=\"utf-8\"))
|
||||
for src, n in (d.get('coverage') or {}).items():
|
||||
covered[src] = max(covered.get(src, 0), int(n))
|
||||
|
||||
partial = []
|
||||
for src, n in sorted(covered.items()):
|
||||
p = Path(src)
|
||||
if not p.is_absolute():
|
||||
p = Path(INPUT_PATH) / src
|
||||
if not p.is_file():
|
||||
continue
|
||||
total = sum(1 for _ in p.open(encoding='utf-8', errors='replace'))
|
||||
if total and n < total:
|
||||
partial.append((src, n, total))
|
||||
|
||||
Path('graphify-out/.graphify_partial.txt').write_text('\n'.join(s for s, _, _ in partial), encoding=\"utf-8\")
|
||||
if partial:
|
||||
print(f'WARNING: {len(partial)} file(s) were only partly read. They are cached as partial and retried on the next --update:')
|
||||
for src, n, total in partial:
|
||||
print(f' {src}: read {n} of {total} lines')
|
||||
else:
|
||||
print('Coverage OK - every file a subagent reported was read to its last line')
|
||||
"
|
||||
```
|
||||
|
||||
Tell the user which files came up short and what was lost (the line counts are printed above). The nodes already extracted for those files are kept - they are still true - but the file is cached as partial and left unstamped in the manifest below, so the next `--update` re-dispatches it and replace-on-re-extract swaps the partial node set for a complete one. This is the same mechanism the library path uses for a truncated LLM response (#933, #1950), so both paths now recover instead of silently keeping a half-read file forever. A file a subagent did not report at all is not counted as partial - there is nothing to compare against - which is why the prompt rule in `references/extraction-spec.md` asks for the count in the first place.
|
||||
|
||||
Merge all chunk files into `.graphify_semantic_new.json`. **After each Agent call completes, read the real token counts from the Agent tool result's `usage` field and write them back into the chunk JSON before merging** — the chunk JSON itself always has placeholder zeros. Then run:
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json, glob
|
||||
from pathlib import Path
|
||||
|
||||
chunks = sorted(glob.glob('graphify-out/.graphify_chunk_*.json'))
|
||||
all_nodes, all_edges, all_hyperedges = [], [], []
|
||||
total_in, total_out = 0, 0
|
||||
for c in chunks:
|
||||
d = json.loads(Path(c).read_text(encoding=\"utf-8\"))
|
||||
all_nodes += d.get('nodes', [])
|
||||
all_edges += d.get('edges', [])
|
||||
all_hyperedges += d.get('hyperedges', [])
|
||||
total_in += d.get('input_tokens', 0)
|
||||
total_out += d.get('output_tokens', 0)
|
||||
Path('graphify-out/.graphify_semantic_new.json').write_text(json.dumps({
|
||||
'nodes': all_nodes, 'edges': all_edges, 'hyperedges': all_hyperedges,
|
||||
'input_tokens': total_in, 'output_tokens': total_out,
|
||||
}, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'Merged {len(chunks)} chunks: {total_in:,} in / {total_out:,} out tokens')
|
||||
"
|
||||
```
|
||||
|
||||
Save new results to cache. Pass the same SPEC_PATH as Step B0 — it stamps each entry with the prompt that produced it, and a write under a different prompt than the read lands where the next run won't look (#1939):
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from graphify.cache import save_semantic_cache
|
||||
from pathlib import Path
|
||||
|
||||
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
uncached = [line for line in Path('graphify-out/.graphify_uncached.txt').read_text(encoding=\"utf-8\").splitlines() if line]
|
||||
# A partly-read file is cached with partial: True, which load_cached treats as a MISS,
|
||||
# so the next --update re-dispatches it instead of serving a truncated extraction forever
|
||||
# (#2225). Same contract the library path uses for a truncated LLM response (#933/#1950).
|
||||
partial = [line for line in Path('graphify-out/.graphify_partial.txt').read_text(encoding=\"utf-8\").splitlines() if line] if Path('graphify-out/.graphify_partial.txt').exists() else []
|
||||
INPUT_PATH = Path('graphify-out/.graphify_root').read_text(encoding=\"utf-8-sig\").strip()
|
||||
saved = save_semantic_cache(new.get('nodes', []), new.get('edges', []), new.get('hyperedges', []), root=INPUT_PATH, allowed_source_files=uncached, prompt_file='SPEC_PATH', partial_source_files=partial)
|
||||
note = f' ({len(partial)} left partial for retry)' if partial else ''
|
||||
print(f'Cached {saved} files' + note)
|
||||
"
|
||||
```
|
||||
|
||||
Merge cached + new results into `graphify-out/.graphify_semantic.json`:
|
||||
```bash
|
||||
$(cat graphify-out/.graphify_python) -c "
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
cached = json.loads(Path('graphify-out/.graphify_cached.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_cached.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
new = json.loads(Path('graphify-out/.graphify_semantic_new.json').read_text(encoding=\"utf-8\")) if Path('graphify-out/.graphify_semantic_new.json').exists() else {'nodes':[],'edges':[],'hyperedges':[]}
|
||||
|
||||
all_nodes = cached['nodes'] + new.get('nodes', [])
|
||||
all_edges = cached['edges'] + new.get('edges', [])
|
||||
all_hyperedges = cached.get('hyperedges', []) + new.get('hyperedges', [])
|
||||
seen = set()
|
||||
deduped = []
|
||||
for n in all_nodes:
|
||||
if n['id'] not in seen:
|
||||
seen.add(n['id'])
|
||||
deduped.append(n)
|
||||
|
||||
merged = {
|
||||
'nodes': deduped,
|
||||
'edges': all_edges,
|
||||
'hyperedges': all_hyperedges,
|
||||
'input_tokens': new.get('input_tokens', 0),
|
||||
'output_tokens': new.get('output_tokens', 0),
|
||||
}
|
||||
Path('graphify-out/.graphify_semantic.json').write_text(json.dumps(merged, indent=2, ensure_ascii=False), encoding=\"utf-8\")
|
||||
print(f'Extraction complete - {len(deduped)} nodes, {len(all_edges)} edges ({len(cached[\"nodes\"])} from cache, {len(new.get(\"nodes\",[]))} new)')
|
||||
"
|
||||
```
|
||||
Clean up temp files: `rm -f graphify-out/.graphify_cached.json graphify-out/.graphify_uncached.txt graphify-out/.graphify_semantic_new.json`
|
||||
|
||||
Reference in new issue
Block a user