perf(scanner): ottimizza scansione destinazione con pass unico e barra determinata
Sostituisce le due scansioni sequenziali (cartelle + file CAD) con un unico passaggio parallelo in services/destinationScanner.js. La lettura di primo livello fornisce il totale delle sezioni, rendendo la barra di progresso determinata (N / totale) durante l'analisi della destinazione. Il label mostra contemporaneamente sezioni completate e file CAD trovati
This commit is contained in:
@@ -32,16 +32,22 @@ function hideProgress() {
|
|||||||
progressContainer.style.display = 'none';
|
progressContainer.style.display = 'none';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let _indexFilesFound = 0;
|
||||||
|
|
||||||
function updateProgress({ phase, current, total, file, scanned, folder }) {
|
function updateProgress({ phase, current, total, file, scanned, folder }) {
|
||||||
if (phase === 'scan') {
|
if (phase === 'scan') {
|
||||||
|
_indexFilesFound = 0;
|
||||||
progressLabel.textContent = 'Scansione file sorgente...';
|
progressLabel.textContent = 'Scansione file sorgente...';
|
||||||
progressBar.removeAttribute('value');
|
progressBar.removeAttribute('value');
|
||||||
} else if (phase === 'index-dest') {
|
} else if (phase === 'index-dest') {
|
||||||
progressLabel.textContent = `Analisi destinazione: ${scanned} cartelle scansionate${folder ? ` — ${folder}` : ''}`;
|
progressBar.max = total;
|
||||||
progressBar.removeAttribute('value');
|
progressBar.value = current;
|
||||||
|
const filesInfo = _indexFilesFound > 0 ? ` — ${_indexFilesFound} file CAD trovati` : '';
|
||||||
|
progressLabel.textContent = `Analisi destinazione: ${current} / ${total}${folder ? ` — ${folder}` : ''}${filesInfo}`;
|
||||||
} else if (phase === 'index-dup') {
|
} else if (phase === 'index-dup') {
|
||||||
progressLabel.textContent = `Ricerca duplicati: ${scanned} file CAD trovati${file ? ` — ${file}` : ''}`;
|
_indexFilesFound = scanned;
|
||||||
progressBar.removeAttribute('value');
|
const dirInfo = progressBar.max > 0 ? `${progressBar.value} / ${progressBar.max} — ` : '';
|
||||||
|
progressLabel.textContent = `Analisi destinazione: ${dirInfo}${scanned} file CAD trovati`;
|
||||||
} else if (phase === 'copy') {
|
} else if (phase === 'copy') {
|
||||||
progressBar.max = total;
|
progressBar.max = total;
|
||||||
progressBar.value = current;
|
progressBar.value = current;
|
||||||
|
|||||||
114
services/destinationScanner.js
Normal file
114
services/destinationScanner.js
Normal file
@@ -0,0 +1,114 @@
|
|||||||
|
const fs = require('fs-extra');
|
||||||
|
const path = require('path');
|
||||||
|
const { getCadInfo } = require('./router');
|
||||||
|
|
||||||
|
const CODE_FOLDER_REGEX = /^\d{3}$/;
|
||||||
|
const MAX_SCAN_DEPTH = 6;
|
||||||
|
|
||||||
|
function toCadKey(cadInfo) {
|
||||||
|
return String(cadInfo?.key || '').toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Scansiona la cartella di destinazione in un unico passaggio parallelo e
|
||||||
|
* costruisce contemporaneamente:
|
||||||
|
* - folderIndex: Map<group3digits, fullPath[]> (per il routing)
|
||||||
|
* - cadKeyIndex: Set<cadKey> (per il rilevamento duplicati)
|
||||||
|
*
|
||||||
|
* Legge prima le sottocartelle di primo livello (1 sola readdir) per ottenere
|
||||||
|
* il totale e mostrare una barra determinata durante la scansione.
|
||||||
|
*/
|
||||||
|
async function buildDestinationIndexes(destinationRoot, onProgress) {
|
||||||
|
const folderIndex = new Map();
|
||||||
|
const cadKeyIndex = new Set();
|
||||||
|
|
||||||
|
if (!destinationRoot || !(await fs.pathExists(destinationRoot))) {
|
||||||
|
return { folderIndex, cadKeyIndex };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Lettura di primo livello: 1 sola chiamata per avere il totale
|
||||||
|
let topEntries;
|
||||||
|
try {
|
||||||
|
topEntries = await fs.readdir(destinationRoot, { withFileTypes: true });
|
||||||
|
} catch {
|
||||||
|
return { folderIndex, cadKeyIndex };
|
||||||
|
}
|
||||||
|
|
||||||
|
const topDirs = topEntries
|
||||||
|
.filter((e) => e.isDirectory())
|
||||||
|
.map((e) => ({ name: e.name, fullPath: path.join(destinationRoot, e.name) }));
|
||||||
|
|
||||||
|
const total = topDirs.length;
|
||||||
|
let completed = 0;
|
||||||
|
let scannedFiles = 0;
|
||||||
|
|
||||||
|
// Registra i file CAD al primo livello (non cartelle)
|
||||||
|
for (const entry of topEntries) {
|
||||||
|
if (!entry.isFile()) continue;
|
||||||
|
const cadInfo = getCadInfo(entry.name);
|
||||||
|
if (cadInfo) {
|
||||||
|
scannedFiles += 1;
|
||||||
|
cadKeyIndex.add(toCadKey(cadInfo));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Walk ricorsivo del sottoalbero di una cartella di primo livello
|
||||||
|
async function walkSubtree(dirPath, depth) {
|
||||||
|
if (depth > MAX_SCAN_DEPTH) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let entries;
|
||||||
|
try {
|
||||||
|
entries = await fs.readdir(dirPath, { withFileTypes: true });
|
||||||
|
} catch {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const subdirPromises = [];
|
||||||
|
|
||||||
|
for (const entry of entries) {
|
||||||
|
const fullPath = path.join(dirPath, entry.name);
|
||||||
|
|
||||||
|
if (entry.isDirectory()) {
|
||||||
|
if (CODE_FOLDER_REGEX.test(entry.name)) {
|
||||||
|
const rows = folderIndex.get(entry.name) || [];
|
||||||
|
rows.push(fullPath);
|
||||||
|
folderIndex.set(entry.name, rows);
|
||||||
|
}
|
||||||
|
subdirPromises.push(walkSubtree(fullPath, depth + 1));
|
||||||
|
} else if (entry.isFile()) {
|
||||||
|
const cadInfo = getCadInfo(entry.name);
|
||||||
|
if (cadInfo) {
|
||||||
|
scannedFiles += 1;
|
||||||
|
onProgress?.({ phase: 'index-dup', scanned: scannedFiles, file: entry.name });
|
||||||
|
cadKeyIndex.add(toCadKey(cadInfo));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await Promise.all(subdirPromises);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Processa tutte le cartelle di primo livello in parallelo
|
||||||
|
await Promise.all(
|
||||||
|
topDirs.map(async ({ name, fullPath }) => {
|
||||||
|
onProgress?.({ phase: 'index-dest', current: completed, total, folder: name });
|
||||||
|
|
||||||
|
if (CODE_FOLDER_REGEX.test(name)) {
|
||||||
|
const rows = folderIndex.get(name) || [];
|
||||||
|
rows.push(fullPath);
|
||||||
|
folderIndex.set(name, rows);
|
||||||
|
}
|
||||||
|
|
||||||
|
await walkSubtree(fullPath, 1);
|
||||||
|
|
||||||
|
completed += 1;
|
||||||
|
onProgress?.({ phase: 'index-dest', current: completed, total, folder: name });
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
return { folderIndex, cadKeyIndex };
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = { buildDestinationIndexes, toCadKey };
|
||||||
@@ -1,8 +1,7 @@
|
|||||||
const fs = require('fs-extra');
|
const fs = require('fs-extra');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
const { getDestinationDecision, getCadInfo } = require('./router');
|
const { getDestinationDecision, getCadInfo } = require('./router');
|
||||||
const { buildDestinationIndex } = require('./destinationIndex');
|
const { buildDestinationIndexes, toCadKey } = require('./destinationScanner');
|
||||||
const { buildExistingCadKeyIndex, toCadKey } = require('./duplicateIndex');
|
|
||||||
const { prepareUnroutedTarget, prepareDuplicateTarget, getSkippedTarget } = require('./unrouted');
|
const { prepareUnroutedTarget, prepareDuplicateTarget, getSkippedTarget } = require('./unrouted');
|
||||||
|
|
||||||
function parseNumericVersion(version) {
|
function parseNumericVersion(version) {
|
||||||
@@ -70,8 +69,7 @@ async function processFolder(folder, config, onProgress) {
|
|||||||
onProgress?.({ phase: 'scan' });
|
onProgress?.({ phase: 'scan' });
|
||||||
const files = await collectFilesRecursively(folder);
|
const files = await collectFilesRecursively(folder);
|
||||||
|
|
||||||
const destinationIndex = await buildDestinationIndex(config?.destination, onProgress);
|
const { folderIndex: destinationIndex, cadKeyIndex: existingCadKeys } = await buildDestinationIndexes(config?.destination, onProgress);
|
||||||
const existingCadKeys = await buildExistingCadKeyIndex(config?.destination, onProgress);
|
|
||||||
const sourceMaxVersions = buildHighestNumericVersionByCadKey(files);
|
const sourceMaxVersions = buildHighestNumericVersionByCadKey(files);
|
||||||
const result = {
|
const result = {
|
||||||
scanned: 0,
|
scanned: 0,
|
||||||
|
|||||||
@@ -3,8 +3,7 @@ const fs = require('fs-extra');
|
|||||||
const path = require('path');
|
const path = require('path');
|
||||||
const { pipeline } = require('stream/promises');
|
const { pipeline } = require('stream/promises');
|
||||||
const { getDestinationDecision, getCadInfo } = require('./router');
|
const { getDestinationDecision, getCadInfo } = require('./router');
|
||||||
const { buildDestinationIndex } = require('./destinationIndex');
|
const { buildDestinationIndexes, toCadKey } = require('./destinationScanner');
|
||||||
const { buildExistingCadKeyIndex, toCadKey } = require('./duplicateIndex');
|
|
||||||
const { prepareUnroutedTarget, prepareDuplicateTarget, getSkippedTarget } = require('./unrouted');
|
const { prepareUnroutedTarget, prepareDuplicateTarget, getSkippedTarget } = require('./unrouted');
|
||||||
|
|
||||||
function parseNumericVersion(version) {
|
function parseNumericVersion(version) {
|
||||||
@@ -60,8 +59,7 @@ async function processZip(zipPath, config, onProgress) {
|
|||||||
onProgress?.({ phase: 'scan' });
|
onProgress?.({ phase: 'scan' });
|
||||||
const { index: sourceMaxVersions, total } = await buildZipHighestNumericVersionByCadKey(zipPath);
|
const { index: sourceMaxVersions, total } = await buildZipHighestNumericVersionByCadKey(zipPath);
|
||||||
|
|
||||||
const destinationIndex = await buildDestinationIndex(config?.destination, onProgress);
|
const { folderIndex: destinationIndex, cadKeyIndex: existingCadKeys } = await buildDestinationIndexes(config?.destination, onProgress);
|
||||||
const existingCadKeys = await buildExistingCadKeyIndex(config?.destination, onProgress);
|
|
||||||
|
|
||||||
const stream = fs.createReadStream(zipPath).pipe(unzipper.Parse({ forceStream: true }));
|
const stream = fs.createReadStream(zipPath).pipe(unzipper.Parse({ forceStream: true }));
|
||||||
const result = {
|
const result = {
|
||||||
|
|||||||
Reference in New Issue
Block a user