Fix scanner latency for large uploads
This commit is contained in:
1 parent
304a30c7c2
commit
f3e5386d51
1 file changed
+49
-37
+49
-37
@@ -52,48 +52,59 @@ export async function runFullScan(): Promise<{
|
||||
let removed = 0;
|
||||
let unchanged = 0;
|
||||
|
||||
// 3. Find new and changed files
|
||||
// 3. Find new and changed files
|
||||
const newFiles: ScannedFile[] = [];
|
||||
for (const [relPath, file] of scannedFiles) {
|
||||
const existing = dbPaths.get(relPath);
|
||||
|
||||
if (!existing) {
|
||||
// New file — check stability first
|
||||
const isStable = await checkFileStability(file.absolutePath);
|
||||
if (!isStable) {
|
||||
console.log(`[Scanner] Skipping unstable file: ${relPath}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const mediaType = getMediaType(file.ext);
|
||||
const mimeType = getMimeType(file.ext);
|
||||
|
||||
await prisma.mediaItem.create({
|
||||
data: {
|
||||
relativePath: relPath,
|
||||
filename: file.filename,
|
||||
directory: file.directory,
|
||||
mediaType,
|
||||
mimeType,
|
||||
fileSize: file.size,
|
||||
lastModified: file.mtime,
|
||||
lastScanned: new Date(),
|
||||
published: false,
|
||||
},
|
||||
});
|
||||
|
||||
// Enqueue processing jobs
|
||||
await enqueueProcessing(relPath, mediaType);
|
||||
added++;
|
||||
console.log(`[Scanner] Added: ${relPath}`);
|
||||
} catch (error) {
|
||||
const msg = `Failed to add ${relPath}: ${error instanceof Error ? error.message : String(error)}`;
|
||||
errors.push(msg);
|
||||
console.error(`[Scanner] ${msg}`);
|
||||
}
|
||||
newFiles.push(file);
|
||||
} else {
|
||||
unchanged++;
|
||||
dbPaths.delete(relPath); // Mark as seen
|
||||
dbPaths.delete(relPath);
|
||||
}
|
||||
}
|
||||
|
||||
// Check stability for all new files concurrently (takes max 3 seconds total)
|
||||
const stableResults = await Promise.all(
|
||||
newFiles.map(async (file) => {
|
||||
const isStable = await checkFileStability(file.absolutePath);
|
||||
return { file, isStable };
|
||||
})
|
||||
);
|
||||
|
||||
for (const { file, isStable } of stableResults) {
|
||||
const relPath = file.relativePath;
|
||||
if (!isStable) {
|
||||
console.log(`[Scanner] Skipping unstable file: ${relPath}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const mediaType = getMediaType(file.ext);
|
||||
const mimeType = getMimeType(file.ext);
|
||||
|
||||
await prisma.mediaItem.create({
|
||||
data: {
|
||||
relativePath: relPath,
|
||||
filename: file.filename,
|
||||
directory: file.directory,
|
||||
mediaType,
|
||||
mimeType,
|
||||
fileSize: file.size,
|
||||
lastModified: file.mtime,
|
||||
lastScanned: new Date(),
|
||||
published: false,
|
||||
},
|
||||
});
|
||||
|
||||
// Enqueue processing jobs
|
||||
await enqueueProcessing(relPath, mediaType);
|
||||
added++;
|
||||
console.log(`[Scanner] Added: ${relPath}`);
|
||||
} catch (error) {
|
||||
const msg = `Failed to add ${relPath}: ${error instanceof Error ? error.message : String(error)}`;
|
||||
errors.push(msg);
|
||||
console.error(`[Scanner] ${msg}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -203,3 +214,4 @@ async function checkFileStability(filePath: string): Promise<boolean> {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user