From ab19a3d0bfb398c239ec0d156bcfc2f67d527ac1 Mon Sep 17 00:00:00 2001 From: Levy Tate <78818969+iLevyTate@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:39:53 -0500 Subject: [PATCH 01/10] fix(organize): send files no Smart Folder fits to review instead of a weak match A file that fit no folder still went to whichever Smart Folder ranked first, since every semantic candidate only had to clear 0.4. An unrelated Markdown report could land in an SEO folder. Routing now goes to a Smart Folder only on a confident match: - a learned pattern, learned correction or feedback rule points there - the content score is 0.55 or more (FOLDER_MATCH_CONFIDENCE) and no other folder is within 0.03 (NO_SPREAD_THRESHOLD) - the folder claims the extension, is named after the file type, or, in the modes that rank by the label, is named outright by it Otherwise the file needs review: it goes to the default folder, or stays unassigned when there is none. The explanation says why ("No Smart Folder matched this file well", "matched this file about equally"). - Date, workflow and other template defaults no longer pick a folder. - getSuggestionsForFile returns needsReview and routingDecision; getBatchSuggestions lists unassigned files. - Auto-organize, batch organize and the download watcher hold a needs-review file whatever the threshold. - The Organize card shows "Needs review" with the reason and where the file goes, lists the closest folders, and skips name matching for it. --- .../autoOrganize/AutoOrganizeServiceCore.js | 11 +- .../services/autoOrganize/batchProcessor.js | 64 ++- .../services/autoOrganize/fileProcessor.js | 31 +- .../OrganizationSuggestionServiceCore.js | 126 ++++- .../services/organization/routingDecision.js | 438 ++++++++++++++++++ .../services/organization/suggestionRanker.js | 39 ++ .../components/organize/ReadyFileItem.jsx | 67 ++- .../phases/organize/organizeDestination.js | 40 +- .../phases/organize/useOrganizeSuggestions.js | 22 +- src/shared/categoryFolderMatching.js | 62 +++ test/OrganizationSuggestionService.test.js | 19 +- ...onSuggestionServiceCore.labelScore.test.js | 21 +- ...nSuggestionServiceCore.needsReview.test.js | 380 +++++++++++++++ test/autoOrganize.needsReview.test.js | 254 ++++++++++ ...ategoryFolderMatching.namedByLabel.test.js | 51 ++ test/components/ReadyFileItem.routing.test.js | 69 +++ test/learning.llmModeCorrection.test.js | 5 +- test/organizeDestination.test.js | 76 +++ test/routingDecision.test.js | 116 +++++ test/useOrganizeSuggestions.test.js | 28 ++ 20 files changed, 1849 insertions(+), 70 deletions(-) create mode 100644 src/main/services/organization/routingDecision.js create mode 100644 test/OrganizationSuggestionServiceCore.needsReview.test.js create mode 100644 test/autoOrganize.needsReview.test.js create mode 100644 test/categoryFolderMatching.namedByLabel.test.js create mode 100644 test/routingDecision.test.js diff --git a/src/main/services/autoOrganize/AutoOrganizeServiceCore.js b/src/main/services/autoOrganize/AutoOrganizeServiceCore.js index 4ca589de..12ab2390 100644 --- a/src/main/services/autoOrganize/AutoOrganizeServiceCore.js +++ b/src/main/services/autoOrganize/AutoOrganizeServiceCore.js @@ -222,7 +222,10 @@ class AutoOrganizeServiceCore { if ( !batchSuggestions?.success || !batchSuggestions.groups || - batchSuggestions.groups.length === 0 + (batchSuggestions.groups.length === 0 && + !( + Array.isArray(batchSuggestions.unassigned) && batchSuggestions.unassigned.length > 0 + )) ) { // Fallback to individual processing if batch fails logger.warn( @@ -254,6 +257,12 @@ class AutoOrganizeServiceCore { processedFileIdentifiers.add(fileWithSuggestion.path || fileWithSuggestion.name); } } + // Held for review by processBatchResults; not asked about again. + for (const entry of Array.isArray(batchSuggestions.unassigned) + ? batchSuggestions.unassigned + : []) { + processedFileIdentifiers.add(entry?.path || entry?.name); + } // Process any unprocessed files individually as fallback const unprocessedFiles = batch.filter( diff --git a/src/main/services/autoOrganize/batchProcessor.js b/src/main/services/autoOrganize/batchProcessor.js index bb1faed1..d62dd84d 100644 --- a/src/main/services/autoOrganize/batchProcessor.js +++ b/src/main/services/autoOrganize/batchProcessor.js @@ -22,6 +22,8 @@ const CONFIDENCE_THRESHOLDS = { FALLBACK: 0.3, // Confidence assigned to fallback destinations DEFAULT: 0.75 // Default threshold when not specified }; +/** Used when the engine flagged a file for review without saying why. */ +const NEEDS_REVIEW_EXPLANATION = 'No Smart Folder matched this file well; held for review'; /** * Process batch suggestion results @@ -50,6 +52,20 @@ function processBatchResults(batchSuggestions, files, options, results, smartFol // Validate groups array defensively const groups = Array.isArray(batchSuggestions?.groups) ? batchSuggestions.groups : []; + // No Smart Folder was a confident match and there is no default folder: + // the engine left these unassigned. They are held, never moved. + for (const entry of Array.isArray(batchSuggestions?.unassigned) + ? batchSuggestions.unassigned + : []) { + const file = fileMap.get(entry?.path || entry?.name) || entry; + if (!file?.path) continue; + holdForReview(results, file, { + alternatives: entry.alternatives, + confidence: 0, + explanation: entry.explanation || NEEDS_REVIEW_EXPLANATION + }); + } + for (const group of groups) { // Validate group.files is an array before iterating if (!Array.isArray(group?.files)) { @@ -74,6 +90,19 @@ function processBatchResults(batchSuggestions, files, options, results, smartFol continue; } + // No Smart Folder was a confident match. Whatever the threshold, the + // file is held: the default folder it was given is not a destination + // anyone chose. + if (fileWithSuggestion.needsReview === true || suggestion?.needsReview === true) { + holdForReview(results, file, { + suggestion, + alternatives: fileWithSuggestion.alternatives, + confidence, + explanation: fileWithSuggestion.explanation || NEEDS_REVIEW_EXPLANATION + }); + continue; + } + const canonicalSuggestion = resolveSuggestionToSmartFolder(suggestion, smartFolders); if (!suggestion) { @@ -179,6 +208,17 @@ async function batchOrganize( // Track planned destinations across all groups in the batch to prevent cross-group collisions const plannedDestinations = new Map(); + // Needs review with no default folder: the engine left these unassigned. + const unassigned = Array.isArray(batchSuggestions.unassigned) ? batchSuggestions.unassigned : []; + if (unassigned.length > 0) { + results.skipped.push({ + folder: null, + files: unassigned, + confidence: 0, + reason: 'Needs review' + }); + } + // Process groups with error handling const groups = Array.isArray(batchSuggestions.groups) ? batchSuggestions.groups : []; for (const group of groups) { @@ -199,7 +239,21 @@ async function batchOrganize( }); continue; } - const groupFiles = group.files; + // A file no Smart Folder matched confidently is never auto-approved, even + // in a group whose average clears the threshold. + const reviewFiles = group.files.filter( + (/** @type {any} */ f) => f?.needsReview === true || f?.suggestion?.needsReview === true + ); + if (reviewFiles.length > 0) { + results.skipped.push({ + folder: group.folder, + files: reviewFiles, + confidence: group.confidence, + reason: 'Needs review' + }); + } + const groupFiles = group.files.filter((/** @type {any} */ f) => !reviewFiles.includes(f)); + if (groupFiles.length === 0) continue; try { if (group.confidence >= confidenceThreshold) { @@ -210,7 +264,7 @@ async function batchOrganize( // folder is held, not dumped into Uncategorized. results.skipped.push({ folder: group.folder, - files: group.files, + files: groupFiles, confidence: group.confidence, reason: 'Not a smart folder; held for review' }); @@ -293,9 +347,7 @@ async function batchOrganize( const failedPaths = new Set(groupFailures.map((failure) => failure.filePath)); results.groups.push({ folder: canonicalGroup.folder, - files: group.files.filter( - (/** @type {any} */ f) => f?.path && !failedPaths.has(f.path) - ), + files: groupFiles.filter((/** @type {any} */ f) => f?.path && !failedPaths.has(f.path)), confidence: group.confidence, autoApproved: true, partialSuccess: groupFailures.length > 0 @@ -305,7 +357,7 @@ async function batchOrganize( // Skip low confidence groups or groups with no valid smart folder, putting them in skipped/manual review results.skipped.push({ folder: group.folder, - files: group.files, + files: groupFiles, confidence: group.confidence, reason: group.confidence < confidenceThreshold ? 'Low confidence' : 'Not a smart folder' }); diff --git a/src/main/services/autoOrganize/fileProcessor.js b/src/main/services/autoOrganize/fileProcessor.js index 8baf32fa..b38eb8b3 100644 --- a/src/main/services/autoOrganize/fileProcessor.js +++ b/src/main/services/autoOrganize/fileProcessor.js @@ -176,6 +176,18 @@ async function processFilesIndividually(files, smartFolders, options, results, s continue; } + // No Smart Folder was a confident match. Held whatever the threshold: + // the default folder it was given is not a destination anyone chose. + if (suggestion?.success && suggestion.needsReview === true) { + holdForReview(results, file, { + suggestion: suggestion.primary || null, + alternatives: suggestion.alternatives, + confidence: suggestion.confidence || 0, + explanation: suggestion.explanation || 'No Smart Folder matched this file well' + }); + continue; + } + if (!suggestion || !suggestion.success || !suggestion.primary) { // D2: no suggestion, nothing confident. Offer the rule-based folder // for review instead of moving there at confidence 0.3. @@ -355,9 +367,16 @@ async function processNewFile(filePath, smartFolders, options, suggestionService }); const canonicalPrimary = resolveSuggestionToSmartFolder(suggestion?.primary, smartFolders); + // No Smart Folder was a confident match: never moved, whatever the threshold. + const needsReview = suggestion?.needsReview === true; // Only auto-organize if confidence is very high and destination resolves to a configured smart folder - if (suggestion?.success && canonicalPrimary && suggestion.confidence >= effectiveThreshold) { + if ( + suggestion?.success && + !needsReview && + canonicalPrimary && + suggestion.confidence >= effectiveThreshold + ) { const safePrimary = safeSuggestion(canonicalPrimary); const initialDest = buildDestinationPath(file, safePrimary, options.defaultLocation, false); const resolved = await resolveDestinationCollision(initialDest); @@ -399,18 +418,22 @@ async function processNewFile(filePath, smartFolders, options, suggestionService // left it both moved and unorganized and contradicted the setting's own // copy ("Lower confidence matches require manual review"). const confidence = suggestion?.confidence || 0; + // A file that needs review is not offered its analysis label as the + // folder: no folder matched it, and the label may name one anyway. const suggestedFolder = canonicalPrimary?.name || suggestion?.primary?.folder || suggestion?.primary?.name || - (typeof analysis?.category === 'string' ? analysis.category : null); + (!needsReview && typeof analysis?.category === 'string' ? analysis.category : null); logger.info('[AutoOrganize] Holding file for review (not moved)', { file: filePath, confidence, threshold: effectiveThreshold, suggestedFolder, - resolvedToSmartFolder: Boolean(canonicalPrimary) + resolvedToSmartFolder: Boolean(canonicalPrimary), + needsReview, + reviewReason: suggestion?.routingDecision?.reason || null }); return { @@ -421,6 +444,8 @@ async function processNewFile(filePath, smartFolders, options, suggestionService confidence, analysisConfidence: analysis.confidence, requiresManualReview: true, + needsReview, + ...(needsReview && suggestion?.explanation ? { explanation: suggestion.explanation } : {}), suggestedFolder }; } catch (error) { diff --git a/src/main/services/organization/OrganizationSuggestionServiceCore.js b/src/main/services/organization/OrganizationSuggestionServiceCore.js index 5390387f..39346201 100644 --- a/src/main/services/organization/OrganizationSuggestionServiceCore.js +++ b/src/main/services/organization/OrganizationSuggestionServiceCore.js @@ -64,8 +64,10 @@ const { rankSuggestions, calculateConfidence, capSuggestionConfidence, - generateExplanation + generateExplanation, + explainReview } = require('./suggestionRanker'); +const { decideRouting, CONFIDENT_CONTENT_SCORE, AMBIGUOUS_MARGIN } = require('./routingDecision'); const { calculateFolderFitScore, @@ -851,11 +853,14 @@ class OrganizationSuggestionServiceCore { // A template that names its folder after the analysis label (or its // purpose sentence) is the label again, at a flat 0.6. The label enters // below through _buildLlmCategorySuggestion, gated on routing mode. + // A template whose folder is its own placeholder default ("Uncategorized" + // from the date template, "Misc") says nothing about the file: date and + // workflow templates shape names and subfolders, not the destination. const strategyMatches = getStrategyBasedSuggestions( normalizedFile, smartFolders, this.config.strategyMatchThreshold - ).filter((match) => match.namedBy !== 'category' && match.namedBy !== 'purpose'); + ).filter((match) => match.namedBy === 'fileType'); const patternMatches = this.patternMatcher.getPatternBasedSuggestions(normalizedFile); const semanticTopScore = @@ -972,24 +977,70 @@ class OrganizationSuggestionServiceCore { patternMatches ); - // A folder that "could be enhanced" for this file is an idea, not a - // destination: its +0.2 score could reach the auto-move threshold. - let primary = - rankedSuggestions.find( - (suggestion) => suggestion.isSmartFolder && suggestion.source !== 'improvement' - ) || null; - if (!primary) { + // Route only to a confident match (routingDecision.js). A folder that + // "could be enhanced" for this file is an idea, never a destination, and + // a file no folder fits confidently needs review: it goes to the default + // folder, or stays unassigned when there is none. + const decision = decideRouting({ + file: normalizedFile, + ranked: rankedSuggestions, + sourceSuggestions: normalizedSuggestions, + patternMatches, + smartFolders, + // The label names a folder only in the modes that rank by it. + useLabels: shouldIncludeLlmCategory + }); + /** @type {any} */ + let primary = null; + if (!decision.needsReview) { + primary = decision.candidate; + // The label path in LLM routing keeps its own explanation. + if (decision.nameMatch && primary.method !== 'llm_category') { + primary = { ...primary, nameMatch: decision.nameMatch }; + } + const routed = primary; + rankedSuggestions = [ + routed, + ...rankedSuggestions.filter( + (suggestion) => suggestion !== decision.candidate && suggestion !== routed + ) + ]; + } else { + const review = { + reason: decision.reason, + best: decision.best, + runnerUp: decision.runnerUp + }; const defaultSuggestion = await this._getDefaultFolderSuggestion( normalizedFile, smartFolders ); if (defaultSuggestion) { - primary = defaultSuggestion; - rankedSuggestions = [defaultSuggestion, ...rankedSuggestions]; - } else if (rankedSuggestions.length > 0) { - primary = rankedSuggestions[0]; + primary = { ...defaultSuggestion, needsReview: true, review }; + const defaultName = String(defaultSuggestion.folder || '').toLowerCase(); + rankedSuggestions = [ + primary, + ...rankedSuggestions.filter( + (suggestion) => String(suggestion?.folder || '').toLowerCase() !== defaultName + ) + ]; } + logger.info('[OrganizationSuggestionService] File needs review', { + file: normalizedFile.name, + reason: decision.reason, + best: decision.best, + runnerUp: decision.runnerUp, + defaultFolder: defaultSuggestion?.folder || null + }); } + logger.debug('[OrganizationSuggestionService] Routing decision', { + file: normalizedFile.name, + basis: decision.basis, + reason: decision.reason, + folder: primary?.folder || null, + best: decision.best, + runnerUp: decision.runnerUp + }); // Get folder improvements let folderImprovements = []; @@ -1003,8 +1054,10 @@ class OrganizationSuggestionServiceCore { const primaryFolderName = String(primary?.folder || '').toLowerCase(); systemAnalytics.recordOrganizationRouting({ - resolvedToUncategorized: primaryFolderName === 'uncategorized', - reason: primary?.method || primary?.source || routing?.reason || 'primary' + resolvedToUncategorized: decision.needsReview || primaryFolderName === 'uncategorized', + reason: decision.needsReview + ? `needs_review:${decision.reason}` + : primary?.method || primary?.source || routing?.reason || 'primary' }); return { @@ -1015,7 +1068,26 @@ class OrganizationSuggestionServiceCore { : [], strategies: getApplicableStrategies(normalizedFile), confidence: calculateConfidence(primary), - explanation: generateExplanation(primary, normalizedFile), + explanation: decision.needsReview + ? explainReview({ + reason: decision.reason, + best: decision.best, + runnerUp: decision.runnerUp + }) + : generateExplanation(primary, normalizedFile), + // True when no Smart Folder is a confident match. Nothing moves such a + // file on its own; Organize shows it as "Needs review". + needsReview: decision.needsReview, + routingDecision: { + basis: decision.basis, + reason: decision.reason, + folder: primary?.folder || null, + best: decision.best, + runnerUp: decision.runnerUp, + nameMatch: decision.nameMatch, + confidentScore: CONFIDENT_CONTENT_SCORE, + ambiguousMargin: AMBIGUOUS_MARGIN + }, folderImprovements, routing: { mode: routing.mode, @@ -1201,6 +1273,8 @@ class OrganizationSuggestionServiceCore { } ); + /** @type {any[]} */ + const unassigned = []; for (const result of /** @type {any} */ (batchResult).results) { if (result.error) { logger.warn('[OrganizationSuggestionService] File failed', { @@ -1211,6 +1285,20 @@ class OrganizationSuggestionServiceCore { } const { file, suggestion } = result; + // Needs review and no default folder: the file stays unassigned. It is + // listed so Organize can show it and nothing places it by default. + if (suggestion?.success && !suggestion.primary && suggestion.needsReview) { + unassigned.push({ + ...file, + suggestion: null, + alternatives: suggestion.alternatives, + confidence: 0, + explanation: suggestion.explanation, + needsReview: true, + routingDecision: suggestion.routingDecision + }); + continue; + } if (!suggestion?.success || !suggestion.primary) { logger.warn('[OrganizationSuggestionService] Skipping file with no primary suggestion', { file: file?.name @@ -1262,7 +1350,9 @@ class OrganizationSuggestionServiceCore { alternatives: suggestion.alternatives, confidence: suggestion.confidence, explanation: suggestion.explanation, - suggestedNewFolder + suggestedNewFolder, + needsReview: suggestion.needsReview === true, + routingDecision: suggestion.routingDecision }); // Update average confidence (guard against division by zero) @@ -1279,6 +1369,8 @@ class OrganizationSuggestionServiceCore { return { success: true, groups: Array.from(groups.values()), + // Files that need review with no default folder to hold them. + unassigned, patterns, recommendations, suggestedStrategy: selectBestStrategy(patterns, files), diff --git a/src/main/services/organization/routingDecision.js b/src/main/services/organization/routingDecision.js new file mode 100644 index 00000000..49510926 --- /dev/null +++ b/src/main/services/organization/routingDecision.js @@ -0,0 +1,438 @@ +// @ts-check +/** + * Needs-review routing: is the engine's best folder a confident match? + * + * Every candidate the engine ranks clears some floor (semantic matches only + * need 0.4), so a file that fits no folder still landed in whichever folder + * came out on top. A file is routed to a Smart Folder only when one of these + * holds for that folder: + * + * - learned: a learned pattern, a learned correction or a saved feedback + * rule points there + * - content: what the file contains matches it at CONFIDENT_CONTENT_SCORE + * or more, and no other folder comes within AMBIGUOUS_MARGIN + * - name: the folder claims the file's extension, is named outright by the + * file's type, or (in the routing modes that rank by the analysis label) + * is named outright by that label or by the file's vendor, organization, + * sender, entity or project + * + * Otherwise the file needs review: it goes to the default folder when one + * exists, and stays unassigned when none does. Template strategies (date, + * project, workflow) never decide: they describe naming and subfolders, not + * which folder a file belongs in. + * + * @module services/organization/routingDecision + */ + +const path = require('path'); +const { THRESHOLDS } = require('../../../shared/performanceConstants'); +const { + findSmartFolderNamedByLabel, + findSmartFolderForExtension, + isDefaultCategoryName +} = require('../../../shared/categoryFolderMatching'); +const { isFallbackAnalysis } = require('../../../shared/analysisProvenance'); +const { normalizePathForIndex } = require('../../../shared/pathSanitization'); +const { getFileTypeCategory } = require('../autoOrganize/fileTypeUtils'); + +/** + * Content score a folder needs before it counts as a confident match. The + * analysis step already treats this as a confident folder match + * (THRESHOLDS.FOLDER_MATCH_CONFIDENCE), and Hybrid routing asks the text model + * for help below it (hybridLLMThreshold). The engine's own floor for listing + * a semantic candidate at all is 0.4, which is where an unrelated file could + * still pick up a folder. + */ +const CONFIDENT_CONTENT_SCORE = THRESHOLDS.FOLDER_MATCH_CONFIDENCE ?? 0.55; + +/** + * Two folders whose content scores are closer than this cannot be told + * apart. Same value the analysis step uses for "no spread" between its top + * two folders (CONFIDENCE_CALIBRATION.NO_SPREAD_THRESHOLD), where it caps the + * file's confidence for the same reason. + */ +const AMBIGUOUS_MARGIN = THRESHOLDS.CONFIDENCE_CALIBRATION?.NO_SPREAD_THRESHOLD ?? 0.03; + +/** + * A folder chosen by name alone, with no ranked evidence of its own, is + * capped below the auto-organize default (0.75): it routes the file in + * Organize, but it never moves a file unattended. + */ +const NAME_ONLY_MAX_CONFIDENCE = 0.7; + +const ROUTING_BASIS = Object.freeze({ + LEARNED: 'learned', + CONTENT: 'content_match', + NAME: 'name_match', + NEEDS_REVIEW: 'needs_review' +}); + +const REVIEW_REASONS = Object.freeze({ + NO_MATCH: 'no_match', + WEAK_MATCH: 'weak_match', + AMBIGUOUS: 'ambiguous' +}); + +/** Analysis fields whose value can name a folder outright. */ +const NAME_FIELDS = ['category', 'vendor', 'organization', 'entity', 'sender', 'project']; + +/** + * @param {any} suggestion + * @returns {string} same smart folder → same key + */ +function folderKey(suggestion) { + if (suggestion?.folderId) return `id:${suggestion.folderId}`; + if (typeof suggestion?.path === 'string' && suggestion.path) { + return `path:${normalizePathForIndex(suggestion.path)}`; + } + return `name:${String(suggestion?.folder || suggestion?.name || '').toLowerCase()}`; +} + +/** + * @param {any} folder - smart folder record + * @returns {string[]} every key a suggestion for this folder may carry + */ +function folderKeys(folder) { + const keys = []; + if (folder?.id) keys.push(`id:${folder.id}`); + if (typeof folder?.path === 'string' && folder.path) { + keys.push(`path:${normalizePathForIndex(folder.path)}`); + } + keys.push(`name:${String(folder?.name || '').toLowerCase()}`); + return keys; +} + +/** + * @param {any} a - suggestion + * @param {any} b - suggestion or smart folder record + */ +function isSameFolder(a, b) { + if (!a || !b) return false; + const bId = b.folderId || b.id; + if (a.folderId && bId) return a.folderId === bId; + const bPath = typeof b.path === 'string' ? b.path : ''; + if (typeof a.path === 'string' && a.path && bPath) { + return normalizePathForIndex(a.path) === normalizePathForIndex(bPath); + } + return ( + String(a.folder || a.name || '').toLowerCase() === + String(b.folder || b.name || '').toLowerCase() + ); +} + +/** + * @param {any} suggestion + * @returns {boolean} the default catch-all (it is never a match on its own) + */ +function isDefaultFolderSuggestion(suggestion) { + return ( + suggestion?.source === 'default' || + suggestion?.isDefault === true || + isDefaultCategoryName(suggestion?.folder) + ); +} + +/** + * @param {any} suggestion + * @returns {boolean} a smart folder the engine could send the file to + */ +function isDestinationCandidate(suggestion) { + return Boolean( + suggestion && + suggestion.isSmartFolder && + !suggestion.unresolvedSmartFolder && + suggestion.source !== 'improvement' && + suggestion.folder && + !isDefaultFolderSuggestion(suggestion) + ); +} + +/** + * Best content score per folder, from the per-source suggestions before they + * are merged. Content evidence is the semantic match, the text model's own + * folder picks, and cluster membership. The analysis label (method + * llm_category) is not: its score is lifted to 0.95 whenever it names a + * folder, so it is judged by the name rule instead. + * + * @param {any[]} sourceSuggestions - canonicalized, source-tagged suggestions + * @returns {Map} + */ +function buildContentScores(sourceSuggestions) { + /** @type {Map} */ + const scores = new Map(); + for (const suggestion of Array.isArray(sourceSuggestions) ? sourceSuggestions : []) { + if (!isDestinationCandidate(suggestion)) continue; + const isContent = + suggestion.source === 'semantic' || + suggestion.source === 'cluster' || + (suggestion.source === 'llm' && suggestion.method !== 'llm_category'); + if (!isContent) continue; + const raw = Number(suggestion.score); + if (!Number.isFinite(raw)) continue; + const score = Math.max(0, Math.min(1, raw)); + const key = folderKey(suggestion); + scores.set(key, Math.max(scores.get(key) || 0, score)); + } + return scores; +} + +/** + * @param {Map} contentScores + * @param {any} suggestion + */ +function contentScoreOf(contentScores, suggestion) { + return contentScores.get(folderKey(suggestion)) || 0; +} + +/** + * @param {Map} contentScores + * @param {any} suggestion + * @returns {number} the best content score of any other folder + */ +function bestOtherContentScore(contentScores, suggestion) { + const own = folderKey(suggestion); + let best = 0; + for (const [key, score] of contentScores) { + if (key !== own && score > best) best = score; + } + return best; +} + +/** + * The folder the file names outright, if any: its extension, then its + * analysis label and entity fields, then its file type. + * + * The label and entity fields count only when `useLabels` is set, which the + * engine does in the routing modes that rank by the label (LLM-only, and + * Hybrid when embeddings are weak). Embedding routing leaves the label out on + * purpose: the analysis model is shown the folder list and told to answer + * with one of its names, so its label names a folder almost every time, + * including for files that fit none. + * + * @param {any} file + * @param {any[]} smartFolders + * @param {{ useLabels?: boolean }} [options] + * @returns {{ folder: any, kind: 'extension'|'label'|'type', field?: string, value?: string } | null} + */ +function findNameMatch(file, smartFolders, { useLabels = true } = {}) { + const folders = (Array.isArray(smartFolders) ? smartFolders : []).filter( + (f) => f?.name && !f.isDefault && !isDefaultCategoryName(f.name) + ); + if (folders.length === 0) return null; + + const extension = String(file?.extension || path.extname(file?.name || file?.path || '')) + .replace(/^\./, '') + .toLowerCase(); + + const claimed = extension ? findSmartFolderForExtension(extension, folders) : null; + if (claimed) return { folder: claimed, kind: 'extension', value: extension }; + + const analysis = file?.analysis && typeof file.analysis === 'object' ? file.analysis : {}; + // A fallback analysis guessed its label from the file name, and an + // embedding override replaced the model's label with the folder the + // content matched (judged by the content rule). Neither is a name. + const labelsUsable = + useLabels && !isFallbackAnalysis(analysis) && analysis.categorySource !== 'embedding_override'; + if (labelsUsable) { + for (const field of NAME_FIELDS) { + const value = analysis[field]; + if (typeof value !== 'string' || !value.trim()) continue; + const named = findSmartFolderNamedByLabel(value, folders); + if (named) return { folder: named, kind: 'label', field, value: value.trim() }; + } + } + + if (extension) { + const typeName = getFileTypeCategory(extension); + if (typeName && typeName !== 'Files') { + const typed = findSmartFolderNamedByLabel(typeName, folders); + if (typed) return { folder: typed, kind: 'type', value: typeName }; + } + } + return null; +} + +/** + * @param {any} candidate + * @param {Set} learnedKeys + * @returns {boolean} + */ +function isLearned(candidate, learnedKeys) { + if (!candidate) return false; + if (candidate.method === 'learned_correction' || candidate.source === 'user_pattern') { + return true; + } + const memory = candidate.memoryAdjustment; + if (memory && Number(memory.boost) > Number(memory.penalty || 0)) return true; + return learnedKeys.has(folderKey(candidate)); +} + +/** + * @typedef {Object} RoutingDecision + * @property {boolean} needsReview + * @property {string} basis - one of ROUTING_BASIS + * @property {string|null} reason - one of REVIEW_REASONS when needsReview + * @property {any|null} candidate - the suggestion to route to (null when needsReview) + * @property {{ kind: string, field?: string, value?: string }|null} nameMatch + * @property {{ folder: string, score: number }|null} best - best content match + * @property {{ folder: string, score: number }|null} runnerUp - next folder by content + */ + +/** + * Decide where a file goes, or that it needs review. + * + * @param {Object} params + * @param {any} params.file + * @param {any[]} params.ranked - final ranked suggestions (after memory and learned corrections) + * @param {any[]} params.sourceSuggestions - canonicalized per-source suggestions, before merging + * @param {any[]} [params.patternMatches] - learned user-pattern suggestions + * @param {any[]} params.smartFolders + * @param {boolean} [params.useLabels] - the routing mode ranks by the analysis label + * @returns {RoutingDecision} + */ +function decideRouting({ + file, + ranked, + sourceSuggestions, + patternMatches = [], + smartFolders, + useLabels = true +}) { + const candidates = (Array.isArray(ranked) ? ranked : []).filter(isDestinationCandidate); + const contentScores = buildContentScores(sourceSuggestions); + const learnedKeys = new Set( + (Array.isArray(patternMatches) ? patternMatches : []) + .filter((match) => match?.method === 'user_pattern' && match.folder) + .map((match) => { + const folder = (Array.isArray(smartFolders) ? smartFolders : []).find( + (f) => f?.name && String(f.name).toLowerCase() === String(match.folder).toLowerCase() + ); + return folder ? folderKeys(folder) : [folderKey(match)]; + }) + .flat() + ); + const nameMatch = findNameMatch(file, smartFolders, { useLabels }); + + /** @param {any} candidate */ + const isContentConfident = (candidate) => { + const score = contentScoreOf(contentScores, candidate); + return ( + score >= CONFIDENT_CONTENT_SCORE && + score - bestOtherContentScore(contentScores, candidate) >= AMBIGUOUS_MARGIN + ); + }; + /** @param {any} candidate */ + const isNamed = (candidate) => Boolean(nameMatch && isSameFolder(candidate, nameMatch.folder)); + + // Content summary for logs and the review message. + /** @type {{ key: string, score: number }[]} */ + const byContent = [...contentScores.entries()] + .map(([key, score]) => ({ key, score })) + .sort((a, b) => b.score - a.score); + /** @param {{ key: string, score: number }|undefined} entry */ + const describe = (entry) => { + if (!entry) return null; + const match = candidates.find((c) => folderKey(c) === entry.key); + return match ? { folder: match.folder, score: Math.round(entry.score * 100) / 100 } : null; + }; + const best = describe(byContent[0]); + const runnerUp = describe(byContent[1]); + + /** + * @param {any} candidate + * @param {string} basis + * @returns {RoutingDecision} + */ + const route = (candidate, basis) => ({ + needsReview: false, + basis, + reason: null, + candidate, + nameMatch: basis === ROUTING_BASIS.NAME && nameMatch ? stripFolder(nameMatch) : null, + best, + runnerUp + }); + + const top = candidates[0] || null; + if (top) { + // The folder already in front keeps its place whenever it is confident, + // so confident matches and learned corrections route as before. + if (isLearned(top, learnedKeys)) return route(top, ROUTING_BASIS.LEARNED); + if (isContentConfident(top)) return route(top, ROUTING_BASIS.CONTENT); + if (isNamed(top)) return route(top, ROUTING_BASIS.NAME); + } + + const learned = candidates.find((candidate) => isLearned(candidate, learnedKeys)); + if (learned) return route(learned, ROUTING_BASIS.LEARNED); + + const contentBest = byContent[0] + ? candidates.find((candidate) => folderKey(candidate) === byContent[0].key) + : null; + if (contentBest && isContentConfident(contentBest)) { + return route(contentBest, ROUTING_BASIS.CONTENT); + } + + if (nameMatch) { + const rankedNamed = candidates.find(isNamed); + return route(rankedNamed || buildNameCandidate(nameMatch, file), ROUTING_BASIS.NAME); + } + + const topContent = byContent[0]?.score || 0; + /** @type {string} */ + let reason = REVIEW_REASONS.NO_MATCH; + if (topContent >= CONFIDENT_CONTENT_SCORE) reason = REVIEW_REASONS.AMBIGUOUS; + else if (candidates.length > 0) reason = REVIEW_REASONS.WEAK_MATCH; + return { + needsReview: true, + basis: ROUTING_BASIS.NEEDS_REVIEW, + reason, + candidate: null, + nameMatch: null, + best: best || (top ? { folder: top.folder, score: 0 } : null), + runnerUp + }; +} + +/** + * @param {{ folder: any, kind: string, field?: string, value?: string }} nameMatch + */ +function stripFolder(nameMatch) { + const { kind, field, value } = nameMatch; + return { kind, ...(field ? { field } : {}), ...(value ? { value } : {}) }; +} + +/** + * A suggestion for a folder chosen by name alone. Its confidence is the + * analysis's own, capped at NAME_ONLY_MAX_CONFIDENCE. + * + * @param {{ folder: any, kind: string }} nameMatch + * @param {any} file + */ +function buildNameCandidate(nameMatch, file) { + const folder = nameMatch.folder; + const raw = Number(file?.analysis?.confidence); + const own = Number.isFinite(raw) ? (raw > 1 ? raw / 100 : raw) : NAME_ONLY_MAX_CONFIDENCE; + const confidence = Math.max(0, Math.min(NAME_ONLY_MAX_CONFIDENCE, own)); + return { + folder: folder.name, + path: folder.path, + folderId: folder.id || null, + description: folder.description || '', + score: confidence, + confidence, + method: 'name_match', + source: 'name', + isSmartFolder: true + }; +} + +module.exports = { + CONFIDENT_CONTENT_SCORE, + AMBIGUOUS_MARGIN, + NAME_ONLY_MAX_CONFIDENCE, + ROUTING_BASIS, + REVIEW_REASONS, + decideRouting, + findNameMatch, + isDestinationCandidate +}; diff --git a/src/main/services/organization/suggestionRanker.js b/src/main/services/organization/suggestionRanker.js index f62f51d7..094eb272 100644 --- a/src/main/services/organization/suggestionRanker.js +++ b/src/main/services/organization/suggestionRanker.js @@ -153,6 +153,25 @@ function generateExplanation(suggestion, file) { return 'No clear match found. Consider creating a new folder.'; } + // No Smart Folder was a confident match (routingDecision.js): say why, + // not what the default folder is for. + if (suggestion.needsReview) { + return explainReview(suggestion.review); + } + + // Routed because the file names the folder outright. + if (suggestion.nameMatch) { + const { kind, field, value } = suggestion.nameMatch; + if (kind === 'extension') { + return `"${suggestion.folder}" is the folder for ${String(value || '').toUpperCase()} files`; + } + if (kind === 'type') return `"${suggestion.folder}" matches this file's type`; + if (kind === 'label' && field && field !== 'category') { + return `"${suggestion.folder}" matches this file's ${field}`; + } + return `The analysis labeled this file "${suggestion.folder}"`; + } + // The user's own corrections overrode the folder that ranked first (item 16). // Checked before the source: the folder keeps the source of its evidence // (usually user_pattern), but the reason it is first is the corrections. @@ -201,6 +220,25 @@ function generateExplanation(suggestion, file) { ); } +/** + * Why a file needs review, in the words the Organize card shows. + * + * @param {{ reason?: string|null, best?: { folder: string }|null, + * runnerUp?: { folder: string }|null }|null|undefined} review + * @returns {string} + */ +function explainReview(review) { + const best = review?.best?.folder; + const runnerUp = review?.runnerUp?.folder; + if (review?.reason === 'ambiguous' && best && runnerUp) { + return `"${best}" and "${runnerUp}" matched this file about equally.`; + } + if (review?.reason === 'weak_match' && best) { + return `No Smart Folder matched this file well. The closest was "${best}".`; + } + return 'No Smart Folder matched this file.'; +} + /** * Combine suggestions from multiple sources * @param {Record} sources - Object with suggestion arrays by source @@ -225,5 +263,6 @@ module.exports = { calculateConfidence, capSuggestionConfidence, generateExplanation, + explainReview, combineSuggestions }; diff --git a/src/renderer/components/organize/ReadyFileItem.jsx b/src/renderer/components/organize/ReadyFileItem.jsx index 03d1a3c0..855f9db1 100644 --- a/src/renderer/components/organize/ReadyFileItem.jsx +++ b/src/renderer/components/organize/ReadyFileItem.jsx @@ -1,7 +1,7 @@ import React, { memo, useCallback, useMemo } from 'react'; import PropTypes from 'prop-types'; import { useSelector } from 'react-redux'; -import { FileText, ChevronRight, Info, FolderPlus } from 'lucide-react'; +import { AlertCircle, FileText, ChevronRight, Info, FolderPlus } from 'lucide-react'; import Input from '../ui/Input'; import Select from '../ui/Select'; import { Button, StatusBadge } from '../ui'; @@ -73,6 +73,8 @@ function ReadyFileItem({ // it is about this folder), not the LLM's confidence in its own label. const engine = routing?.engine || null; const routedByEngine = routing?.source === 'engine' && engine; + // No Smart Folder is a confident match: the file waits for the user. + const needsReview = routing?.source === 'review'; const rawConfidence = routedByEngine ? engine.confidence : analysis?.confidence; const confidenceValue = typeof rawConfidence === 'number' && Number.isFinite(rawConfidence) && rawConfidence >= 0 @@ -102,11 +104,14 @@ function ReadyFileItem({ }, [engine, selectableFolders]); const routingNote = useMemo(() => { if (!routing) return null; + if (routing.source === 'review') { + return engine?.explanation || 'No Smart Folder matched this file.'; + } if (routing.source === 'engine' && engine) { return engine.explanation || 'Ranked by what the file contains'; } if (routing.source === 'user') { - return engine?.folder && engine.folder !== routing.folder?.name + return engine?.folder && !engine.needsReview && engine.folder !== routing.folder?.name ? `Your choice (suggested: ${engine.folder})` : 'Your choice'; } @@ -128,7 +133,7 @@ function ReadyFileItem({ // same type and label, and only after a few (the engine weighs it against // what the file contains). Say so, so the user knows what to expect. const learningNote = useMemo(() => { - if (routing?.source !== 'user' || !engine?.folder) return null; + if (routing?.source !== 'user' || (!engine?.folder && !engine?.needsReview)) return null; const chosen = routing.folder?.name; if (!chosen || engine.folder === chosen) return null; const dot = String(file?.name || '').lastIndexOf('.'); @@ -148,7 +153,12 @@ function ReadyFileItem({ computedConfidence !== null && routing?.source !== 'name' && routing?.source !== 'user' && + !needsReview && !(routedByEngine && engine?.source === 'default'); + // Where the file goes if the user does not pick a folder. + const reviewAction = needsReview + ? `It goes to "${routing?.folder?.name || category || 'Uncategorized'}" unless you pick a folder.` + : null; const fileKey = toCanonicalPath(file?.path) || file?.path || ''; const handleToggle = useCallback(() => onToggleSelected(fileKey), [onToggleSelected, fileKey]); const handleEditName = useCallback( @@ -257,10 +267,12 @@ function ReadyFileItem({ flattened, fragments are not), so the groups go in an array. */} {rankedNames.length > 0 ? [ - + // A file that needs review has no best match, only the + // folders that came closest. + {rankedNames.map((name, rank) => ( ))} , @@ -280,14 +292,43 @@ function ReadyFileItem({ ))} - {routingNote && ( - - {routingNote} - + +
+ + Needs review + + + {routingNote} + + + {reviewAction} + +
+ + ) : ( + routingNote && ( + + {routingNote} + + ) )} {learningNote && ( & { folder: string|null, path: string|null } & EngineSuggestionExtra} EngineSuggestion */ /** * @typedef {Object} OrganizeDestination * @property {any|null} folder - the smart folder, or null for "use Uncategorized" - * @property {'user'|'engine'|'name'|'none'} source + * @property {'user'|'engine'|'name'|'review'|'none'} source - 'review': the + * engine found no confident match; `folder` is the default folder or null * @property {EngineSuggestion|null} engine - the engine's ranking for this file, if any */ @@ -73,10 +88,12 @@ function toCandidate(raw) { * @returns {EngineSuggestion|null} */ export function normalizeEngineEntry(entry) { + const needsReview = entry?.needsReview === true || entry?.suggestion?.needsReview === true; const primary = toCandidate(entry?.suggestion); - if (!primary) return null; - const confidence = toUnitConfidence(entry?.confidence) ?? primary.confidence; - const seen = new Set([toCanonicalPath(primary.path)]); + // A file that needs review may have no folder at all (no default folder). + if (!primary && !needsReview) return null; + const confidence = toUnitConfidence(entry?.confidence) ?? primary?.confidence ?? null; + const seen = new Set(primary ? [toCanonicalPath(primary.path)] : []); /** @type {EngineCandidate[]} */ const alternatives = []; for (const raw of Array.isArray(entry?.alternatives) ? entry.alternatives : []) { @@ -87,14 +104,16 @@ export function normalizeEngineEntry(entry) { seen.add(key); alternatives.push(candidate); } + const reviewReason = entry?.routingDecision?.reason || entry?.suggestion?.review?.reason || null; return { - ...primary, + ...(primary || { folder: null, path: null, folderId: null, source: null }), confidence, explanation: typeof entry?.explanation === 'string' && entry.explanation.trim() ? entry.explanation.trim() : null, - alternatives + alternatives, + ...(needsReview ? { needsReview: true, reviewReason } : {}) }; } @@ -102,7 +121,7 @@ export function normalizeEngineEntry(entry) { * The live smart folder an engine candidate names, matched by id and then by * path. A candidate for a folder that has since been removed resolves to null. * - * @param {EngineCandidate|null|undefined} candidate + * @param {{ folderId?: string|null, path?: string|null }|null|undefined} candidate * @param {any[]} smartFolders * @returns {any|null} */ @@ -145,6 +164,9 @@ export function resolveOrganizeDestination({ } const engineFolder = findFolderForCandidate(engine, smartFolders); + if (engine?.needsReview) { + return { folder: engineFolder, source: 'review', engine }; + } if (engineFolder && !isGenericBucketFolder(engineFolder)) { return { folder: engineFolder, source: 'engine', engine }; } @@ -174,6 +196,8 @@ export function resolveOrganizeDestination({ * @returns {{ folder: string, path?: string, source: string } | undefined} */ export function suggestionForOperation(destination, smartFolders) { + // A file that needs review had no engine pick to correct. + if (destination?.engine?.needsReview) return undefined; const engineFolder = findFolderForCandidate(destination?.engine, smartFolders); if (!engineFolder || isGenericBucketFolder(engineFolder)) return undefined; return { diff --git a/src/renderer/phases/organize/useOrganizeSuggestions.js b/src/renderer/phases/organize/useOrganizeSuggestions.js index 99235401..f79abaa6 100644 --- a/src/renderer/phases/organize/useOrganizeSuggestions.js +++ b/src/renderer/phases/organize/useOrganizeSuggestions.js @@ -8,8 +8,9 @@ * embeddings, learned placements and feedback memory in seconds rather than * one LLM call per file. Each path is fetched once per smart-folder set: a * later edit on the card does not refetch, so the ranking stays what the app - * suggested before the user chose. Files the engine cannot place, and any - * failed chunk, fall back to name matching in `resolveOrganizeDestination`. + * suggested before the user chose. Files the engine has no answer for, and + * any failed chunk, fall back to name matching in `resolveOrganizeDestination`. + * Files it marks as needing review do not: they wait for the user. * * @module organize/useOrganizeSuggestions */ @@ -154,12 +155,17 @@ export function useOrganizeSuggestions(files, smartFolders) { /** @type {Map} */ const updates = new Map(); - for (const group of response.groups) { - for (const entry of Array.isArray(group?.files) ? group.files : []) { - const key = toCanonicalPath(entry?.path); - const normalized = normalizeEngineEntry(entry); - if (key && normalized) updates.set(key, normalized); - } + const entries = [ + ...response.groups.flatMap((/** @type {any} */ group) => + Array.isArray(group?.files) ? group.files : [] + ), + // Files that need review with no default folder to hold them. + ...(Array.isArray(response.unassigned) ? response.unassigned : []) + ]; + for (const entry of entries) { + const key = toCanonicalPath(entry?.path); + const normalized = normalizeEngineEntry(entry); + if (key && normalized) updates.set(key, normalized); } if (updates.size > 0) { setSuggestionsByPath((prev) => { diff --git a/src/shared/categoryFolderMatching.js b/src/shared/categoryFolderMatching.js index 76d212b5..e2680b7f 100644 --- a/src/shared/categoryFolderMatching.js +++ b/src/shared/categoryFolderMatching.js @@ -387,6 +387,67 @@ function findSmartFolderForLabel(label, smartFolders) { return null; } +/** + * The one smart folder a label names outright: the same words, ignoring + * case, a plural "s", and spaces, dashes or underscores; or every word of the + * folder name in the label; or, for a label of two words or more, every word + * of the label in the folder name. Description and keyword overlap do not + * count, and neither does a one-word label that is only part of a folder + * name ("Data" is not "SEO data"). A label that fits more than one folder + * ("SEO reports" for "SEO" and "Reports") names none. The default catch-all + * never counts: "Uncategorized" means no folder fit. + * + * Used by needs-review routing, where only an exact or near-exact name is + * evidence on its own; `findSmartFolderForLabel` stays the broader matcher. + * + * @param {string} label + * @param {any[]} smartFolders + * @returns {any|null} + */ +function findSmartFolderNamedByLabel(label, smartFolders) { + const folders = (Array.isArray(smartFolders) ? smartFolders : []).filter( + (f) => + f && + typeof f.name === 'string' && + f.name.trim() && + !f.isDefault && + !isDefaultCategoryName(f.name) + ); + const raw = String(label || '').trim(); + if (!raw || folders.length === 0 || isDefaultCategoryName(raw)) return null; + + const seen = new Set(); + for (const candidate of [raw, stripCategorySubjectSuffix(raw)]) { + const key = candidate.toLowerCase(); + if (!candidate || seen.has(key)) continue; + seen.add(key); + + const variants = new Set(buildLabelVariants(candidate)); + const candidateCanon = canon(candidate); + const exact = folders.filter( + (folder) => + buildLabelVariants(folder.name).some((variant) => variants.has(variant)) || + (candidateCanon && canon(folder.name) === candidateCanon) + ); + if (exact.length === 1) return exact[0]; + if (exact.length > 1) return null; + + const labelWords = new Set(candidateCanon.split(/\s+/).filter(Boolean).map(stemToken)); + if (labelWords.size === 0) continue; + const near = folders.filter((folder) => { + const folderWords = new Set(canon(folder.name).split(/\s+/).filter(Boolean).map(stemToken)); + if (folderWords.size === 0) return false; + const folderInLabel = [...folderWords].every((word) => labelWords.has(word)); + const labelInFolder = + labelWords.size >= 2 && [...labelWords].every((word) => folderWords.has(word)); + return folderInLabel || labelInFolder; + }); + if (near.length === 1) return near[0]; + if (near.length > 1) return null; + } + return null; +} + /** * @param {string} raw * @param {any[]} folders @@ -691,6 +752,7 @@ module.exports = { scoreLabelAgainstFolderSemantics, findBestSemanticFolder, findSmartFolderForLabel, + findSmartFolderNamedByLabel, findSmartFolderForExtension, findSmartFolderForFile, isGenericBucketFolder diff --git a/test/OrganizationSuggestionService.test.js b/test/OrganizationSuggestionService.test.js index ee6abeaf..08f911d0 100644 --- a/test/OrganizationSuggestionService.test.js +++ b/test/OrganizationSuggestionService.test.js @@ -227,11 +227,11 @@ describe('OrganizationSuggestionService', () => { const result = await service.getSuggestionsForFile(file, smartFolders); expect(result.success).toBe(true); - expect(result.primary).toBeDefined(); - expect(result.primary.folder).toBeDefined(); - // With no smart folders, primary may be a non-smart suggestion for review - expect(result.primary.isSmartFolder).toBeFalsy(); - expect(result.confidence).toBeLessThanOrEqual(1.0); + // No folder and no default folder: the file stays unassigned and + // needs review rather than taking a non-folder suggestion. + expect(result.primary).toBeNull(); + expect(result.needsReview).toBe(true); + expect(result.confidence).toBe(0); // Verify no folder embeddings were attempted expect(mockFolderMatchingService.upsertFolderEmbedding).not.toHaveBeenCalled(); @@ -370,7 +370,8 @@ describe('OrganizationSuggestionService', () => { { folderId: 'folder-3', name: 'Semantic Match', - score: 0.5, // Lower score + // A confident content match, so a smart folder is the primary + score: 0.7, path: '/docs/Semantic' } ]); @@ -842,9 +843,9 @@ describe('OrganizationSuggestionService', () => { const result = await service.getSuggestionsForFile(file, []); expect(result.success).toBe(true); - expect(result.primary).toBeDefined(); - // With no smart folders, primary may be a non-smart suggestion for review - expect(result.primary.isSmartFolder).toBeFalsy(); + // No folder and no default folder: unassigned, needs review. + expect(result.primary).toBeNull(); + expect(result.needsReview).toBe(true); }); test('should handle concurrent calls without interference', async () => { diff --git a/test/OrganizationSuggestionServiceCore.labelScore.test.js b/test/OrganizationSuggestionServiceCore.labelScore.test.js index db3963df..f3d2cea3 100644 --- a/test/OrganizationSuggestionServiceCore.labelScore.test.js +++ b/test/OrganizationSuggestionServiceCore.labelScore.test.js @@ -115,20 +115,35 @@ describe('the analysis label does not reach embedding ranking through a strategy expect(result.primary.folder).toBe('Uncategorized'); expect(result.primary.source).toBe('default'); - expect(result.explanation).toBe('No Smart Folder matched what this file contains'); + expect(result.needsReview).toBe(true); + expect(result.explanation).toBe('No Smart Folder matched this file.'); const sources = [result.primary, ...result.alternatives].map((s) => s?.source); expect(sources).not.toContain('strategy'); }); test('a content match shows its own score, not a flat 60%', async () => { const service = makeService([ - { folderId: 'folder-1', name: 'Invoices', path: '/docs/Invoices', score: 0.52 } + { folderId: 'folder-1', name: 'Invoices', path: '/docs/Invoices', score: 0.62 } ]); const result = await service.getSuggestionsForFile(makeFile(), SMART_FOLDERS, INTERACTIVE); expect(result.primary.folder).toBe('Invoices'); expect(result.primary.source).toBe('semantic'); - expect(result.confidence).toBe(0.52); + expect(result.confidence).toBe(0.62); + }); + + test('a weak content match is not routed even when the label names the folder', async () => { + const service = makeService([ + { folderId: 'folder-1', name: 'Invoices', path: '/docs/Invoices', score: 0.52 } + ]); + const result = await service.getSuggestionsForFile(makeFile(), SMART_FOLDERS, INTERACTIVE); + + expect(result.needsReview).toBe(true); + expect(result.primary.folder).toBe('Uncategorized'); + expect(result.explanation).toBe( + 'No Smart Folder matched this file well. The closest was "Invoices".' + ); + expect(result.alternatives.map((s) => s.folder)).toContain('Invoices'); }); test('LLM routing still uses the label', async () => { diff --git a/test/OrganizationSuggestionServiceCore.needsReview.test.js b/test/OrganizationSuggestionServiceCore.needsReview.test.js new file mode 100644 index 00000000..d3f1ff00 --- /dev/null +++ b/test/OrganizationSuggestionServiceCore.needsReview.test.js @@ -0,0 +1,380 @@ +/** + * Needs-review routing (routingDecision.js). + * + * A Markdown report about local AI models was suggested into an "SEO data" + * folder: every ranked candidate clears a 0.4 semantic floor, so a file that + * fits no folder still went to whichever folder came out on top. A file now + * goes to a Smart Folder only on a confident match (a learned pattern, a + * content score of 0.55 or more that no other folder comes within 0.03 of, + * or a name the file carries outright). Otherwise it needs review: it goes + * to the default folder, or stays unassigned when there is none. + */ + +const mockLlamaService = { + initialize: jest.fn().mockResolvedValue(undefined), + getConfig: jest.fn().mockReturnValue({ textModel: 'test-model.gguf' }), + generateText: jest.fn().mockResolvedValue({ response: '{"suggestions":[]}' }) +}; + +jest.mock('../src/main/services/LlamaService', () => ({ + getInstance: () => mockLlamaService +})); +jest.mock('../src/main/services/PerformanceService', () => ({ + getRecommendedConcurrency: jest.fn().mockResolvedValue({ + maxConcurrent: 1, + embeddingConcurrency: 3 + }) +})); + +const OrganizationSuggestionService = require('../src/main/services/organization'); +const { + CONFIDENT_CONTENT_SCORE, + AMBIGUOUS_MARGIN +} = require('../src/main/services/organization/routingDecision'); + +jest.setTimeout(30000); + +const SEO_DATA = { + id: 'seo-data', + name: 'SEO data', + path: '/Users/alex/Library/SEO data', + description: 'Keyword rankings and search traffic exports.' +}; +const SEO_AUDITS = { + id: 'seo-audits', + name: 'SEO audits', + path: '/Users/alex/Library/SEO audits', + description: 'PDF SEO audit reports with site scores, backlinks and recommendations.' +}; +const INVOICES = { + id: 'invoices', + name: 'Invoices', + path: '/Users/alex/Library/Invoices', + description: 'Invoices and receipts from vendors.' +}; +const RECEIPTS = { + id: 'receipts', + name: 'Receipts', + path: '/Users/alex/Library/Receipts', + description: 'Store and card receipts.' +}; +const PRINTS = { + id: 'prints', + name: '3D Prints', + path: '/Users/alex/Library/3D Prints', + description: 'Models to print.', + keywords: ['stl'] +}; +const UNCATEGORIZED = { + id: 'uncategorized', + name: 'Uncategorized', + path: '/Users/alex/Library/Uncategorized', + isDefault: true +}; + +const FOLDERS = [SEO_DATA, SEO_AUDITS, INVOICES, RECEIPTS, PRINTS, UNCATEGORIZED]; +const FOLDERS_NO_DEFAULT = FOLDERS.filter((f) => f !== UNCATEGORIZED); + +/** @param {{ id: string, name: string, path: string }} folder @param {number} score */ +const match = (folder, score) => ({ + folderId: folder.id, + name: folder.name, + path: folder.path, + score +}); + +/** The report from the original case: a Markdown file about local AI. */ +function localAiReport(overrides = {}) { + return { + name: 'state-of-local-ai.md', + extension: 'md', + path: '/Users/alex/Downloads/state-of-local-ai.md', + analysis: { + // The analysis model is told to answer with a folder name, so it named one. + category: 'SEO data', + date: '2026-10-01', + time_period: '2026', + keywords: ['local models', 'inference', 'privacy'], + purpose: 'Survey of running language models on your own computer', + confidence: 70 + }, + ...overrides + }; +} + +function makeService({ folderMatches = [], mode = 'embedding' } = {}) { + const vectorDb = { + initialize: jest.fn().mockResolvedValue(true), + isServerAvailable: jest.fn().mockResolvedValue(true), + getStats: jest.fn().mockResolvedValue({ files: 120, fileChunks: 0, folders: 6 }), + upsertFolder: jest.fn().mockResolvedValue({ success: true }), + upsertFile: jest.fn().mockResolvedValue({ success: true }), + queryFeedbackMemory: jest.fn().mockResolvedValue([]), + queryFolders: jest.fn().mockResolvedValue([]), + queryFiles: jest.fn().mockResolvedValue([]) + }; + const folderMatching = { + upsertFolderEmbedding: jest.fn().mockResolvedValue({ success: true }), + upsertFileEmbedding: jest.fn().mockResolvedValue({ success: true }), + matchVectorToFolders: jest.fn().mockResolvedValue(folderMatches), + matchFileToFolders: jest.fn().mockResolvedValue([]), + embedText: jest.fn().mockResolvedValue({ vector: new Array(1024).fill(0.1) }) + }; + const settings = { + get: jest.fn(() => null), + load: jest.fn().mockResolvedValue({ smartFolderRoutingMode: mode }), + save: jest.fn().mockResolvedValue({ success: true }) + }; + const service = new OrganizationSuggestionService({ + vectorDbService: vectorDb, + folderMatchingService: folderMatching, + settingsService: settings + }); + service.userPatterns = new Map(); + service.feedbackHistory = []; + service.folderUsageStats = new Map(); + return service; +} + +/** The options the Organize grid ranks with (useOrganizeSuggestions). */ +const INTERACTIVE = { + skipSmartFolderEmbeddings: true, + skipGeneration: true, + skipImprovementCandidates: true, + includeStructureAnalysis: false +}; + +const suggest = (service, file, folders = FOLDERS, mode = 'embedding') => + service.getSuggestionsForFile(file, folders, { ...INTERACTIVE, routingModeOverride: mode }); + +describe('thresholds', () => { + test('come from the analysis step scale', () => { + expect(CONFIDENT_CONTENT_SCORE).toBe(0.55); + expect(AMBIGUOUS_MARGIN).toBe(0.03); + }); +}); + +describe('a file that fits no folder', () => { + test('an unrelated Markdown report is not routed by a 0.4-level match', async () => { + const service = makeService({ + folderMatches: [match(SEO_AUDITS, 0.45), match(SEO_DATA, 0.42)] + }); + const result = await suggest(service, localAiReport()); + + expect(result.success).toBe(true); + expect(result.needsReview).toBe(true); + expect(result.routingDecision.reason).toBe('weak_match'); + expect(result.primary.folder).toBe('Uncategorized'); + expect(result.primary.needsReview).toBe(true); + expect(result.explanation).toBe( + 'No Smart Folder matched this file well. The closest was "SEO audits".' + ); + // Still listed, so the user can pick one by hand. + expect(result.alternatives.map((s) => s.folder)).toEqual( + expect.arrayContaining(['SEO audits', 'SEO data']) + ); + // Not a confidence anyone could auto-move on. + expect(result.confidence).toBeLessThan(0.3); + }); + + test('stays unassigned when there is no default folder', async () => { + const service = makeService({ + folderMatches: [match(SEO_AUDITS, 0.45), match(SEO_DATA, 0.42)] + }); + const result = await suggest(service, localAiReport(), FOLDERS_NO_DEFAULT); + + expect(result.success).toBe(true); + expect(result.needsReview).toBe(true); + expect(result.primary).toBeNull(); + expect(result.confidence).toBe(0); + expect(result.alternatives.map((s) => s.folder)).toContain('SEO audits'); + }); + + test('just below the confident score needs review; at it, the file routes', async () => { + const below = makeService({ folderMatches: [match(INVOICES, 0.54)] }); + const at = makeService({ folderMatches: [match(INVOICES, 0.55)] }); + const file = localAiReport({ analysis: { category: 'Invoices', confidence: 80 } }); + + expect((await suggest(below, file)).needsReview).toBe(true); + const routed = await suggest(at, file); + expect(routed.needsReview).toBe(false); + expect(routed.primary.folder).toBe('Invoices'); + }); + + test('the date template does not choose the folder or the explanation', async () => { + // A date and a time period scored the date template 0.6, and with no + // label its folder was its own default, Uncategorized, explained as + // "Using Date-Based organization method". + const service = makeService(); + const file = localAiReport({ + analysis: { date: '2026-10-01', time_period: '2026', confidence: 70 } + }); + const result = await suggest(service, file); + + expect(result.needsReview).toBe(true); + expect(result.routingDecision.reason).toBe('no_match'); + expect(result.explanation).toBe('No Smart Folder matched this file.'); + expect(result.explanation).not.toMatch(/Date-Based/); + const sources = [result.primary, ...result.alternatives].map((s) => s?.source); + expect(sources).not.toContain('strategy'); + }); +}); + +describe('an empty file or a failed extraction', () => { + test('a filename guess does not route the file, even in LLM routing', async () => { + const service = makeService({ mode: 'llm' }); + const file = { + name: 'invoice-scan.pdf', + extension: 'pdf', + path: '/Users/alex/Downloads/invoice-scan.pdf', + analysis: { + category: 'Invoices', + isFallback: true, + extractionMethod: 'filename', + confidence: 30 + } + }; + const result = await suggest(service, file, FOLDERS, 'llm'); + + expect(result.needsReview).toBe(true); + expect(result.primary.folder).toBe('Uncategorized'); + expect(result.explanation).toBe( + 'No Smart Folder matched this file well. The closest was "Invoices".' + ); + }); + + test('an empty file with no analysis fields needs review', async () => { + const service = makeService(); + const file = { + name: 'empty.txt', + extension: 'txt', + path: '/Users/alex/Downloads/empty.txt', + analysis: { isFallback: true, extractionMethod: 'empty', confidence: 0 } + }; + const result = await suggest(service, file); + + expect(result.needsReview).toBe(true); + expect(result.routingDecision.reason).toBe('no_match'); + expect(result.primary.folder).toBe('Uncategorized'); + }); +}); + +describe('a file that fits two folders equally', () => { + test('is ambiguous and says which two', async () => { + const service = makeService({ + folderMatches: [match(SEO_DATA, 0.66), match(SEO_AUDITS, 0.65)] + }); + const result = await suggest(service, localAiReport()); + + expect(result.needsReview).toBe(true); + expect(result.routingDecision.reason).toBe('ambiguous'); + expect(result.primary.folder).toBe('Uncategorized'); + expect(result.explanation).toBe('"SEO data" and "SEO audits" matched this file about equally.'); + }); + + test('a clear gap between the two is not ambiguous', async () => { + const service = makeService({ + folderMatches: [match(SEO_DATA, 0.66), match(SEO_AUDITS, 0.62)] + }); + const result = await suggest(service, localAiReport()); + + expect(result.needsReview).toBe(false); + expect(result.primary.folder).toBe('SEO data'); + }); +}); + +describe('confident matches route as before', () => { + test('a strong content match goes to its folder with its own explanation', async () => { + const service = makeService({ + folderMatches: [match(INVOICES, 0.72), match(RECEIPTS, 0.5)] + }); + const file = { + name: 'march-invoice.pdf', + extension: 'pdf', + path: '/Users/alex/Downloads/march-invoice.pdf', + analysis: { category: 'Invoices', date: '2026-03-02', confidence: 90 } + }; + const result = await suggest(service, file); + + expect(result.needsReview).toBe(false); + expect(result.routingDecision.basis).toBe('content_match'); + expect(result.primary.folder).toBe('Invoices'); + expect(result.confidence).toBe(0.72); + expect(result.explanation).toBe('This file\'s meaning matches "Invoices"'); + }); + + test('a learned pattern routes a file whose content match is weak', async () => { + const service = makeService({ folderMatches: [match(SEO_DATA, 0.42)] }); + const file = localAiReport(); + const suggestion = { folder: SEO_DATA.name, path: SEO_DATA.path }; + service.userPatterns.set(service.patternMatcher.extractPattern(file, suggestion), { + ...suggestion, + count: 4, + confidence: 0.8, + lastUsed: Date.now() + }); + const result = await suggest(service, file); + + expect(result.needsReview).toBe(false); + expect(result.routingDecision.basis).toBe('learned'); + expect(result.primary.folder).toBe('SEO data'); + }); + + test('LLM routing still sends a label that names a folder there', async () => { + const service = makeService({ mode: 'llm' }); + const file = { + name: 'march-invoice.pdf', + extension: 'pdf', + path: '/Users/alex/Downloads/march-invoice.pdf', + analysis: { category: 'Invoices', confidence: 90 } + }; + const result = await suggest(service, file, FOLDERS, 'llm'); + + expect(result.needsReview).toBe(false); + expect(result.routingDecision.basis).toBe('name_match'); + expect(result.primary.folder).toBe('Invoices'); + }); + + test('a folder that claims the extension takes it, below the auto-move threshold', async () => { + const service = makeService(); + const file = { + name: 'bracket.stl', + extension: 'stl', + path: '/Users/alex/Downloads/bracket.stl', + analysis: { category: 'Uncategorized', confidence: 90 } + }; + const result = await suggest(service, file); + + expect(result.needsReview).toBe(false); + expect(result.primary.folder).toBe('3D Prints'); + expect(result.explanation).toBe('"3D Prints" is the folder for STL files'); + expect(result.confidence).toBeLessThan(0.75); + }); +}); + +describe('getBatchSuggestions', () => { + test('lists unassigned files and flags review files in their group', async () => { + const service = makeService({ + folderMatches: [match(SEO_AUDITS, 0.45), match(SEO_DATA, 0.42)] + }); + const withDefault = await service.getBatchSuggestions([localAiReport()], FOLDERS, { + ...INTERACTIVE + }); + const entry = withDefault.groups.flatMap((g) => g.files)[0]; + expect(entry.needsReview).toBe(true); + expect(entry.suggestion.folder).toBe('Uncategorized'); + expect(withDefault.unassigned).toEqual([]); + + const noDefault = await service.getBatchSuggestions([localAiReport()], FOLDERS_NO_DEFAULT, { + ...INTERACTIVE + }); + expect(noDefault.groups).toEqual([]); + expect(noDefault.unassigned).toHaveLength(1); + expect(noDefault.unassigned[0]).toMatchObject({ + path: '/Users/alex/Downloads/state-of-local-ai.md', + suggestion: null, + needsReview: true, + explanation: 'No Smart Folder matched this file well. The closest was "SEO audits".' + }); + }); +}); diff --git a/test/autoOrganize.needsReview.test.js b/test/autoOrganize.needsReview.test.js new file mode 100644 index 00000000..57c499af --- /dev/null +++ b/test/autoOrganize.needsReview.test.js @@ -0,0 +1,254 @@ +/** + * A file the suggestion engine marks as needing review (no Smart Folder is a + * confident match) is never moved by auto-organize or the download watcher + * path, whatever confidence it carries and whatever the threshold is set to. + * It is held where it is and listed for review. + */ + +jest.mock('../src/shared/logger', () => { + const logger = { + setContext: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + warn: jest.fn(), + error: jest.fn() + }; + return { logger, createLogger: jest.fn(() => logger) }; +}); + +jest.mock('electron', () => ({ + app: { getPath: jest.fn().mockReturnValue('/Users/alex/Documents') } +})); + +const mockFs = { + mkdir: jest.fn().mockResolvedValue(undefined), + lstat: jest.fn().mockRejectedValue({ code: 'ENOENT' }), + access: jest.fn().mockResolvedValue(undefined), + stat: jest.fn().mockResolvedValue({ size: 1000, birthtime: new Date(), mtime: new Date() }) +}; +jest.mock('fs', () => ({ promises: mockFs })); + +jest.mock('../src/main/analysis/documentAnalysis', () => ({ + analyzeDocumentFile: jest.fn().mockResolvedValue({ category: 'SEO data', confidence: 90 }) +})); +jest.mock('../src/main/analysis/imageAnalysis', () => ({ + analyzeImageFile: jest.fn().mockResolvedValue({ category: 'Photos', confidence: 90 }) +})); + +const { + processFilesIndividually, + processNewFile +} = require('../src/main/services/autoOrganize/fileProcessor'); +const { + processBatchResults, + batchOrganize +} = require('../src/main/services/autoOrganize/batchProcessor'); + +const UNCATEGORIZED = { + id: 'u', + name: 'Uncategorized', + path: '/Users/alex/Library/Uncategorized', + isDefault: true +}; +const SEO_DATA = { id: 's', name: 'SEO data', path: '/Users/alex/Library/SEO data' }; +const FOLDERS = [SEO_DATA, UNCATEGORIZED]; + +const REPORT = { + name: 'state-of-local-ai.md', + extension: 'md', + path: '/Users/alex/Downloads/state-of-local-ai.md', + analysis: { category: 'SEO data', confidence: 90 } +}; + +const EXPLANATION = 'No Smart Folder matched this file well. The closest was "SEO data".'; + +/** An engine answer that would clear a low threshold but needs review. */ +function reviewSuggestion() { + return { + success: true, + primary: { + folder: 'Uncategorized', + path: UNCATEGORIZED.path, + folderId: 'u', + isSmartFolder: true, + source: 'default', + needsReview: true + }, + alternatives: [{ folder: 'SEO data', path: SEO_DATA.path, isSmartFolder: true }], + confidence: 0.9, + explanation: EXPLANATION, + needsReview: true, + routingDecision: { basis: 'needs_review', reason: 'weak_match' } + }; +} + +const freshResults = () => ({ organized: [], needsReview: [], failed: [], operations: [] }); + +describe('processFilesIndividually', () => { + test('holds a needs-review file even above the threshold', async () => { + const results = freshResults(); + const service = { getSuggestionsForFile: jest.fn().mockResolvedValue(reviewSuggestion()) }; + + await processFilesIndividually( + [REPORT], + FOLDERS, + { confidenceThreshold: 0.1, defaultLocation: '/Users/alex/Library' }, + results, + service + ); + + expect(results.operations).toEqual([]); + expect(results.organized).toEqual([]); + expect(results.needsReview).toHaveLength(1); + expect(results.needsReview[0].explanation).toBe(EXPLANATION); + }); + + test('holds an unassigned file (no default folder) for review', async () => { + const results = freshResults(); + const suggestion = { ...reviewSuggestion(), primary: null, confidence: 0 }; + const service = { getSuggestionsForFile: jest.fn().mockResolvedValue(suggestion) }; + + await processFilesIndividually( + [REPORT], + [SEO_DATA], + { confidenceThreshold: 0, defaultLocation: '/Users/alex/Library' }, + results, + service + ); + + expect(results.operations).toEqual([]); + expect(results.needsReview).toHaveLength(1); + expect(results.needsReview[0].explanation).toBe(EXPLANATION); + }); +}); + +describe('processBatchResults', () => { + test('holds needs-review entries and unassigned files, moves confident ones', () => { + const results = freshResults(); + const confident = { + ...REPORT, + name: 'rankings.csv', + path: '/Users/alex/Downloads/rankings.csv' + }; + const orphan = { ...REPORT, name: 'notes.md', path: '/Users/alex/Downloads/notes.md' }; + const batch = { + success: true, + groups: [ + { + folder: 'Uncategorized', + path: UNCATEGORIZED.path, + confidence: 0.9, + files: [ + { + ...REPORT, + suggestion: reviewSuggestion().primary, + confidence: 0.9, + explanation: EXPLANATION, + needsReview: true + } + ] + }, + { + folder: 'SEO data', + path: SEO_DATA.path, + confidence: 0.9, + files: [ + { + ...confident, + suggestion: { folder: 'SEO data', path: SEO_DATA.path, isSmartFolder: true }, + confidence: 0.9, + needsReview: false + } + ] + } + ], + unassigned: [{ ...orphan, suggestion: null, needsReview: true, explanation: EXPLANATION }] + }; + + processBatchResults( + batch, + [REPORT, confident, orphan], + { confidenceThreshold: 0.5, defaultLocation: '/Users/alex/Library' }, + results, + FOLDERS + ); + + expect(results.operations.map((op) => op.source)).toEqual([confident.path]); + expect(results.needsReview.map((r) => r.file.path).sort()).toEqual( + [orphan.path, REPORT.path].sort() + ); + }); +}); + +describe('batchOrganize', () => { + test('never auto-approves a needs-review file inside a confident group', async () => { + const other = { ...REPORT, name: 'rankings.csv', path: '/Users/alex/Downloads/rankings.csv' }; + const service = { + getBatchSuggestions: jest.fn().mockResolvedValue({ + success: true, + groups: [ + { + folder: 'Uncategorized', + path: UNCATEGORIZED.path, + confidence: 0.9, + files: [ + { ...REPORT, needsReview: true }, + { ...other, needsReview: false } + ] + } + ], + unassigned: [{ ...REPORT, path: '/Users/alex/Downloads/notes.md', needsReview: true }] + }) + }; + + const result = await batchOrganize( + [REPORT, other], + FOLDERS, + { confidenceThreshold: 0.5, defaultLocation: '/Users/alex/Library' }, + service + ); + + expect(result.operations.map((op) => op.source)).toEqual([other.path]); + const held = result.skipped.filter((s) => s.reason === 'Needs review'); + expect(held.flatMap((s) => s.files.map((f) => f.path)).sort()).toEqual( + ['/Users/alex/Downloads/notes.md', REPORT.path].sort() + ); + }); +}); + +describe('processNewFile (download watcher)', () => { + test('does not move a needs-review file, and does not offer its label as the folder', async () => { + const service = { getSuggestionsForFile: jest.fn().mockResolvedValue(reviewSuggestion()) }; + + const result = await processNewFile( + '/Users/alex/Downloads/state-of-local-ai.md', + FOLDERS, + { autoOrganizeEnabled: true, confidenceThreshold: 0.1, defaultLocation: '/Users/alex' }, + service + ); + + expect(result.destination).toBeNull(); + expect(result.held).toBe(true); + expect(result.needsReview).toBe(true); + expect(result.explanation).toBe(EXPLANATION); + expect(result.suggestedFolder).toBe('Uncategorized'); + }); + + test('with no default folder the label is not offered either', async () => { + const service = { + getSuggestionsForFile: jest + .fn() + .mockResolvedValue({ ...reviewSuggestion(), primary: null, confidence: 0 }) + }; + + const result = await processNewFile( + '/Users/alex/Downloads/state-of-local-ai.md', + [SEO_DATA], + { autoOrganizeEnabled: true, confidenceThreshold: 0.1, defaultLocation: '/Users/alex' }, + service + ); + + expect(result.destination).toBeNull(); + expect(result.suggestedFolder).toBeNull(); + }); +}); diff --git a/test/categoryFolderMatching.namedByLabel.test.js b/test/categoryFolderMatching.namedByLabel.test.js new file mode 100644 index 00000000..39cad062 --- /dev/null +++ b/test/categoryFolderMatching.namedByLabel.test.js @@ -0,0 +1,51 @@ +/** + * findSmartFolderNamedByLabel: the exact or near-exact name match that + * needs-review routing accepts as evidence on its own. Description and + * keyword overlap, and a label that fits two folders, do not count. + */ + +const { + findSmartFolderNamedByLabel, + findSmartFolderForLabel +} = require('../src/shared/categoryFolderMatching'); + +const FOLDERS = [ + { name: 'SEO data', description: 'Keyword rankings and search traffic exports.' }, + { name: 'SEO audits', description: 'PDF SEO audit reports.' }, + { name: 'Invoices', description: 'Invoices, receipts and tax documents' }, + { name: 'Marketing Materials' }, + { name: 'Uncategorized', isDefault: true } +]; + +describe('findSmartFolderNamedByLabel', () => { + test.each([ + ['SEO data', 'SEO data'], + ['seo-data', 'SEO data'], + ['Invoice', 'Invoices'], + ['invoices', 'Invoices'], + ['Marketing Materials Q3', 'Marketing Materials'], + ['Nature_17803333378895', null] + ])('%s names %s', (label, expected) => { + const folder = findSmartFolderNamedByLabel(label, FOLDERS); + expect(folder ? folder.name : null).toBe(expected); + }); + + test('a label that fits two folders names neither', () => { + expect(findSmartFolderNamedByLabel('SEO', FOLDERS)).toBeNull(); + }); + + test('a one-word label that is only part of a folder name is not a name', () => { + expect(findSmartFolderNamedByLabel('Data', FOLDERS)).toBeNull(); + expect(findSmartFolderNamedByLabel('Marketing', FOLDERS)).toBeNull(); + }); + + test('description words are not a name, though the broader matcher uses them', () => { + expect(findSmartFolderForLabel('Tax documents', FOLDERS)?.name).toBe('Invoices'); + expect(findSmartFolderNamedByLabel('Tax documents', FOLDERS)).toBeNull(); + }); + + test('the default folder never counts', () => { + expect(findSmartFolderNamedByLabel('Uncategorized', FOLDERS)).toBeNull(); + expect(findSmartFolderNamedByLabel('Inbox', [{ name: 'Inbox', isDefault: true }])).toBeNull(); + }); +}); diff --git a/test/components/ReadyFileItem.routing.test.js b/test/components/ReadyFileItem.routing.test.js index bb4d3059..f4678bf7 100644 --- a/test/components/ReadyFileItem.routing.test.js +++ b/test/components/ReadyFileItem.routing.test.js @@ -104,6 +104,75 @@ test('a name match says so and keeps the plain folder list', () => { expect(options.map((o) => o.textContent)).toEqual(['Archive', 'Finance', 'Taxes']); }); +describe('a file that needs review', () => { + const UNCATEGORIZED = { id: 'unc', name: 'Uncategorized', path: '/L/Uncategorized' }; + const REVIEW_ENGINE = { + folder: 'Uncategorized', + path: '/L/Uncategorized', + folderId: 'unc', + confidence: 0.1, + source: 'default', + needsReview: true, + reviewReason: 'weak_match', + explanation: 'No Smart Folder matched this file well. The closest was "Finance".', + alternatives: [ + { folder: 'Finance', path: '/L/Finance', folderId: 'fin', confidence: 0.45 }, + { folder: 'Taxes', path: '/L/Taxes', folderId: 'tax', confidence: 0.41 } + ] + }; + + test('says so in plain words, with the reason and where it goes', () => { + renderItem({ + category: 'Uncategorized', + routing: { folder: UNCATEGORIZED, source: 'review', engine: REVIEW_ENGINE } + }); + const note = screen.getByTestId('organize-needs-review'); + expect(note).toHaveTextContent('Needs review'); + expect(note).toHaveTextContent( + 'No Smart Folder matched this file well. The closest was "Finance".' + ); + expect(note).toHaveTextContent('It goes to "Uncategorized" unless you pick a folder.'); + // The engine's 10% is not a match score: show none, and never the label's 95%. + expect(screen.queryByText(/% confidence/)).not.toBeInTheDocument(); + }); + + test('lists the closest folders first, none marked as the best match', () => { + renderItem({ + category: 'Uncategorized', + routing: { folder: UNCATEGORIZED, source: 'review', engine: REVIEW_ENGINE } + }); + fireEvent.click(screen.getByRole('combobox', { name: /Category/i })); + const options = within(screen.getByRole('listbox')).getAllByRole('option'); + const labels = options.map((o) => o.textContent); + expect(labels).toEqual(expect.arrayContaining(['Finance', 'Taxes', 'Archive'])); + expect(labels.some((label) => label.includes('best match'))).toBe(false); + expect(labels.indexOf('Finance')).toBeLessThan(labels.indexOf('Archive')); + }); + + test('with no default folder it names where the file goes', () => { + renderItem({ + category: 'Uncategorized', + routing: { + folder: null, + source: 'review', + engine: { ...REVIEW_ENGINE, folder: null, path: null, folderId: null } + } + }); + expect(screen.getByTestId('organize-needs-review')).toHaveTextContent( + 'It goes to "Uncategorized" unless you pick a folder.' + ); + }); + + test('once the user picks a folder it is their choice, with no suggestion named', () => { + renderItem({ + category: 'Finance', + routing: { folder: FOLDERS[1], source: 'user', engine: REVIEW_ENGINE } + }); + expect(screen.queryByTestId('organize-needs-review')).not.toBeInTheDocument(); + expect(screen.getByTestId('organize-routing-note')).toHaveTextContent(/^Your choice$/); + }); +}); + test('without routing (legacy callers) there is no note', () => { renderItem({ routing: null }); expect(screen.queryByTestId('organize-routing-note')).not.toBeInTheDocument(); diff --git a/test/learning.llmModeCorrection.test.js b/test/learning.llmModeCorrection.test.js index 16ef8e9c..00567c40 100644 --- a/test/learning.llmModeCorrection.test.js +++ b/test/learning.llmModeCorrection.test.js @@ -342,7 +342,8 @@ describe('other routing modes', () => { test('(l) Embedding mode with no corrections ranks as before', async () => { const service = makeService({ mode: 'embedding', - queryFolders: [invoicesMatch(0.62), receiptsMatch(0.6)] + // 0.07 apart: far enough to tell apart (0.02 would need review). + queryFolders: [invoicesMatch(0.62), receiptsMatch(0.55)] }); const r = await suggest(service); expect(r.routing.mode).toBe('embedding'); @@ -350,7 +351,7 @@ describe('other routing modes', () => { expect(r.primary.method).toBe('semantic_vector_db'); expect(r.primary.score).toBeCloseTo(0.62); expect(r.alternatives[0].folder).toBe('Receipts'); - expect(r.alternatives[0].score).toBeCloseTo(0.6); + expect(r.alternatives[0].score).toBeCloseTo(0.55); expect([r.primary, ...r.alternatives].some((s) => s.learnedOverride)).toBe(false); expect(r.explanation).toBe('This file\'s meaning matches "Invoices"'); await service.shutdown(); diff --git a/test/organizeDestination.test.js b/test/organizeDestination.test.js index 0d0a5c03..5b02cbcf 100644 --- a/test/organizeDestination.test.js +++ b/test/organizeDestination.test.js @@ -143,6 +143,62 @@ describe('normalizeEngineEntry', () => { }); }); +describe('a file the engine marks as needing review', () => { + const REVIEW_TEXT = 'No Smart Folder matched this file well. The closest was "Taxes".'; + + function reviewEngine(withDefault = true) { + return normalizeEngineEntry({ + path: '/in/doc.pdf', + suggestion: withDefault + ? { + ...UNCATEGORIZED, + folder: UNCATEGORIZED.name, + folderId: UNCATEGORIZED.id, + isSmartFolder: true, + source: 'default' + } + : null, + confidence: withDefault ? 0.1 : 0, + explanation: REVIEW_TEXT, + alternatives: [{ ...TAXES, folder: TAXES.name, folderId: TAXES.id, isSmartFolder: true }], + needsReview: true, + routingDecision: { reason: 'weak_match' } + }); + } + + test('stays in the default folder; name matching does not pick one', () => { + const result = resolve({ + analysis: { category: 'Taxes', vendor: 'Acme' }, + engine: reviewEngine() + }); + expect(result.source).toBe('review'); + expect(result.folder).toBe(UNCATEGORIZED); + expect(result.engine.explanation).toBe(REVIEW_TEXT); + expect(result.engine.reviewReason).toBe('weak_match'); + expect(result.engine.alternatives.map((a) => a.folder)).toEqual(['Taxes']); + }); + + test('with no default folder it stays unassigned', () => { + const engine = reviewEngine(false); + expect(engine).not.toBeNull(); + expect(engine.needsReview).toBe(true); + expect(engine.folder).toBeNull(); + const result = resolve({ analysis: { category: 'Taxes' }, engine }); + expect(result.source).toBe('review'); + expect(result.folder).toBeNull(); + }); + + test('the user can still pick a folder', () => { + const result = resolve({ + analysis: { category: 'Taxes' }, + edits: { category: 'Finance' }, + engine: reviewEngine() + }); + expect(result.source).toBe('user'); + expect(result.folder).toBe(FINANCE); + }); +}); + describe('findFolderForCandidate', () => { test('matches by id first, then by path', () => { expect(findFolderForCandidate({ folderId: 'f-tax', path: '/elsewhere' }, FOLDERS)).toBe(TAXES); @@ -166,6 +222,26 @@ describe('suggestionForOperation', () => { }); }); + test('no suggestion for a file that needs review, even from a non-generic default', () => { + const INBOX = { id: 'f-inbox', name: 'Inbox', path: '/library/Inbox' }; + const engine = normalizeEngineEntry({ + path: '/in/doc.pdf', + suggestion: { ...INBOX, folder: 'Inbox', folderId: INBOX.id, isSmartFolder: true }, + confidence: 0.1, + needsReview: true + }); + const destination = resolveOrganizeDestination({ + fileWithEdits: { path: '/in/doc.pdf', name: 'doc.pdf', analysis: {} }, + edits: { category: 'Finance' }, + engineSuggestion: engine, + smartFolders: [...FOLDERS, INBOX], + findSmartFolderForCategory: byName, + findSmartFolderForFile: byVendor + }); + expect(destination.source).toBe('user'); + expect(suggestionForOperation(destination, [...FOLDERS, INBOX])).toBe(undefined); + }); + test('no suggestion for a name match or a generic-bucket engine pick', () => { expect(suggestionForOperation(resolve({ analysis: { category: 'Taxes' } }), FOLDERS)).toBe( undefined diff --git a/test/routingDecision.test.js b/test/routingDecision.test.js new file mode 100644 index 00000000..2f241ebb --- /dev/null +++ b/test/routingDecision.test.js @@ -0,0 +1,116 @@ +/** + * decideRouting: route only to a confident match, otherwise needs review. + */ + +const { + decideRouting, + findNameMatch, + NAME_ONLY_MAX_CONFIDENCE +} = require('../src/main/services/organization/routingDecision'); + +const A = { id: 'a', name: 'Alpha', path: '/Users/alex/L/Alpha' }; +const B = { id: 'b', name: 'Beta', path: '/Users/alex/L/Beta' }; +const DOCS = { id: 'd', name: 'Documents', path: '/Users/alex/L/Documents' }; +const UNC = { + id: 'u', + name: 'Uncategorized', + path: '/Users/alex/L/Uncategorized', + isDefault: true +}; +const FOLDERS = [A, B, UNC]; + +/** A canonicalized, source-tagged suggestion as the engine builds them. */ +const s = (folder, score, source, extra = {}) => ({ + folder: folder.name, + path: folder.path, + folderId: folder.id, + isSmartFolder: true, + score, + confidence: score, + source, + ...extra +}); + +const file = (analysis = {}, name = 'notes.md') => ({ + name, + extension: name.split('.').pop(), + path: `/Users/alex/Downloads/${name}`, + analysis +}); + +test('a template folder in front does not hide a confident content match', () => { + const strategy = s(B, 0.9, 'strategy', { method: 'strategy_based' }); + const semantic = s(A, 0.7, 'semantic'); + const decision = decideRouting({ + file: file(), + ranked: [strategy, semantic], + sourceSuggestions: [strategy, semantic], + smartFolders: FOLDERS + }); + expect(decision.needsReview).toBe(false); + expect(decision.basis).toBe('content_match'); + expect(decision.candidate.folder).toBe('Alpha'); +}); + +test('a feedback rule that lifted the folder counts as learned', () => { + const semantic = s(A, 0.42, 'semantic', { memoryAdjustment: { boost: 0.1, penalty: 0 } }); + const decision = decideRouting({ + file: file(), + ranked: [semantic], + sourceSuggestions: [s(A, 0.42, 'semantic')], + smartFolders: FOLDERS + }); + expect(decision.basis).toBe('learned'); +}); + +test('a confident folder ranked second wins over a weaker one in front', () => { + const llm = s(A, 0.6, 'llm', { method: 'llm_creative' }); + const semantic = s(B, 0.8, 'semantic'); + const decision = decideRouting({ + file: file(), + ranked: [llm, semantic], + sourceSuggestions: [llm, semantic], + smartFolders: FOLDERS + }); + // The second folder is the confident one, so it is routed there. + expect(decision.needsReview).toBe(false); + expect(decision.candidate.folder).toBe('Beta'); +}); + +test('a semantic match to the default folder is not a match', () => { + const semantic = s(UNC, 0.9, 'semantic'); + const decision = decideRouting({ + file: file(), + ranked: [semantic], + sourceSuggestions: [semantic], + smartFolders: FOLDERS + }); + expect(decision.needsReview).toBe(true); + expect(decision.reason).toBe('no_match'); +}); + +test('a folder named after the file type is a name match, capped below auto-move', () => { + const decision = decideRouting({ + file: file({ confidence: 95 }, 'report.pdf'), + ranked: [], + sourceSuggestions: [], + smartFolders: [A, DOCS, UNC] + }); + expect(decision.needsReview).toBe(false); + expect(decision.basis).toBe('name_match'); + expect(decision.candidate.folder).toBe('Documents'); + expect(decision.candidate.confidence).toBe(NAME_ONLY_MAX_CONFIDENCE); + expect(decision.nameMatch).toEqual({ kind: 'type', value: 'Documents' }); +}); + +test('labels only name a folder when the routing mode ranks by them', () => { + const named = file({ category: 'Alpha', vendor: 'Beta' }); + expect(findNameMatch(named, FOLDERS, { useLabels: false })).toBeNull(); + expect(findNameMatch(named, FOLDERS, { useLabels: true })).toMatchObject({ + kind: 'label', + field: 'category' + }); + // An embedding override replaced the label with the content's folder. + const overridden = file({ category: 'Alpha', categorySource: 'embedding_override' }); + expect(findNameMatch(overridden, FOLDERS, { useLabels: true })).toBeNull(); +}); diff --git a/test/useOrganizeSuggestions.test.js b/test/useOrganizeSuggestions.test.js index 81802199..9a4e834c 100644 --- a/test/useOrganizeSuggestions.test.js +++ b/test/useOrganizeSuggestions.test.js @@ -66,6 +66,34 @@ beforeEach(() => { global.window.electronAPI = { suggestions: { getBatchSuggestions } }; }); +test('keeps files that need review, including unassigned ones', async () => { + const files = makeFiles(2, 'review'); + getBatchSuggestions.mockImplementation(async () => ({ + success: true, + groups: [], + unassigned: [ + { + ...files[0], + suggestion: null, + alternatives: [], + confidence: 0, + explanation: 'No Smart Folder matched this file.', + needsReview: true, + routingDecision: { reason: 'no_match' } + } + ] + })); + const { result } = renderHook(() => useOrganizeSuggestions(files, FOLDERS)); + + await waitFor(() => expect(result.current.suggestionsByPath.size).toBe(1)); + const entry = result.current.suggestionsByPath.get('/in/review-0.pdf'); + expect(entry.needsReview).toBe(true); + expect(entry.folder).toBeNull(); + expect(entry.reviewReason).toBe('no_match'); + expect(entry.explanation).toBe('No Smart Folder matched this file.'); + await waitFor(() => expect(result.current.isRanking).toBe(false)); +}); + test('fetches in chunks with generation and improvement candidates off', async () => { const files = makeFiles(SUGGESTION_CHUNK_SIZE + 5); const { result } = renderHook(() => useOrganizeSuggestions(files, FOLDERS)); From 61f3956eecbde9bc68cbec3f5ece2aeb6c99ea06 Mon Sep 17 00:00:00 2001 From: Levy Tate <78818969+iLevyTate@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:40:10 -0500 Subject: [PATCH 02/10] fix: ask before sorting overlapping folders and before moving a Smart Folder Sorting a folder that is, sits inside, or holds a Smart Folder now needs a choice. Discover asks before it sorts such a folder, names the folder picked and the Smart Folder with its path, and offers "Sort this folder" or "Pick another folder" ("Skip this folder" for a drop). The answer is kept on the Smart Folder (sourceOverlapOk), so it is asked once. On Organize, files the plan leaves in the folder they are already in need a choice before Move: leave them there or skip them. Move stays off until the user chooses, handleOrganizeFiles refuses to run without one, skipped files are left out of the move and its preview, and the choice is kept on the Smart Folder (inPlaceFiles) with a way to change it. Changing a Smart Folder's path to a folder that does not exist yet used to move the old folder there with its files without asking. The edit now changes nothing and replies PATH_CHANGE_CHOICE_REQUIRED, and Setup asks: move the folder and its files, use a new empty folder, or cancel. The edit takes the answer as an optional third argument ({ pathChange }), checked by its Zod schema. On macOS a decomposed and a composed spelling of one path count as the same folder. --- src/main/ipc/smartFolders.js | 90 ++++- src/main/ipc/validationSchemas.js | 19 +- src/preload/preload.js | 11 +- .../components/organize/InPlaceMoveNotice.jsx | 97 +++-- src/renderer/components/ui/Modal.jsx | 21 +- src/renderer/hooks/useConfirmDialog.js | 84 ++++- .../hooks/useRememberSmartFolderChoice.js | 57 +++ src/renderer/phases/DiscoverPhase.jsx | 7 +- src/renderer/phases/OrganizePhase.jsx | 45 ++- src/renderer/phases/SetupPhase.jsx | 38 +- .../phases/discover/useFileHandlers.js | 94 +++-- .../phases/organize/useOrganization.js | 208 +++++++++-- src/renderer/services/ipc/smartFoldersIpc.js | 10 +- src/renderer/utils/folderOverlap.js | 78 +++- src/shared/errorCodes.js | 4 + .../ConfirmModal.asyncError.test.js | 25 ++ test/components/InPlaceMoveNotice.test.js | 68 +++- test/components/OrganizePhase.test.js | 38 ++ .../SetupPhase.pathChangeChoice.test.js | 246 ++++++++++++ test/renderer/utils/folderOverlap.test.js | 77 +++- ...smartFolders.edit.pathChangeChoice.test.js | 353 ++++++++++++++++++ test/smartFolders.rename.knowledge.test.js | 10 +- test/smartFolders.rename.nested.test.js | 18 +- test/smartFolders.rename.remap.test.js | 10 +- test/smartFolders.rename.toctou.test.js | 2 +- test/smartFolders.vetRoot.abort.test.js | 3 +- test/useConfirmDialog.test.js | 62 ++- test/useFileHandlers.sourceOverlap.test.js | 196 +++++++++- test/useOrganization.inPlaceChoice.test.js | 298 +++++++++++++++ test/useOrganization.inPlaceMoves.test.js | 53 ++- 30 files changed, 2180 insertions(+), 142 deletions(-) create mode 100644 src/renderer/hooks/useRememberSmartFolderChoice.js create mode 100644 test/phases/SetupPhase.pathChangeChoice.test.js create mode 100644 test/smartFolders.edit.pathChangeChoice.test.js create mode 100644 test/useOrganization.inPlaceChoice.test.js diff --git a/src/main/ipc/smartFolders.js b/src/main/ipc/smartFolders.js index c2f13e31..4fb48f1b 100644 --- a/src/main/ipc/smartFolders.js +++ b/src/main/ipc/smartFolders.js @@ -128,15 +128,19 @@ const WATCHER_STOP_REPLY_WAIT_MS = 20_000; * Windows is always case-insensitive. macOS APFS/HFS+ defaults to * case-insensitive (case-sensitive volumes are rare and opt-in), so darwin * is treated the same way to prevent silent duplicate-path entries that the - * underlying filesystem would treat as the same location. + * underlying filesystem would treat as the same location. On macOS accents + * are composed (NFC) first: Finder and the file system hand back decomposed + * names, and APFS treats both spellings as one name. * @param {string} a * @param {string} b * @returns {boolean} */ function areEquivalentPaths(a, b) { if (typeof a !== 'string' || typeof b !== 'string') return false; - const normA = path.resolve(a); - const normB = path.resolve(b); + const compose = (/** @type {string} */ p) => + process.platform === 'darwin' ? p.normalize('NFC') : p; + const normA = compose(path.resolve(a)); + const normB = compose(path.resolve(b)); if (isCaseInsensitiveFileSystem()) { return normA.toLowerCase() === normB.toLowerCase(); } @@ -1513,11 +1517,58 @@ async function rememberFolderDescriptionChange({ } } +/** + * Does changing a Smart Folder from `oldPath` to `newPath` need the user to + * say what happens to the old folder? Only when the old folder is there and + * the new one is not: the edit could then move the old folder, files and + * all, or start a new empty one. Any other state (old one gone, new one + * already there) has one outcome, and an error reading either path is left + * for the edit to report. + * @param {string} oldPath + * @param {string} newPath + * @returns {Promise} + */ +async function pathChangeNeedsChoice(oldPath, newPath) { + try { + const oldStats = await fs.stat(oldPath); + if (!oldStats.isDirectory()) return false; + } catch { + return false; + } + try { + await fs.stat(newPath); + return false; + } catch (error) { + return /** @type {any} */ (error)?.code === 'ENOENT'; + } +} + +/** + * The reply when a path change needs the user's choice. Nothing has changed + * on disk or in the folder list. + * @param {string} oldPath + * @param {string} newPath + */ +function pathChangeChoiceRequired(oldPath, newPath) { + return { + success: false, + error: + 'The new location does not exist yet. Choose whether to move the folder and its files there, or start with a new empty folder.', + errorCode: ERROR_CODES.PATH_CHANGE_CHOICE_REQUIRED, + oldPath, + newPath + }; +} + /** * @param {SmartFoldersHandlerContext} ctx * @param {IpcMainInvokeEvent} event * @param {string} folderId * @param {Record} updatedFolder + * @param {{ pathChange?: 'move' | 'new' }} [options] - when the new path does + * not exist yet: 'move' moves the old folder there with its files, 'new' + * creates a new empty folder and leaves the old one where it is. Without + * it such an edit changes nothing and asks (PATH_CHANGE_CHOICE_REQUIRED). */ async function handleSmartFoldersEdit( { @@ -1532,8 +1583,11 @@ async function handleSmartFoldersEdit( }, event, folderId, - updatedFolder + updatedFolder, + options = {} ) { + const pathChange = + options?.pathChange === 'move' || options?.pathChange === 'new' ? options.pathChange : null; try { if (!folderId || typeof folderId !== 'string') return { @@ -1609,6 +1663,15 @@ async function handleSmartFoldersEdit( // A new root is held to the destination policy before any directory // is created for it; a rename beside the old root passes on its own. if (!areEquivalentPaths(normalizedPath, customFolders[folderIndex].path)) { + // Asked before the destination policy (which can show its own + // prompt) and before any directory is created, so a cancelled + // choice leaves everything as it was. + if ( + !pathChange && + (await pathChangeNeedsChoice(customFolders[folderIndex].path, normalizedPath)) + ) { + return pathChangeChoiceRequired(customFolders[folderIndex].path, normalizedPath); + } const rootRefusal = await vetSmartFolderRoot({ folderPath: normalizedPath, previousPath: customFolders[folderIndex].path, @@ -1694,7 +1757,17 @@ async function handleSmartFoldersEdit( }; } - if (oldExists && oldIsDirectory && !newExists) { + if (oldExists && oldIsDirectory && !newExists && pathChange !== 'move') { + // Only a 'move' carries the old folder and its files along. Without + // a choice (the new path vanished after the check above) nothing is + // changed; with 'new' the old folder stays where it is. + if (!pathChange) return pathChangeChoiceRequired(oldPath, newPath); + await fs.mkdir(newPath, { recursive: true }); + logger.info('[SMART-FOLDERS] Pointed Smart Folder at a new empty folder', { + oldPath, + newPath + }); + } else if (oldExists && oldIsDirectory && !newExists) { // Revalidate oldPath at the moment of rename. It comes from // persisted customFolders on disk and was sanitized when // written, but the JSON file is user-writable — a tampered @@ -1824,6 +1897,8 @@ async function handleSmartFoldersEdit( return { success: true, folder: customFolders[folderIndex], + // The old folder was moved to the new path with its files. + movedFolder: renamedOnDisk, message: 'Smart folder updated successfully' }; } catch (saveError) { @@ -2439,7 +2514,7 @@ function registerSmartFoldersIpc(servicesOrParams) { logger, context, schema: schemaEditFolder, - handler: (event, folderId, updatedFolder) => + handler: (event, folderId, updatedFolder, options) => handleSmartFoldersEdit( { logger, @@ -2453,7 +2528,8 @@ function registerSmartFoldersIpc(servicesOrParams) { }, event, folderId, - updatedFolder + updatedFolder, + options ) }) ); diff --git a/src/main/ipc/validationSchemas.js b/src/main/ipc/validationSchemas.js index de2af5bc..9c8732a3 100644 --- a/src/main/ipc/validationSchemas.js +++ b/src/main/ipc/validationSchemas.js @@ -748,7 +748,14 @@ if (!z) { description: z.string().max(1000).optional(), keywords: z.array(z.string().max(200)).max(200).optional(), category: z.string().max(255).optional(), - isDefault: z.boolean().optional() + isDefault: z.boolean().optional(), + // Choices the user made about this folder overlapping folders they sort. + // Listed so a whole-list save (Zod strips unknown keys) keeps them. + // sourceOverlapOk: sorting a folder that is, sits inside, or holds this + // one is on purpose. inPlaceFiles: what Organize does with files already + // in this folder that are sent back to it (null: ask again). + sourceOverlapOk: z.boolean().optional(), + inPlaceFiles: z.enum(['leave', 'skip']).nullable().optional() }); /** @@ -758,11 +765,17 @@ if (!z) { const smartFoldersArraySchema = z.array(smartFolderSchema).max(1000); /** - * Smart folder edit input (folderId + updated data) + * Smart folder edit input (folderId + updated data + options). The options + * say what to do when the new path does not exist yet: move the old folder + * there with its files ('move'), or create a new empty folder ('new'). */ const smartFolderEditSchema = z.tuple([ z.string().min(1, 'Folder ID is required'), - smartFolderSchema.partial() + smartFolderSchema.partial(), + z + .object({ pathChange: z.enum(['move', 'new']).optional() }) + .strict() + .optional() ]); // ===== File Operation Schemas ===== diff --git a/src/preload/preload.js b/src/preload/preload.js index ff348e38..54578423 100644 --- a/src/preload/preload.js +++ b/src/preload/preload.js @@ -1829,8 +1829,15 @@ contextBridge.exposeInMainWorld('electronAPI', { secureIPC.safeInvoke(IPC_CHANNELS.SMART_FOLDERS.SCAN_STRUCTURE, rootPath), add: (/** @type {any} */ folder) => secureIPC.safeInvoke(IPC_CHANNELS.SMART_FOLDERS.ADD, folder), - edit: (/** @type {any} */ folderId, /** @type {any} */ updatedFolder) => - secureIPC.safeInvoke(IPC_CHANNELS.SMART_FOLDERS.EDIT, folderId, updatedFolder), + // `options.pathChange` ('move' | 'new') answers PATH_CHANGE_CHOICE_REQUIRED. + edit: ( + /** @type {any} */ folderId, + /** @type {any} */ updatedFolder, + /** @type {any} */ options + ) => + options === undefined + ? secureIPC.safeInvoke(IPC_CHANNELS.SMART_FOLDERS.EDIT, folderId, updatedFolder) + : secureIPC.safeInvoke(IPC_CHANNELS.SMART_FOLDERS.EDIT, folderId, updatedFolder, options), delete: (/** @type {any} */ folderId) => secureIPC.safeInvoke(IPC_CHANNELS.SMART_FOLDERS.DELETE, folderId), match: (/** @type {any} */ text, /** @type {any} */ folders) => diff --git a/src/renderer/components/organize/InPlaceMoveNotice.jsx b/src/renderer/components/organize/InPlaceMoveNotice.jsx index 9927ae88..77a7b3cf 100644 --- a/src/renderer/components/organize/InPlaceMoveNotice.jsx +++ b/src/renderer/components/organize/InPlaceMoveNotice.jsx @@ -1,6 +1,6 @@ import React from 'react'; import PropTypes from 'prop-types'; -import { AlertBox } from '../ui'; +import { AlertBox, Button } from '../ui'; const FOLDERS_SHOWN = 2; @@ -12,37 +12,90 @@ function listFolders(names) { return shown.join(' and '); } +/** What leaving the files there does, by how many of them get a new name. */ +function leaveOutcome(moves) { + const one = moves.length === 1; + const renamedCount = moves.filter((m) => m.renamed).length; + if (renamedCount === moves.length) { + return `${one ? 'It' : 'They'} will be renamed there, not moved to another folder.`; + } + if (renamedCount === 0) { + return `${one ? 'It' : 'They'} will stay as ${one ? 'it is' : 'they are'}.`; + } + return 'They will stay where they are, and the ones with a new name are renamed there.'; +} + +/** The leave/skip outcome once the user has chosen for every file. */ +function chosenOutcome(moves) { + const skipped = moves.filter((m) => m.choice === 'skip'); + const left = moves.filter((m) => m.choice !== 'skip'); + const one = moves.length === 1; + if (skipped.length === 0) return leaveOutcome(moves); + if (left.length === 0) { + return `${one ? 'It' : 'They'} will be skipped and keep ${one ? 'its name' : 'their names'}.`; + } + return `${left.length} will stay where ${left.length === 1 ? 'it is' : 'they are'}, and ${skipped.length} will be skipped.`; +} + /** * Says, before a run, which ready files will not change folder: the smart * folder chosen for them is the folder they are in. Without it the run * reports them as moved and nothing appears anywhere new. + * + * With `onChoose`, a file the user has not decided about (`choice` null) is + * a question: leave it there, or skip it. Move stays off until they answer + * (see useOrganization). The answer is kept for each folder; `onChoose(null)` + * asks again. */ -function InPlaceMoveNotice({ moves, className = '' }) { +function InPlaceMoveNotice({ moves, onChoose = null, className = '' }) { if (!Array.isArray(moves) || moves.length === 0) return null; - const one = moves.length === 1; - const folders = Array.from(new Set(moves.map((m) => m.folderName).filter(Boolean))); - const renamedCount = moves.filter((m) => m.renamed).length; - let outcome; - if (renamedCount === moves.length) { - outcome = `${one ? 'It' : 'They'} will be renamed there, not moved to another folder.`; - } else if (renamedCount === 0) { - outcome = `${one ? 'It' : 'They'} will stay as ${one ? 'it is' : 'they are'}.`; - } else { - outcome = 'They will stay where they are, and the ones with a new name are renamed there.'; - } + const undecided = moves.filter((m) => !m.choice); + const asking = typeof onChoose === 'function' && undecided.length > 0; + const shown = asking ? undecided : moves; + const one = shown.length === 1; + const folders = Array.from(new Set(shown.map((m) => m.folderName).filter(Boolean))); + const them = one ? 'it' : 'them'; + const hasChoice = moves.some((m) => m.choice); return ( - + - {one ? '1 file is' : `${moves.length} files are`} already in the folder chosen for{' '} - {one ? 'it' : 'them'} + {one ? '1 file is' : `${shown.length} files are`} already in the folder chosen for {them} {folders.length > 0 ? `: ${listFolders(folders)}` : ''} -
- {outcome} To send a file somewhere else, choose a different folder on its card. Preview - marks the files that stay. -
+ {asking ? ( + <> +
+ Leave {them} there: {leaveOutcome(shown)} Skip {them}: {one ? 'it is' : 'they are'} left + out of this move and {one ? 'keeps its name' : 'keep their names'}. Move stays off until + you choose. StratoSort keeps your choice for each folder. +
+
+ + +
+ + ) : ( + <> +
+ {typeof onChoose === 'function' ? chosenOutcome(shown) : leaveOutcome(shown)} To send a + file somewhere else, choose a different folder on its card. Preview marks the files that + stay. +
+ {typeof onChoose === 'function' && hasChoice && ( +
+ +
+ )} + + )}
); } @@ -51,9 +104,11 @@ InPlaceMoveNotice.propTypes = { moves: PropTypes.arrayOf( PropTypes.shape({ folderName: PropTypes.string, - renamed: PropTypes.bool + renamed: PropTypes.bool, + choice: PropTypes.oneOf(['leave', 'skip', null]) }) ), + onChoose: PropTypes.func, className: PropTypes.string }; diff --git a/src/renderer/components/ui/Modal.jsx b/src/renderer/components/ui/Modal.jsx index c40fd1a2..6a0b9231 100644 --- a/src/renderer/components/ui/Modal.jsx +++ b/src/renderer/components/ui/Modal.jsx @@ -399,7 +399,11 @@ export const ConfirmModal = memo(function ConfirmModal({ // When set, the user must type this exact string before the Confirm // button enables. Used for irreversible actions (delete backup, full // embedding rebuild) so accidental clicks can't fire them. - requireConfirmText = null + requireConfirmText = null, + // A second real choice beside Confirm (e.g. 'Start with an empty folder' + // beside 'Move the folder'), shown between Cancel and Confirm. + alternativeText = null, + onAlternative = null }) { const [isConfirming, setIsConfirming] = useState(false); const [typedConfirmation, setTypedConfirmation] = useState(''); @@ -532,6 +536,17 @@ export const ConfirmModal = memo(function ConfirmModal({ + {alternativeText && typeof onAlternative === 'function' && ( + + )} {/* eslint-disable jsx-a11y/no-autofocus -- deliberate initial focus per WAI-ARIA dialog pattern */} , + IconButton: ({ icon, ...props }) => , + MinimizableGuideCard: ({ children }) =>
{children}
, + StateMessage: ({ title }) =>
{title}
, + StatusBadge: ({ children }) =>
{children}
+})); + +jest.mock('../../src/renderer/components/ui/Typography', () => ({ + Heading: ({ children }) =>

{children}

, + Text: ({ children, id }) => {children}, + Caption: ({ children }) => {children} +})); + +jest.mock('../../src/renderer/components/layout', () => ({ + ActionBar: ({ children }) =>
{children}
, + Inline: ({ children }) =>
{children}
, + Stack: ({ children }) =>
{children}
+})); + +jest.mock('../../src/renderer/components/ui/LoadingSkeleton', () => ({ + SmartFolderSkeleton: () => null +})); + +jest.mock('../../src/renderer/components/setup/WatcherStatusLine', () => ({ + __esModule: true, + default: () => null +})); + +jest.mock('../../src/renderer/components/setup/SmartFolderAIOverrideField', () => ({ + __esModule: true, + default: () => null +})); + +jest.mock('../../src/renderer/components/setup', () => ({ + SmartFolderItem: jest.requireActual('../../src/renderer/components/setup/SmartFolderItem') + .default, + AddSmartFolderModal: () => null +})); + +import SetupPhase from '../../src/renderer/phases/SetupPhase'; + +jest.setTimeout(30000); +const LOAD_TIMEOUT = { timeout: 10000 }; + +const choiceRequired = { + success: false, + errorCode: 'PATH_CHANGE_CHOICE_REQUIRED', + error: 'The new location does not exist yet.', + oldPath: REPORTS.path, + newPath: NEW_PATH +}; + +async function changePathAndSave() { + render(); + await screen.findAllByTestId('folder-item', {}, LOAD_TIMEOUT); + const card = screen + .getAllByTestId('folder-item') + .find((item) => item.textContent.includes('Reports')); + await act(async () => { + fireEvent.click(card); + }); + const expanded = screen + .getAllByTestId('folder-item') + .find((item) => item.textContent.includes('Reports')); + const editButton = Array.from(expanded.querySelectorAll('button')).find( + (button) => button.getAttribute('title') === 'Edit folder' + ); + await act(async () => { + fireEvent.click(editButton); + }); + fireEvent.change(screen.getByLabelText('Folder path'), { target: { value: NEW_PATH } }); + await act(async () => { + fireEvent.click(screen.getByText('Save changes')); + }); +} + +describe('SetupPhase: a new path that does not exist yet', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + test('asks first, naming both folders and the three ways out', async () => { + mockEdit.mockResolvedValueOnce(choiceRequired); + mockShowChoice.mockResolvedValue(null); + + await changePathAndSave(); + + await waitFor(() => expect(mockShowChoice).toHaveBeenCalledTimes(1)); + const [question] = mockShowChoice.mock.calls[0]; + expect(question.message).toContain(REPORTS.path); + expect(question.message).toContain(NEW_PATH); + expect(question.confirmText).toBe('Move the folder and its files'); + expect(question.alternativeText).toBe('Use a new empty folder'); + expect(question.cancelText).toBe('Cancel'); + }); + + test('Cancel changes nothing and keeps the editor open', async () => { + mockEdit.mockResolvedValueOnce(choiceRequired); + mockShowChoice.mockResolvedValue(null); + + await changePathAndSave(); + + await waitFor(() => expect(mockShowChoice).toHaveBeenCalled()); + expect(mockEdit).toHaveBeenCalledTimes(1); + expect(mockNotificationApi.showSuccess).not.toHaveBeenCalled(); + expect(mockNotificationApi.showError).not.toHaveBeenCalled(); + expect(screen.getByLabelText('Folder path')).toHaveValue(NEW_PATH); + }); + + test('"Move the folder and its files" sends the move and says what moved', async () => { + mockEdit + .mockResolvedValueOnce(choiceRequired) + .mockResolvedValueOnce({ success: true, movedFolder: true }); + mockShowChoice.mockResolvedValue('confirm'); + + await changePathAndSave(); + + await waitFor(() => expect(mockEdit).toHaveBeenCalledTimes(2)); + expect(mockEdit.mock.calls[1][0]).toBe('reports'); + expect(mockEdit.mock.calls[1][2]).toEqual({ pathChange: 'move' }); + await waitFor(() => + expect(mockNotificationApi.showSuccess).toHaveBeenCalledWith( + 'Moved "Reports" and its files to the new location.' + ) + ); + }); + + test('"Use a new empty folder" leaves the old folder and its files where they are', async () => { + mockEdit + .mockResolvedValueOnce(choiceRequired) + .mockResolvedValueOnce({ success: true, movedFolder: false }); + mockShowChoice.mockResolvedValue('alternative'); + + await changePathAndSave(); + + await waitFor(() => expect(mockEdit).toHaveBeenCalledTimes(2)); + expect(mockEdit.mock.calls[1][2]).toEqual({ pathChange: 'new' }); + await waitFor(() => + expect(mockNotificationApi.showSuccess).toHaveBeenCalledWith( + 'Updated "Reports". It now uses a new empty folder. The old folder and its files stay where they are.' + ) + ); + }); + + test('a new path that already exists is saved without asking', async () => { + mockEdit.mockResolvedValueOnce({ success: true, movedFolder: false }); + + await changePathAndSave(); + + await waitFor(() => expect(mockNotificationApi.showSuccess).toHaveBeenCalled()); + expect(mockShowChoice).not.toHaveBeenCalled(); + expect(mockEdit).toHaveBeenCalledTimes(1); + expect(mockEdit.mock.calls[0]).toHaveLength(2); + }); +}); diff --git a/test/renderer/utils/folderOverlap.test.js b/test/renderer/utils/folderOverlap.test.js index 64b40b54..ca63bc2f 100644 --- a/test/renderer/utils/folderOverlap.test.js +++ b/test/renderer/utils/folderOverlap.test.js @@ -7,7 +7,9 @@ import { isSameOrInside, findNestedDestinations, describeSourceOverlap, - sourceOverlapMessage + sourceOverlapMessage, + foldersToConfirm, + sourceOverlapQuestion } from '../../../src/renderer/utils/folderOverlap'; const CI = { caseInsensitive: true }; @@ -158,3 +160,76 @@ describe('describeSourceOverlap and sourceOverlapMessage', () => { expect(sourceOverlapMessage(null)).toBeNull(); }); }); + +describe('the question before sorting a folder that overlaps a Smart Folder', () => { + const reports = { id: 'r', name: 'Reports', path: 'C:\\Users\\Alex\\Sorted\\Reports' }; + + test('the same folder in another case, slashes and a trailing slash still asks', () => { + const result = describeSourceOverlap('c:/users/alex/sorted/reports/', [reports], CI); + expect(result).toEqual({ kind: 'is', folder: reports }); + expect(foldersToConfirm(result)).toEqual([reports]); + }); + + test('a decomposed macOS spelling of the Smart Folder still asks', () => { + const cafe = { id: 'c', name: 'Café', path: '/Users/alex/Caf\u00e9' }; + const result = describeSourceOverlap('/Users/alex/Cafe\u0301', [cafe], CS); + expect(result?.kind).toBe('is'); + expect(foldersToConfirm(result)).toEqual([cafe]); + }); + + test('a Smart Folder the user already said may overlap is not asked about again', () => { + const ok = { ...reports, sourceOverlapOk: true }; + expect(foldersToConfirm(describeSourceOverlap(reports.path, [ok], CI))).toEqual([]); + // Chosen this session, before the saved list shows it. + expect( + foldersToConfirm(describeSourceOverlap(reports.path, [reports], CI), new Set(['r'])) + ).toEqual([]); + }); + + test('a folder holding several asks only about the ones not yet chosen', () => { + const a = { id: 'a', name: 'A', path: '/P/Out/A', sourceOverlapOk: true }; + const b = { id: 'b', name: 'B', path: '/P/Out/B' }; + expect(foldersToConfirm(describeSourceOverlap('/P/Out', [a, b], CS))).toEqual([b]); + }); + + test('names the folder picked and the Smart Folder with its path', () => { + expect( + sourceOverlapQuestion( + '/P/Out/GSC/raw', + describeSourceOverlap('/P/Out/GSC/raw', [{ name: 'GSC', path: '/P/Out/GSC' }], CS) + ) + ).toBe( + 'You picked /P/Out/GSC/raw to sort. It is inside your Smart Folder "GSC" at /P/Out/GSC. Files that belong in "GSC" will move out of the folder you picked to the top of "GSC". If you sort it, StratoSort will not ask again for this Smart Folder.' + ); + expect( + sourceOverlapQuestion( + '/P/Out', + describeSourceOverlap('/P/Out', [{ name: 'GSC', path: '/P/Out/GSC' }], CS) + ) + ).toBe( + 'You picked /P/Out to sort. It holds your Smart Folder "GSC" at /P/Out/GSC. Files already sorted there will be analyzed and sorted again. If you sort it, StratoSort will not ask again for this Smart Folder.' + ); + const two = describeSourceOverlap( + '/P', + [ + { name: 'A', path: '/P/A' }, + { name: 'B', path: '/P/B' } + ], + CS + ); + expect(sourceOverlapQuestion('/P', two)).toContain( + 'It holds 2 of your Smart Folders: "A" at /P/A, "B" at /P/B.' + ); + expect(sourceOverlapQuestion('/P', two)).toContain('not ask again for these Smart Folders.'); + expect(sourceOverlapQuestion('/P', null)).toBeNull(); + }); + + test('the catch-all is described as where unmatched files go', () => { + const catchAll = { name: 'Uncategorized', category: 'uncategorized', path: '/P/In' }; + expect( + sourceOverlapQuestion('/P/In', describeSourceOverlap('/P/In', [catchAll], CS)) + ).toContain( + 'where files that fit nowhere else go. Files that fit nowhere else will stay in it.' + ); + }); +}); diff --git a/test/smartFolders.edit.pathChangeChoice.test.js b/test/smartFolders.edit.pathChangeChoice.test.js new file mode 100644 index 00000000..fcfa5190 --- /dev/null +++ b/test/smartFolders.edit.pathChangeChoice.test.js @@ -0,0 +1,353 @@ +/** + * @jest-environment node + * + * Changing a Smart Folder's path to a folder that does not exist yet used to + * move the old folder there, files and all, without asking. The edit now + * changes nothing and asks (PATH_CHANGE_CHOICE_REQUIRED) until the caller + * says 'move' (take the old folder and its files along) or 'new' (start with + * a new empty folder, leave the old one where it is). + * + * Harness mirrors smartFolders.rename.nested.test.js (mocked fs + wrappers), + * with fs.stat answering from a set of folders that exist. + */ + +const path = require('path'); +const { IPC_CHANNELS } = require('../src/shared/constants'); + +jest.mock('fs', () => ({ + promises: { + stat: jest.fn(), + mkdir: jest.fn(), + writeFile: jest.fn(), + unlink: jest.fn(), + rm: jest.fn(), + rename: jest.fn(), + access: jest.fn(), + readdir: jest.fn() + } +})); + +jest.mock('electron', () => ({ + app: { + getPath: jest.fn((name) => (name === 'documents' ? '/Users/alex/Documents' : '')) + } +})); + +const mockAuthorizeSmartFolderRoot = jest.fn(async () => ({ allowed: true, reason: 'test' })); +jest.mock('../src/main/ipc/files/destinationPolicy', () => ({ + ...jest.requireActual('../src/main/ipc/files/destinationPolicy'), + authorizeSmartFolderRoot: (...args) => mockAuthorizeSmartFolderRoot(...args) +})); + +jest.mock('../src/shared/logger', () => { + const logger = { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + setContext: jest.fn() + }; + return { createLogger: jest.fn(() => logger) }; +}); + +jest.mock('../src/main/ipc/ipcWrappers', () => ({ + createHandler: jest.fn(({ handler }) => handler), + safeHandle: jest.fn(), + safeSend: jest.fn(), + withErrorLogging: jest.fn((logger, handler) => { + if (typeof handler === 'function') { + return async (...args) => handler(...args); + } + return (fn) => fn; + }), + z: { + object: jest.fn(() => ({ passthrough: jest.fn(() => ({})) })), + string: jest.fn(() => ({ min: jest.fn(() => ({})), optional: jest.fn(() => ({})) })), + array: jest.fn(() => ({})), + any: jest.fn(() => ({})), + void: jest.fn(() => ({})), + record: jest.fn(() => ({})) + } +})); + +jest.mock('../src/main/services/FolderMatchingService', () => ({ getInstance: jest.fn() })); +jest.mock('../src/main/services/ParallelEmbeddingService', () => ({ getInstance: jest.fn() })); +jest.mock('../src/main/services/LlamaService', () => ({ getInstance: jest.fn() })); +jest.mock('../src/main/services/SmartFoldersLLMService', () => ({ + enhanceSmartFolderWithLLM: jest.fn() +})); +jest.mock('../src/main/utils/jsonRepair', () => ({ + extractAndParseJSON: jest.fn((json) => JSON.parse(json)) +})); +jest.mock('../src/shared/pathSanitization', () => ({ + validateFileOperationPathSync: jest.fn((p) => ({ valid: true, normalizedPath: p })), + validateFileOperationPath: jest.fn(async (p) => ({ valid: true, normalizedPath: p })) +})); +jest.mock('../src/shared/crossPlatformUtils', () => ({ + ...jest.requireActual('../src/shared/crossPlatformUtils'), + isUNCPath: jest.fn(() => false), + isWindowsDeviceNamespacePath: jest.fn(() => false) +})); +jest.mock('../src/main/analysis/semanticExtensionMap', () => ({ + enrichFolderTextForEmbedding: jest.fn((name, desc) => `${name} ${desc || ''}`), + enrichFileTextForEmbedding: jest.fn((text) => text) +})); + +// path.resolve gives an absolute path on every platform (C:\Users\alex on +// Windows, /Users/alex elsewhere). +const DOCS = path.resolve('/Users/alex/Documents'); +const OLD_PATH = path.join(DOCS, 'Reports'); +const NEW_PATH = path.join(DOCS, 'Work', 'Reports 2026'); + +describe('editing a Smart Folder path to a folder that does not exist yet', () => { + let safeHandle; + let fs; + /** Folders that exist, keyed by path.resolve. */ + let existing; + + function getHandler(channel) { + const call = safeHandle.mock.calls.find((c) => c[1] === channel); + if (!call) throw new Error(`Handler for ${channel} not registered`); + return call[2]; + } + + function register(folders) { + const mockLogger = require('../src/shared/logger').createLogger(); + const foldersService = { + getCustomFolders: jest.fn(() => folders.map((f) => ({ ...f }))), + setCustomFolders: jest.fn(), + saveCustomFolders: jest.fn().mockResolvedValue(), + scanDirectory: jest.fn().mockResolvedValue([]) + }; + const serviceIntegration = { + smartFolderWatcher: { + isRunning: true, + updateWatchedFolders: jest.fn(async () => {}), + scanFolderForUnanalyzedFiles: jest.fn(async () => ({ scanned: 0, queued: 0 })) + }, + analysisHistory: { analysisHistory: { entries: {} } } + }; + const context = { + ipcMain: { handle: jest.fn() }, + IPC_CHANNELS, + ...foldersService, + foldersService, + getServiceIntegration: () => serviceIntegration, + getFilePathCoordinator: () => null, + logger: mockLogger, + core: { ipcMain: { handle: jest.fn() }, IPC_CHANNELS, logger: mockLogger }, + folders: foldersService + }; + require('../src/main/ipc/smartFolders')(context); + return foldersService; + } + + const enoent = () => Object.assign(new Error('ENOENT'), { code: 'ENOENT' }); + const edit = (...args) => getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT)({}, ...args); + const savedFolder = (foldersService, id) => + foldersService.saveCustomFolders.mock.calls[0][0].find((f) => f.id === id); + + const reports = { + id: 'reports', + name: 'Reports', + path: OLD_PATH, + description: 'Quarterly reports', + inPlaceFiles: 'skip' + }; + const nested = { id: 'drafts', name: 'Drafts', path: path.join(OLD_PATH, 'Drafts') }; + + beforeEach(() => { + jest.clearAllMocks(); + jest.resetModules(); + fs = require('fs').promises; + safeHandle = require('../src/main/ipc/ipcWrappers').safeHandle; + existing = new Set([DOCS, path.dirname(NEW_PATH), OLD_PATH].map((p) => path.resolve(p))); + fs.stat.mockImplementation(async (p) => { + if (existing.has(path.resolve(p))) return { isDirectory: () => true }; + throw enoent(); + }); + fs.access.mockRejectedValue(enoent()); + fs.rename.mockResolvedValue(); + fs.mkdir.mockResolvedValue(); + }); + + test('without a choice nothing changes, and the reply names both folders', async () => { + const foldersService = register([reports, nested]); + + const result = await edit('reports', { path: NEW_PATH }); + + expect(result).toEqual( + expect.objectContaining({ + success: false, + errorCode: 'PATH_CHANGE_CHOICE_REQUIRED', + oldPath: OLD_PATH, + newPath: NEW_PATH + }) + ); + expect(fs.rename).not.toHaveBeenCalled(); + expect(fs.mkdir).not.toHaveBeenCalled(); + expect(foldersService.saveCustomFolders).not.toHaveBeenCalled(); + // Asked before the destination policy, which can show its own prompt. + expect(mockAuthorizeSmartFolderRoot).not.toHaveBeenCalled(); + }); + + test("'move' moves the old folder and its files to the new path", async () => { + const foldersService = register([reports, nested]); + + const result = await edit('reports', { path: NEW_PATH }, { pathChange: 'move' }); + + expect(result.success).toBe(true); + expect(result.movedFolder).toBe(true); + expect(fs.rename).toHaveBeenCalledWith(OLD_PATH, NEW_PATH); + expect(savedFolder(foldersService, 'reports').path).toBe(NEW_PATH); + // The nested Smart Folder moved with it. + expect(savedFolder(foldersService, 'drafts').path).toBe(path.join(NEW_PATH, 'Drafts')); + }); + + test("'new' creates an empty folder and leaves the old one and its files where they are", async () => { + const foldersService = register([reports, nested]); + + const result = await edit('reports', { path: NEW_PATH }, { pathChange: 'new' }); + + expect(result.success).toBe(true); + expect(result.movedFolder).toBe(false); + expect(fs.rename).not.toHaveBeenCalled(); + expect(fs.mkdir).toHaveBeenCalledWith(NEW_PATH, { recursive: true }); + expect(savedFolder(foldersService, 'reports').path).toBe(NEW_PATH); + // Nothing moved, so the nested Smart Folder keeps its path. + expect(savedFolder(foldersService, 'drafts').path).toBe(path.join(OLD_PATH, 'Drafts')); + }); + + test('the user settings on the folder are kept through a path change', async () => { + const foldersService = register([reports]); + + await edit('reports', { path: NEW_PATH }, { pathChange: 'new' }); + + expect(savedFolder(foldersService, 'reports')).toEqual( + expect.objectContaining({ description: 'Quarterly reports', inPlaceFiles: 'skip' }) + ); + }); + + test('a new path that already exists needs no choice: the folder is pointed there', async () => { + existing.add(path.resolve(NEW_PATH)); + const foldersService = register([reports]); + + const result = await edit('reports', { path: NEW_PATH }); + + expect(result.success).toBe(true); + expect(fs.rename).not.toHaveBeenCalled(); + expect(savedFolder(foldersService, 'reports').path).toBe(NEW_PATH); + }); + + test('an old folder that is gone needs no choice: the new one is created', async () => { + existing.delete(path.resolve(OLD_PATH)); + const foldersService = register([reports]); + + const result = await edit('reports', { path: NEW_PATH }); + + expect(result.success).toBe(true); + expect(fs.rename).not.toHaveBeenCalled(); + expect(fs.mkdir).toHaveBeenCalledWith(NEW_PATH, { recursive: true }); + expect(savedFolder(foldersService, 'reports').path).toBe(NEW_PATH); + }); + + test('the same folder spelled with a trailing separator or other slashes is not a path change', async () => { + const respellings = [`${OLD_PATH}${path.sep}`, OLD_PATH.split(path.sep).join('/')]; + for (const spelling of respellings) { + jest.clearAllMocks(); + const foldersService = register([reports]); + const result = await edit('reports', { path: spelling }); + expect(result.success).toBe(true); + expect(fs.rename).not.toHaveBeenCalled(); + expect(foldersService.saveCustomFolders).toHaveBeenCalled(); + } + }); + + describe('on a case-insensitive volume', () => { + const realPlatform = process.platform; + afterEach(() => { + Object.defineProperty(process, 'platform', { value: realPlatform }); + }); + + test('a change of case is the same folder, not a move', async () => { + Object.defineProperty(process, 'platform', { value: 'win32' }); + const foldersService = register([reports]); + + const result = await edit('reports', { path: OLD_PATH.toUpperCase() }); + + expect(result.success).toBe(true); + expect(result.errorCode).toBeUndefined(); + expect(fs.rename).not.toHaveBeenCalled(); + expect(foldersService.saveCustomFolders).toHaveBeenCalled(); + }); + + test('on macOS a decomposed accent is the same folder as the composed one', async () => { + Object.defineProperty(process, 'platform', { value: 'darwin' }); + const composed = path.join(DOCS, 'Caf\u00e9'); + const decomposed = path.join(DOCS, 'Cafe\u0301'); + existing.add(path.resolve(composed)); + const foldersService = register([{ ...reports, path: composed }]); + + const result = await edit('reports', { path: decomposed }); + + expect(result.success).toBe(true); + expect(result.errorCode).toBeUndefined(); + expect(fs.rename).not.toHaveBeenCalled(); + expect(fs.mkdir).not.toHaveBeenCalledWith(decomposed, expect.anything()); + expect(foldersService.saveCustomFolders).toHaveBeenCalled(); + }); + }); + + test('an edit of only a remembered choice keeps everything else', async () => { + const foldersService = register([reports]); + + const result = await edit('reports', { inPlaceFiles: 'leave' }); + + expect(result.success).toBe(true); + expect(fs.rename).not.toHaveBeenCalled(); + expect(savedFolder(foldersService, 'reports')).toEqual( + expect.objectContaining({ + name: 'Reports', + path: OLD_PATH, + description: 'Quarterly reports', + inPlaceFiles: 'leave' + }) + ); + }); +}); + +describe('Smart Folder IPC schemas', () => { + const { schemas } = jest.requireActual('../src/main/ipc/validationSchemas'); + + test('the edit takes an optional path-change choice', () => { + expect(schemas.smartFolderEdit.safeParse(['id', { path: '/Users/alex/Out' }]).success).toBe( + true + ); + const parsed = schemas.smartFolderEdit.safeParse([ + 'id', + { path: '/Users/alex/Out' }, + { pathChange: 'new' } + ]); + expect(parsed.success).toBe(true); + expect(parsed.data[2]).toEqual({ pathChange: 'new' }); + }); + + test('the edit refuses an unknown choice', () => { + expect(schemas.smartFolderEdit.safeParse(['id', {}, { pathChange: 'delete' }]).success).toBe( + false + ); + expect(schemas.smartFolderEdit.safeParse(['id', {}, { force: true }]).success).toBe(false); + }); + + test('a whole-list save keeps the overlap choices', () => { + const parsed = schemas.smartFoldersArray.safeParse([ + { name: 'Out', path: '/Users/alex/Out', sourceOverlapOk: true, inPlaceFiles: 'skip' }, + { name: 'In', path: '/Users/alex/In', inPlaceFiles: null } + ]); + expect(parsed.success).toBe(true); + expect(parsed.data[0]).toEqual( + expect.objectContaining({ sourceOverlapOk: true, inPlaceFiles: 'skip' }) + ); + expect(parsed.data[1].inPlaceFiles).toBeNull(); + }); +}); diff --git a/test/smartFolders.rename.knowledge.test.js b/test/smartFolders.rename.knowledge.test.js index 551fd509..e00b2d9d 100644 --- a/test/smartFolders.rename.knowledge.test.js +++ b/test/smartFolders.rename.knowledge.test.js @@ -207,7 +207,7 @@ describe('smartFolders rename moves the knowledge with the directory [K-08]', () test('routes every contained history entry through the path coordinator', async () => { const handler = getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT); - const result = await handler({}, '1', { path: NEW_PATH }); + const result = await handler({}, '1', { path: NEW_PATH }, { pathChange: 'move' }); expect(result.success).toBe(true); expect(fs.rename).toHaveBeenCalledWith(OLD_PATH, NEW_PATH); @@ -230,7 +230,7 @@ describe('smartFolders rename moves the knowledge with the directory [K-08]', () ]); const handler = getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT); - const result = await handler({}, '1', { path: NEW_PATH }); + const result = await handler({}, '1', { path: NEW_PATH }, { pathChange: 'move' }); expect(result.success).toBe(true); expect(mockCoordinator.batchPathUpdate).not.toHaveBeenCalled(); @@ -240,7 +240,7 @@ describe('smartFolders rename moves the knowledge with the directory [K-08]', () mockCoordinator.batchPathUpdate.mockRejectedValue(new Error('vector store offline')); const handler = getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT); - const result = await handler({}, '1', { path: NEW_PATH }); + const result = await handler({}, '1', { path: NEW_PATH }, { pathChange: 'move' }); expect(result.success).toBe(true); expect(mockFoldersService.saveCustomFolders).toHaveBeenCalled(); @@ -254,7 +254,7 @@ describe('smartFolders rename moves the knowledge with the directory [K-08]', () mockContainer.has.mockReturnValue(false); const handler = getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT); - const result = await handler({}, '1', { path: NEW_PATH }); + const result = await handler({}, '1', { path: NEW_PATH }, { pathChange: 'move' }); expect(result.success).toBe(true); expect(mockCoordinator.batchPathUpdate).not.toHaveBeenCalled(); @@ -266,7 +266,7 @@ describe('smartFolders rename moves the knowledge with the directory [K-08]', () fs.stat.mockResolvedValue({ isDirectory: () => true }); const handler = getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT); - const result = await handler({}, '1', { path: NEW_PATH }); + const result = await handler({}, '1', { path: NEW_PATH }, { pathChange: 'move' }); expect(result.success).toBe(true); expect(fs.rename).not.toHaveBeenCalled(); diff --git a/test/smartFolders.rename.nested.test.js b/test/smartFolders.rename.nested.test.js index 5dcf4f6c..5684a563 100644 --- a/test/smartFolders.rename.nested.test.js +++ b/test/smartFolders.rename.nested.test.js @@ -180,9 +180,12 @@ describe('smart folder rename re-points the Smart Folders nested under it', () = fs.access.mockRejectedValue(enoent()); fs.rename.mockResolvedValue(); - const result = await getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT)({}, 'parent', { - path: NEW_PATH - }); + const result = await getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT)( + {}, + 'parent', + { path: NEW_PATH }, + { pathChange: 'move' } + ); expect(result.success).toBe(true); expect(fs.rename).toHaveBeenCalledTimes(1); @@ -207,9 +210,12 @@ describe('smart folder rename re-points the Smart Folders nested under it', () = .mockResolvedValueOnce({ isDirectory: () => true }) .mockResolvedValueOnce({ isDirectory: () => true }); - const result = await getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT)({}, 'parent', { - path: NEW_PATH - }); + const result = await getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT)( + {}, + 'parent', + { path: NEW_PATH }, + { pathChange: 'move' } + ); expect(result.success).toBe(true); expect(fs.rename).not.toHaveBeenCalled(); diff --git a/test/smartFolders.rename.remap.test.js b/test/smartFolders.rename.remap.test.js index 07a0dcf3..b5c9a34a 100644 --- a/test/smartFolders.rename.remap.test.js +++ b/test/smartFolders.rename.remap.test.js @@ -184,7 +184,7 @@ describe('smart folder rename re-keys the knowledge under it [F-188]', () => { armSuccessfulRename(); const handler = getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT); - const result = await handler({}, '1', { path: NEW_PATH }); + const result = await handler({}, '1', { path: NEW_PATH }, { pathChange: 'move' }); expect(result.success).toBe(true); expect(fs.rename).toHaveBeenCalledTimes(1); @@ -210,7 +210,7 @@ describe('smart folder rename re-keys the knowledge under it [F-188]', () => { armSuccessfulRename(); const handler = getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT); - const result = await handler({}, '1', { path: NEW_PATH }); + const result = await handler({}, '1', { path: NEW_PATH }, { pathChange: 'move' }); expect(result.success).toBe(true); expect(coordinator.batchPathUpdate).not.toHaveBeenCalled(); @@ -226,7 +226,7 @@ describe('smart folder rename re-keys the knowledge under it [F-188]', () => { armSuccessfulRename(); const handler = getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT); - const result = await handler({}, '1', { path: NEW_PATH }); + const result = await handler({}, '1', { path: NEW_PATH }, { pathChange: 'move' }); expect(result.success).toBe(true); expect(logger.warn).toHaveBeenCalledWith( @@ -240,7 +240,7 @@ describe('smart folder rename re-keys the knowledge under it [F-188]', () => { armSuccessfulRename(); const handler = getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT); - const result = await handler({}, '1', { path: NEW_PATH }); + const result = await handler({}, '1', { path: NEW_PATH }, { pathChange: 'move' }); expect(result.success).toBe(true); expect(logger.warn).toHaveBeenCalledWith( @@ -258,7 +258,7 @@ describe('smart folder rename re-keys the knowledge under it [F-188]', () => { .mockResolvedValueOnce({ isDirectory: () => true }); const handler = getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT); - const result = await handler({}, '1', { path: NEW_PATH }); + const result = await handler({}, '1', { path: NEW_PATH }, { pathChange: 'move' }); expect(result.success).toBe(true); expect(fs.rename).not.toHaveBeenCalled(); diff --git a/test/smartFolders.rename.toctou.test.js b/test/smartFolders.rename.toctou.test.js index a3a1cf20..18806f8b 100644 --- a/test/smartFolders.rename.toctou.test.js +++ b/test/smartFolders.rename.toctou.test.js @@ -202,7 +202,7 @@ describe('smartFolders directory rename TOCTOU overwrite guard [Position 04 Gap // silently replacing the squatter — this is the very bug we're guarding. fs.rename.mockResolvedValue(); - const result = await handler({}, '1', { path: newPath }); + const result = await handler({}, '1', { path: newPath }, { pathChange: 'move' }); expect(result.success).toBe(false); expect(result.errorCode).toBe('RENAME_FAILED'); diff --git a/test/smartFolders.vetRoot.abort.test.js b/test/smartFolders.vetRoot.abort.test.js index 06a7b2ae..defadec3 100644 --- a/test/smartFolders.vetRoot.abort.test.js +++ b/test/smartFolders.vetRoot.abort.test.js @@ -179,7 +179,8 @@ describe('Smart Folder root confirmation follows the request abort', () => { const result = await getHandler(IPC_CHANNELS.SMART_FOLDERS.EDIT)( { abortSignal: controller.signal }, '1', - { path: ELSEWHERE } + { path: ELSEWHERE }, + { pathChange: 'move' } ); expect(result).toEqual( diff --git a/test/useConfirmDialog.test.js b/test/useConfirmDialog.test.js index 96f502f7..adf4ebff 100644 --- a/test/useConfirmDialog.test.js +++ b/test/useConfirmDialog.test.js @@ -20,7 +20,17 @@ jest.mock('../src/shared/logger', () => { // Mock Modal component with testable buttons jest.mock('../src/renderer/components/ui/Modal', () => ({ - ConfirmModal: ({ isOpen, onConfirm, onClose, title, confirmText, cancelText, variant }) => { + ConfirmModal: ({ + isOpen, + onConfirm, + onClose, + title, + confirmText, + cancelText, + variant, + alternativeText, + onAlternative + }) => { if (!isOpen) return null; return (
@@ -32,6 +42,11 @@ jest.mock('../src/renderer/components/ui/Modal', () => ({ + {onAlternative && ( + + )}
); } @@ -151,4 +166,49 @@ describe('useConfirmDialog', () => { expect(screen.getByText('Cancel')).toBeDefined(); expect(screen.getByTestId('variant').textContent).toBe('default'); }); + + describe('showChoice: two real choices and a way out', () => { + const open = (result) => { + let answer; + act(() => { + answer = result.current.showChoice({ + title: 'Move the folder or start a new one?', + message: 'The new location does not exist yet.', + confirmText: 'Move the folder and its files', + alternativeText: 'Use a new empty folder', + cancelText: 'Cancel' + }); + }); + const DialogComponent = result.current.ConfirmDialog; + render(); + return answer; + }; + + test.each([ + ['confirm-btn', 'confirm'], + ['alternative-btn', 'alternative'], + ['cancel-btn', null] + ])('%s resolves %p', async (button, expected) => { + const { result } = renderHook(() => useConfirmDialog()); + const answer = open(result); + + expect(screen.getByText('Use a new empty folder')).toBeDefined(); + act(() => { + fireEvent.click(screen.getByTestId(button)); + }); + + await expect(answer).resolves.toBe(expected); + }); + + test('showConfirm shows no third button', () => { + const { result } = renderHook(() => useConfirmDialog()); + act(() => { + result.current.showConfirm({ message: 'Sure?' }); + }); + const DialogComponent = result.current.ConfirmDialog; + render(); + + expect(screen.queryByTestId('alternative-btn')).toBeNull(); + }); + }); }); diff --git a/test/useFileHandlers.sourceOverlap.test.js b/test/useFileHandlers.sourceOverlap.test.js index 4b74df19..ad0997a8 100644 --- a/test/useFileHandlers.sourceOverlap.test.js +++ b/test/useFileHandlers.sourceOverlap.test.js @@ -1,6 +1,8 @@ /** - * useFileHandlers warns (never blocks) when the folder picked for sorting - * overlaps a Smart Folder, where sorted files go. + * A folder picked for sorting that overlaps a Smart Folder (where sorted + * files go) is sorted only once the user says so. The answer is kept on the + * Smart Folder (`sourceOverlapOk`); after that, and when no dialog is wired + * in, the overlap is described in a warning and the folder is sorted. */ import { renderHook, act, waitFor } from '@testing-library/react'; @@ -195,3 +197,193 @@ describe('useFileHandlers source overlap warnings', () => { expect(overlapCalls()).toHaveLength(0); }); }); + +describe('useFileHandlers: sorting a folder that overlaps a Smart Folder needs a choice', () => { + let useFileHandlers; + let mockAddSelectedFiles; + let mockAddNotification; + let mockAnalyzeFiles; + let mockShowConfirm; + let mockRemember; + + const OUT = { id: 'out', name: 'Reports', path: '/Users/alex/Sorted/Reports' }; + + const createOptions = (overrides = {}) => ({ + selectedFiles: [], + setSelectedFiles: jest.fn(), + addSelectedFiles: mockAddSelectedFiles, + updateFileState: jest.fn(), + addNotification: mockAddNotification, + analyzeFiles: mockAnalyzeFiles, + smartFolders: [OUT], + showConfirm: mockShowConfirm, + rememberSmartFolderChoice: mockRemember, + ...overrides + }); + + const overlapWarnings = () => + mockAddNotification.mock.calls.filter(([, , , key]) => key === 'source-overlap'); + const addedPaths = () => + mockAddSelectedFiles.mock.calls.flatMap(([files]) => files.map((f) => f.path)); + + const pick = async (options) => { + const { result } = renderHook(() => useFileHandlers(options)); + await act(async () => { + await result.current.handleFolderSelection(); + }); + }; + + beforeEach(() => { + jest.clearAllMocks(); + global.window = global.window || {}; + global.window.electronAPI = mockElectronAPI; + mockAddSelectedFiles = jest.fn(); + mockAddNotification = jest.fn(); + mockAnalyzeFiles = jest.fn(); + mockShowConfirm = jest.fn(); + mockRemember = jest.fn(async () => true); + mockElectronAPI.files.getStats.mockResolvedValue({ isDirectory: true }); + mockElectronAPI.files.selectDirectory.mockReset(); + mockElectronAPI.smartFolders.scanStructure.mockReset(); + mockElectronAPI.smartFolders.scanStructure.mockImplementation(async (dir) => ({ + files: [{ path: `${dir}/a.csv`, name: 'a.csv' }] + })); + useFileHandlers = require('../src/renderer/phases/discover/useFileHandlers').useFileHandlers; + }); + + test.each([ + ['the Smart Folder itself', '/Users/alex/Sorted/Reports', 'It is also your Smart Folder'], + [ + 'the Smart Folder spelled with a trailing slash', + '/Users/alex/Sorted/Reports/', + 'It is also your Smart Folder' + ], + [ + 'the Smart Folder spelled with Windows separators', + '\\Users\\alex\\Sorted\\Reports', + 'It is also your Smart Folder' + ], + ['a folder inside it', '/Users/alex/Sorted/Reports/2026', 'It is inside your Smart Folder'], + ['a folder that holds it', '/Users/alex/Sorted', 'It holds your Smart Folder'] + ])('picking %s asks first and names both folders', async (_label, picked, relation) => { + mockElectronAPI.files.selectDirectory.mockResolvedValueOnce({ success: true, path: picked }); + mockShowConfirm.mockResolvedValue(true); + + await pick(createOptions()); + + expect(mockShowConfirm).toHaveBeenCalledTimes(1); + const [{ message, confirmText, cancelText }] = mockShowConfirm.mock.calls[0]; + expect(message).toContain(`You picked ${picked} to sort.`); + expect(message).toContain(relation); + expect(message).toContain('"Reports"'); + if (relation !== 'It is also your Smart Folder') { + expect(message).toContain('at /Users/alex/Sorted/Reports'); + } + expect(confirmText).toBe('Sort this folder'); + expect(cancelText).toBe('Pick another folder'); + expect(mockAnalyzeFiles).toHaveBeenCalled(); + }); + + test('sorting it on purpose is kept on the Smart Folder, so the next pick is not asked', async () => { + mockElectronAPI.files.selectDirectory.mockResolvedValue({ + success: true, + path: '/Users/alex/Sorted/Reports' + }); + mockShowConfirm.mockResolvedValue(true); + + await pick(createOptions()); + expect(mockRemember).toHaveBeenCalledWith(['out'], { sourceOverlapOk: true }); + + mockShowConfirm.mockClear(); + await pick(createOptions({ smartFolders: [{ ...OUT, sourceOverlapOk: true }] })); + expect(mockShowConfirm).not.toHaveBeenCalled(); + // Still said, not asked. + expect(overlapWarnings()).toHaveLength(1); + }); + + test('"Pick another folder" sorts nothing from that folder and opens the picker again', async () => { + mockElectronAPI.files.selectDirectory + .mockResolvedValueOnce({ success: true, path: '/Users/alex/Sorted/Reports' }) + .mockResolvedValueOnce({ success: true, path: '/Users/alex/Inbox' }); + mockShowConfirm.mockResolvedValue(false); + + await pick(createOptions()); + + expect(mockElectronAPI.files.selectDirectory).toHaveBeenCalledTimes(2); + expect(mockRemember).not.toHaveBeenCalled(); + expect(addedPaths()).toEqual(['/Users/alex/Inbox/a.csv']); + }); + + test('closing the picker after declining sorts nothing', async () => { + mockElectronAPI.files.selectDirectory + .mockResolvedValueOnce({ success: true, path: '/Users/alex/Sorted/Reports' }) + .mockResolvedValueOnce({ success: false, path: null }); + mockShowConfirm.mockResolvedValue(false); + + await pick(createOptions()); + + expect(mockAddSelectedFiles).not.toHaveBeenCalled(); + expect(mockAnalyzeFiles).not.toHaveBeenCalled(); + }); + + test('a dropped folder that is declined is skipped; the rest of the drop is sorted', async () => { + mockShowConfirm.mockResolvedValue(false); + const { result } = renderHook(() => useFileHandlers(createOptions())); + + await act(async () => { + await result.current.handleFileDrop([ + { path: '/Users/alex/Sorted/Reports', name: 'Reports' }, + { path: '/Users/alex/Inbox', name: 'Inbox' } + ]); + }); + + expect(mockShowConfirm).toHaveBeenCalledTimes(1); + expect(mockShowConfirm.mock.calls[0][0].cancelText).toBe('Skip this folder'); + expect(addedPaths()).toEqual(['/Users/alex/Inbox/a.csv']); + }); + + test('two dropped folders inside one Smart Folder ask once', async () => { + mockShowConfirm.mockResolvedValue(true); + const { result } = renderHook(() => useFileHandlers(createOptions())); + + await act(async () => { + await result.current.handleFileDrop([ + { path: '/Users/alex/Sorted/Reports/2025', name: '2025' }, + { path: '/Users/alex/Sorted/Reports/2026', name: '2026' } + ]); + }); + + expect(mockShowConfirm).toHaveBeenCalledTimes(1); + }); + + test('a choice that could not be saved still sorts the folder, and says it will ask again', async () => { + mockElectronAPI.files.selectDirectory.mockResolvedValueOnce({ + success: true, + path: '/Users/alex/Sorted/Reports' + }); + mockShowConfirm.mockResolvedValue(true); + mockRemember.mockResolvedValue(false); + + await pick(createOptions()); + + expect(mockAnalyzeFiles).toHaveBeenCalled(); + expect(mockAddNotification).toHaveBeenCalledWith( + 'Your choice was not saved. StratoSort will ask again next time.', + 'warning', + 5000, + 'source-overlap-save' + ); + }); + + test('a folder that only shares a name prefix is not asked about', async () => { + mockElectronAPI.files.selectDirectory.mockResolvedValueOnce({ + success: true, + path: '/Users/alex/Sorted/Reports 2' + }); + + await pick(createOptions()); + + expect(mockShowConfirm).not.toHaveBeenCalled(); + expect(mockAnalyzeFiles).toHaveBeenCalled(); + }); +}); diff --git a/test/useOrganization.inPlaceChoice.test.js b/test/useOrganization.inPlaceChoice.test.js new file mode 100644 index 00000000..2325d632 --- /dev/null +++ b/test/useOrganization.inPlaceChoice.test.js @@ -0,0 +1,298 @@ +/** + * A folder can be both a folder being sorted and a smart folder files are + * sent to. A file the plan sends to the folder it is already in used to be + * renamed there (or left as it was) and counted as moved, with only a notice + * on screen. Now the user must choose first: leave such files there, or skip + * them. Move does nothing until they choose, and the choice is kept on the + * smart folder (`inPlaceFiles`) so the next run does not ask again. + */ + +import { renderHook, act } from '@testing-library/react'; + +jest.mock('../src/shared/logger', () => { + const logger = { + setContext: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + warn: jest.fn(), + error: jest.fn() + }; + return { logger, createLogger: jest.fn(() => logger) }; +}); + +jest.mock('../src/shared/constants', () => ({ + PHASES: { DISCOVER: 'discover', ORGANIZE: 'organize', COMPLETE: 'complete' }, + IPC_CHANNELS: { + FILES: { PERFORM_OPERATION: 'files:perform-operation' } + } +})); + +const mockPreview = jest.fn(); +jest.mock('../src/renderer/components/UndoRedoSystem', () => ({ + createOrganizeBatchAction: jest.fn((label, operations, callbacks, options) => ({ + type: 'organize-batch', + label, + operations, + callbacks, + options + })), + previewBatchOrganize: (...args) => mockPreview(...args) +})); + +const { + findInPlaceMoves, + applyInPlaceChoices, + useOrganization +} = require('../src/renderer/phases/organize/useOrganization'); + +const INPUT_PATH = '/Users/alex/Strato Input'; +const input = (extra = {}) => ({ id: 'in', name: 'Strato Input', path: INPUT_PATH, ...extra }); +const BING = { id: 'bing', name: 'Bing data', path: '/Users/alex/Sorted/Bing data' }; + +const file = (path, category) => { + const name = path.split(/[\\/]/).pop(); + return { path, name, analysis: { category, suggestedName: name } }; +}; + +const staying = file(`${INPUT_PATH}/keywords.csv`, 'Strato Input'); +const leaving = file(`${INPUT_PATH}/bing.csv`, 'Bing data'); + +function options({ smartFolders, unprocessedFiles = [staying, leaving], ...overrides } = {}) { + const folders = smartFolders || [input(), BING]; + return { + unprocessedFiles, + editingFiles: {}, + getFileWithEdits: jest.fn((f) => f), + findSmartFolderForCategory: jest.fn( + (label) => + folders.find((folder) => folder.name.toLowerCase() === String(label).toLowerCase()) || null + ), + defaultLocation: '/Users/alex/Documents', + smartFolders: folders, + analysisResults: [], + markFilesAsProcessed: jest.fn(), + unmarkFilesAsProcessed: jest.fn(), + actions: { setPhaseData: jest.fn(), advancePhase: jest.fn() }, + phaseData: { organizedFiles: [] }, + addNotification: jest.fn(), + dispatch: jest.fn(), + executeAction: jest.fn(async (action) => { + const result = { + success: true, + results: action.operations.map((op) => ({ success: true, ...op })) + }; + action?.callbacks?.onExecute?.(result); + return result; + }), + addOrganizedFiles: jest.fn(), + removeOrganizedFiles: jest.fn(), + setOrganizingState: jest.fn(), + setLastOrganizeRun: jest.fn(), + setLastOrganizeRunFiles: jest.fn(), + rememberSmartFolderChoice: jest.fn(async () => true), + ...overrides + }; +} + +const sentSources = (opts) => + opts.executeAction.mock.calls.flatMap(([action]) => action.operations.map((op) => op.source)); + +beforeEach(() => { + jest.clearAllMocks(); + global.window = global.window || {}; + global.window.electronAPI = { + events: { onOperationProgress: jest.fn(() => jest.fn()) }, + files: { normalizePath: jest.fn((p) => p) } + }; +}); + +describe('the choice kept on a smart folder', () => { + const planFor = (smartFolders, files = [staying]) => { + const { unprocessedFiles: _u, ...inputs } = options({ smartFolders }); + return { ...inputs, files }; + }; + + test('is read from the folder, and is null until the user chooses', () => { + expect(findInPlaceMoves(planFor([input(), BING]))[0].choice).toBeNull(); + expect(findInPlaceMoves(planFor([input({ inPlaceFiles: 'skip' }), BING]))[0].choice).toBe( + 'skip' + ); + expect(findInPlaceMoves(planFor([input({ inPlaceFiles: 'leave' }), BING]))[0].choice).toBe( + 'leave' + ); + // Anything else saved there is no choice. + expect(findInPlaceMoves(planFor([input({ inPlaceFiles: 'yes' }), BING]))[0].choice).toBeNull(); + }); + + test('applies to a file however its folder is spelled: separators, case and a trailing slash', () => { + const folder = { + id: 'in', + name: 'Strato Input', + path: 'c:/users/alex/strato input/', + inPlaceFiles: 'skip' + }; + const filed = file('C:\\Users\\Alex\\Strato Input\\a.csv', 'Strato Input'); + const { unprocessedFiles: _u, ...inputs } = options({ smartFolders: [folder] }); + + const [move] = findInPlaceMoves({ + ...inputs, + files: [filed], + defaultLocation: 'C:\\Users\\Alex\\Documents' + }); + expect(move).toEqual(expect.objectContaining({ folderId: 'in', choice: 'skip' })); + }); + + test('applies on macOS to a folder name the file system hands back decomposed', () => { + const folder = { + id: 'cafe', + name: 'Café', + path: '/Users/alex/Cafe\u0301', + inPlaceFiles: 'leave' + }; + const filed = file('/Users/alex/Caf\u00e9/menu.pdf', 'Café'); + const { unprocessedFiles: _u, ...inputs } = options({ smartFolders: [folder] }); + + const [move] = findInPlaceMoves({ ...inputs, files: [filed] }); + expect(move).toEqual(expect.objectContaining({ folderId: 'cafe', choice: 'leave' })); + }); + + test('a file in a folder inside a smart folder moves up to it, so nothing is asked', () => { + const nested = file(`${INPUT_PATH}/2026/keywords.csv`, 'Strato Input'); + expect(findInPlaceMoves(planFor([input(), BING], [nested]))).toEqual([]); + }); + + test('a choice made on screen wins until the saved list shows it, and null asks again', () => { + const moves = findInPlaceMoves(planFor([input({ inPlaceFiles: 'skip' }), BING])); + const key = moves[0].folderKey; + + expect(applyInPlaceChoices(moves, new Map([[key, 'leave']]))[0].choice).toBe('leave'); + expect(applyInPlaceChoices(moves, new Map([[key, null]]))[0].choice).toBeNull(); + expect(applyInPlaceChoices(moves, new Map([['/elsewhere', 'leave']]))[0].choice).toBe('skip'); + }); +}); + +describe('Move with files that would stay where they are', () => { + async function move(opts) { + const rendered = renderHook(() => useOrganization(opts)); + await act(async () => { + await rendered.result.current.handleOrganizeFiles(); + }); + return rendered; + } + + test('does nothing until the user chooses, and keeps the last run report', async () => { + const opts = options(); + await move(opts); + + expect(opts.executeAction).not.toHaveBeenCalled(); + expect(opts.setLastOrganizeRun).not.toHaveBeenCalled(); + expect(opts.setLastOrganizeRunFiles).not.toHaveBeenCalled(); + expect(opts.addNotification).toHaveBeenCalledWith( + expect.stringContaining('Choose whether to leave them there or skip them'), + 'warning', + 6000, + 'organize-in-place-choice' + ); + }); + + test('a folder chosen as "leave" keeps its files in the run', async () => { + const opts = options({ smartFolders: [input({ inPlaceFiles: 'leave' }), BING] }); + await move(opts); + + expect(sentSources(opts)).toEqual([staying.path, leaving.path]); + }); + + test('a folder chosen as "skip" leaves its files out of the run; the others move', async () => { + const opts = options({ smartFolders: [input({ inPlaceFiles: 'skip' }), BING] }); + await move(opts); + + expect(sentSources(opts)).toEqual([leaving.path]); + }); + + test('when every file is skipped nothing runs, and the user is told why', async () => { + const opts = options({ + smartFolders: [input({ inPlaceFiles: 'skip' }), BING], + unprocessedFiles: [staying] + }); + await move(opts); + + expect(opts.executeAction).not.toHaveBeenCalled(); + expect(opts.addNotification).toHaveBeenCalledWith( + 'You chose to skip every one of these files, so nothing was moved.', + 'info', + 4000, + 'organize-in-place-choice' + ); + }); + + test('a queue with no such files moves as before, without asking', async () => { + const opts = options({ unprocessedFiles: [leaving] }); + await move(opts); + + expect(sentSources(opts)).toEqual([leaving.path]); + }); + + test('choosing on screen unlocks Move at once and is kept on the smart folder', async () => { + const opts = options(); + const rendered = renderHook(() => useOrganization(opts)); + expect(rendered.result.current.inPlaceMoves[0].choice).toBeNull(); + + await act(async () => { + await rendered.result.current.setInPlaceChoice('skip'); + }); + + expect(rendered.result.current.inPlaceMoves[0].choice).toBe('skip'); + expect(opts.rememberSmartFolderChoice).toHaveBeenCalledWith(['in'], { inPlaceFiles: 'skip' }); + + await act(async () => { + await rendered.result.current.handleOrganizeFiles(); + }); + expect(sentSources(opts)).toEqual([leaving.path]); + }); + + test('"change this choice" asks again: Move stops until the user chooses', async () => { + const opts = options({ smartFolders: [input({ inPlaceFiles: 'leave' }), BING] }); + const rendered = renderHook(() => useOrganization(opts)); + + await act(async () => { + await rendered.result.current.setInPlaceChoice(null); + }); + + expect(rendered.result.current.inPlaceMoves[0].choice).toBeNull(); + expect(opts.rememberSmartFolderChoice).toHaveBeenCalledWith(['in'], { inPlaceFiles: null }); + await act(async () => { + await rendered.result.current.handleOrganizeFiles(); + }); + expect(opts.executeAction).not.toHaveBeenCalled(); + }); + + test('a choice that could not be saved still applies now, and the user is told', async () => { + const opts = options({ rememberSmartFolderChoice: jest.fn(async () => false) }); + const rendered = renderHook(() => useOrganization(opts)); + + await act(async () => { + await rendered.result.current.setInPlaceChoice('leave'); + }); + + expect(rendered.result.current.inPlaceMoves[0].choice).toBe('leave'); + expect(opts.addNotification).toHaveBeenCalledWith( + 'Your choice was not saved. StratoSort will ask again next time.', + 'warning', + 5000, + 'organize-in-place-save' + ); + }); + + test('the preview leaves skipped files out', async () => { + mockPreview.mockResolvedValue({ success: true, wouldMove: [] }); + const opts = options({ smartFolders: [input({ inPlaceFiles: 'skip' }), BING] }); + const rendered = renderHook(() => useOrganization(opts)); + + await act(async () => { + await rendered.result.current.handlePreviewOrganize(); + }); + + const [ops] = mockPreview.mock.calls[0]; + expect(ops.map((op) => op.source)).toEqual([leaving.path]); + }); +}); diff --git a/test/useOrganization.inPlaceMoves.test.js b/test/useOrganization.inPlaceMoves.test.js index e2acb812..818fa16d 100644 --- a/test/useOrganization.inPlaceMoves.test.js +++ b/test/useOrganization.inPlaceMoves.test.js @@ -34,7 +34,7 @@ const { findInPlaceMoves, useOrganization } = require('../src/renderer/phases/organize/useOrganization'); -const { toCanonicalPath } = require('../src/renderer/utils/canonicalPath'); +const { toCanonicalPath, toFolderKey } = require('../src/renderer/utils/canonicalPath'); const INPUT = { id: 'in', name: 'Strato Input', path: '/Users/alex/Strato Input' }; const BING = { id: 'bing', name: 'Bing data', path: '/Users/alex/Sorted/Bing data' }; @@ -71,9 +71,28 @@ describe('findInPlaceMoves', () => { const renamed = file('/Users/alex/Strato Input/a.csv', 'Strato Input', 'Keywords 2025.csv'); const untouched = file('/Users/alex/Strato Input/b.csv', 'Strato Input'); + // Each one names its smart folder, so the user's choice about such files + // (leave them there or skip them) can be read and kept there. + const where = { + folderId: 'in', + folderKey: toFolderKey('/Users/alex/Strato Input'), + choice: null + }; expect(findInPlaceMoves(plan([renamed, untouched]))).toEqual([ - { sourcePath: renamed.path, fileName: 'a.csv', folderName: 'Strato Input', renamed: true }, - { sourcePath: untouched.path, fileName: 'b.csv', folderName: 'Strato Input', renamed: false } + { + sourcePath: renamed.path, + fileName: 'a.csv', + folderName: 'Strato Input', + renamed: true, + ...where + }, + { + sourcePath: untouched.path, + fileName: 'b.csv', + folderName: 'Strato Input', + renamed: false, + ...where + } ]); }); @@ -108,7 +127,12 @@ describe('findInPlaceMoves', () => { const filed = file('/Users/alex/Documents/Reports/q3.pdf', 'Reports'); expect(findInPlaceMoves(plan([filed], { smartFolders: [reports] }))).toEqual([ - { sourcePath: filed.path, fileName: 'q3.pdf', folderName: 'Reports', renamed: false } + expect.objectContaining({ + sourcePath: filed.path, + fileName: 'q3.pdf', + folderName: 'Reports', + renamed: false + }) ]); }); @@ -116,7 +140,12 @@ describe('findInPlaceMoves', () => { const stray = file('/Users/alex/Documents/Uncategorized/notes.txt', 'Recipes'); expect(findInPlaceMoves(plan([stray]))).toEqual([ - { sourcePath: stray.path, fileName: 'notes.txt', folderName: 'Uncategorized', renamed: false } + expect.objectContaining({ + sourcePath: stray.path, + fileName: 'notes.txt', + folderName: 'Uncategorized', + renamed: false + }) ]); }); @@ -129,7 +158,12 @@ describe('findInPlaceMoves', () => { plan([filed], { smartFolders, defaultLocation: 'C:\\Users\\Alex\\Documents' }) ) ).toEqual([ - { sourcePath: filed.path, fileName: 'a.csv', folderName: 'Strato Input', renamed: false } + expect.objectContaining({ + sourcePath: filed.path, + fileName: 'a.csv', + folderName: 'Strato Input', + renamed: false + }) ]); }); @@ -180,7 +214,12 @@ describe('useOrganization: inPlaceMoves', () => { const { result } = renderHook(() => useOrganization(options)); expect(result.current.inPlaceMoves).toEqual([ - { sourcePath: ready.path, fileName: 'a.csv', folderName: 'Strato Input', renamed: false } + expect.objectContaining({ + sourcePath: ready.path, + fileName: 'a.csv', + folderName: 'Strato Input', + renamed: false + }) ]); }); From eb894c1da467a279b8cdd72ccbdbe8ccc8559601 Mon Sep 17 00:00:00 2001 From: Levy Tate <78818969+iLevyTate@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:43:10 -0500 Subject: [PATCH 03/10] feat(smart-folders): try a few files before a large batch, and flag results from old folder definitions Sample first: when 20 or more added files are about to be analyzed and the current Smart Folder definitions (a hash of names, descriptions and paths) have not been tried, Discover offers to analyze 5 files first, spread across the file types in the batch. The sample uses the normal analysis run and the Organize ranking, shows where each file would go with its confidence and reason, and moves nothing. The user continues with the rest (sample files count as analyzed) or goes to Setup to edit folders. Skipping or reviewing a sample is remembered per definition set in ui.smartFolderSampleHash, so the offer returns only after a folder changes. Stale results: each analysis is stamped with the definitions it was made under. Organize already re-ranks files with the current folders; a file whose folder still comes from the old analysis label is listed with one click to analyze it again, and Move asks before moving it. Documented in organizeDestination.js and docs/USER_GUIDE.md, with a section on writing folder descriptions. --- docs/USER_GUIDE.md | 53 +++ .../components/discover/AnalysisProgress.jsx | 3 +- .../components/discover/SampleRunDialog.jsx | 180 +++++++++++ .../components/organize/ReadyFileItem.jsx | 26 +- .../organize/StaleSuggestionsNotice.jsx | 69 ++++ src/renderer/phases/OrganizePhase.jsx | 89 ++++- .../phases/discover/AnalysisRunProvider.jsx | 32 +- src/renderer/phases/discover/sampleRun.js | 245 ++++++++++++++ src/renderer/phases/discover/useAnalysis.js | 50 ++- src/renderer/phases/discover/useSampleRun.js | 214 ++++++++++++ .../phases/organize/organizeDestination.js | 104 ++++++ src/renderer/store/index.js | 18 +- .../store/middleware/persistenceMiddleware.js | 12 + src/renderer/store/slices/analysisSlice.js | 5 +- src/renderer/store/slices/uiSlice.js | 17 +- src/shared/smartFolderDefinitions.js | 74 +++++ .../AnalysisRunProvider.sample.test.js | 127 ++++++++ test/components/AnalysisRunProvider.test.js | 11 +- test/components/OrganizePhase.test.js | 138 +++++++- test/components/SampleRunDialog.test.js | 141 ++++++++ .../components/StaleSuggestionsNotice.test.js | 46 +++ test/organizeDestination.test.js | 82 ++++- test/sampleRun.test.js | 256 +++++++++++++++ test/smartFolderDefinitions.test.js | 73 +++++ .../smartFolderSampleHash.persistence.test.js | 149 +++++++++ test/useAnalysis.sampleRun.test.js | 195 +++++++++++ test/useSampleRun.test.js | 306 ++++++++++++++++++ 27 files changed, 2656 insertions(+), 59 deletions(-) create mode 100644 src/renderer/components/discover/SampleRunDialog.jsx create mode 100644 src/renderer/components/organize/StaleSuggestionsNotice.jsx create mode 100644 src/renderer/phases/discover/sampleRun.js create mode 100644 src/renderer/phases/discover/useSampleRun.js create mode 100644 src/shared/smartFolderDefinitions.js create mode 100644 test/components/AnalysisRunProvider.sample.test.js create mode 100644 test/components/SampleRunDialog.test.js create mode 100644 test/components/StaleSuggestionsNotice.test.js create mode 100644 test/sampleRun.test.js create mode 100644 test/smartFolderDefinitions.test.js create mode 100644 test/smartFolderSampleHash.persistence.test.js create mode 100644 test/useAnalysis.sampleRun.test.js create mode 100644 test/useSampleRun.test.js diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index d53df9b8..3bd0f42e 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -96,6 +96,59 @@ content against these descriptions to decide where it fits best. | **Project Specs** | "Requirements, architecture docs, and project briefs." | | **Screenshots** | "Screen captures, app screenshots, and UI mockups." | +### Writing folder descriptions + +The AI reads each folder's name and description, and nothing else, to decide where a file goes, so a +few words of description decide where many files land. It is normal to find a description too vague +only after seeing where files land; that is what the sample below is for. + +What to include: + +- The kinds of files that belong here, in the words those files use ("invoice", "W-2", "lease"). +- Who they come from or are about, when that tells folders apart ("from suppliers", "for the Garden + Street house"). +- The most telling words first. Only the first part of a long description reaches the AI, and Setup + says so when a description is cut off. + +What to leave out: + +- Words every folder could use, such as "files", "documents", "stuff" or "misc". +- Files that belong in another folder. If two folders are close, say what goes in this one and not + the other ("Invoices from suppliers. Not receipts for things we bought."). + +Example: a folder named **Invoices** with the description "Documents" leaves the AI with the name +alone. "Bills and invoices from suppliers, with an amount due and a due date. Not receipts." tells +it what belongs there and what does not. + +Setup shows a hint under a description that is missing, short, cut off, or reads like another +folder's. A hint never stops you from saving. + +### Try a few files first + +When you add 20 or more files and your Smart Folders have not been tried since they last changed, +StratoSort asks before it starts: **Analyze 5 files first**, or **Analyze all** of them. The sample +takes files of each type in the batch. Nothing is moved. + +When the sample is done, you see where each file would go, the confidence where there is one, and +why. Then choose **Analyze the other files** to go on, or **Edit Smart Folders** to change a +description first. The sample files count as analyzed, so the rest of the batch does not analyze +them again. Files you have not analyzed yet stay in the queue on Discover. + +StratoSort remembers that you saw a sample, or chose **Analyze all**, for the folders you have now. +It asks again only after you change a folder's name, description or location. + +### When you change Smart Folders after files were analyzed + +When you change a Smart Folder, **Organize** ranks the files waiting there again with the current +folders, without analyzing them again. Most files get a current folder that way. + +Some files can still point to a folder picked from your old Smart Folders: the folder the analysis +picked when nothing else ranks the file. StratoSort does not move those unnoticed. **Organize** +lists them above the files, with **Analyze again** to analyze only those files with the current +folders, and **Move** asks before it moves any of them. StratoSort does not analyze them again on +its own, because on a large batch that takes as long as the first run did. A folder you chose +yourself on a card is never treated as out of date. + ### Teaching StratoSort When you change a file's folder on **Organize**, its card says what StratoSort learns from that. diff --git a/src/renderer/components/discover/AnalysisProgress.jsx b/src/renderer/components/discover/AnalysisProgress.jsx index 03636149..0959ceaf 100644 --- a/src/renderer/components/discover/AnalysisProgress.jsx +++ b/src/renderer/components/discover/AnalysisProgress.jsx @@ -11,7 +11,8 @@ const ORIGIN_TEXT = { queued: 'Files you added while the last run was going', resume: 'Continuing a run that was interrupted', retry: 'Trying again the files that failed', - reanalyze: 'Analyzing again because you chose Reanalyze' + reanalyze: 'Analyzing again because you chose Reanalyze', + sample: 'Trying your Smart Folders on a few files first' }; const AnalysisProgress = memo(function AnalysisProgress({ diff --git a/src/renderer/components/discover/SampleRunDialog.jsx b/src/renderer/components/discover/SampleRunDialog.jsx new file mode 100644 index 00000000..cbdc3f6b --- /dev/null +++ b/src/renderer/components/discover/SampleRunDialog.jsx @@ -0,0 +1,180 @@ +import React, { memo } from 'react'; +import PropTypes from 'prop-types'; +import { useSelector } from 'react-redux'; +import { ArrowRight, FolderOpen } from 'lucide-react'; +import Modal from '../ui/Modal'; +import { Button, StatusBadge } from '../ui'; +import { Text } from '../ui/Typography'; +import { Inline } from '../layout'; +import { selectRedactPaths } from '../../store/selectors'; +import { formatDisplayFileName } from '../../utils/pathDisplay'; + +const plural = (n, word) => `${n} ${word}${n === 1 ? '' : 's'}`; + +/** + * The sample-first dialog (useSampleRun): offer, progress, then where each + * sample file would go and why. Nothing in it moves a file. + */ +function SampleRunDialog({ + session, + progress = null, + onSample, + onSkip, + onStop, + onContinue, + onEditFolders, + onClose +}) { + const redactPaths = Boolean(useSelector(selectRedactPaths)); + if (!session) return null; + + const total = session.files.length; + const sampleCount = session.picked.length; + const restCount = Math.max(0, total - sampleCount); + const nameLabel = (name) => formatDisplayFileName(name, { redact: redactPaths }); + + if (session.stage === 'offer') { + return ( + + + + + } + > +
+ {session.hintCount > 0 && ( + + {`${plural(session.hintCount, 'Smart Folder')} ${ + session.hintCount === 1 ? 'has' : 'have' + } a description hint in Setup. The sample shows whether it matters.`} + + )} +
+ + The sample takes files of each type in the batch: + +
    + {session.picked.map((file) => ( +
  • + + {nameLabel(file.name || String(file.path).split(/[\\/]/).pop())} + +
  • + ))} +
+
+
+
+ ); + } + + if (session.stage === 'running') { + const done = Math.min(Number(progress?.current) || 0, sampleCount); + return ( + + + + } + > + + {`${done} of ${sampleCount} done`} + + + ); + } + + return ( + + + + + } + > +
    + {session.rows.map((row) => ( +
  • +
    + + {nameLabel(row.name)} + + + + + {row.analyzed ? row.folder || 'Uncategorized' : 'Not analyzed'} + + {row.score !== null && ( + + {`${Math.round(row.score * 100)}% confidence`} + + )} +
    + + {row.reason} + +
  • + ))} +
+
+ ); +} + +SampleRunDialog.propTypes = { + session: PropTypes.shape({ + stage: PropTypes.oneOf(['offer', 'running', 'review']).isRequired, + files: PropTypes.array.isRequired, + picked: PropTypes.array.isRequired, + hintCount: PropTypes.number, + rows: PropTypes.array + }), + progress: PropTypes.shape({ current: PropTypes.number, total: PropTypes.number }), + onSample: PropTypes.func.isRequired, + onSkip: PropTypes.func.isRequired, + onStop: PropTypes.func.isRequired, + onContinue: PropTypes.func.isRequired, + onEditFolders: PropTypes.func.isRequired, + onClose: PropTypes.func.isRequired +}; + +export default memo(SampleRunDialog); diff --git a/src/renderer/components/organize/ReadyFileItem.jsx b/src/renderer/components/organize/ReadyFileItem.jsx index 03d1a3c0..59bcb9a9 100644 --- a/src/renderer/components/organize/ReadyFileItem.jsx +++ b/src/renderer/components/organize/ReadyFileItem.jsx @@ -14,6 +14,7 @@ import { getSelectableSmartFolders, isDefaultCategoryName } from '../../../shared/categoryFolderMatching'; +import { describeRouting } from '../../phases/organize/organizeDestination'; function ReadyFileItem({ file, @@ -100,30 +101,7 @@ function ReadyFileItem({ } return names; }, [engine, selectableFolders]); - const routingNote = useMemo(() => { - if (!routing) return null; - if (routing.source === 'engine' && engine) { - return engine.explanation || 'Ranked by what the file contains'; - } - if (routing.source === 'user') { - return engine?.folder && engine.folder !== routing.folder?.name - ? `Your choice (suggested: ${engine.folder})` - : 'Your choice'; - } - if (routing.source === 'name') { - // A name match is either the analysis label or the file's own name/type. - const fromLabel = - typeof category === 'string' && - category.trim().toLowerCase() === - String(routing.folder?.name || '') - .trim() - .toLowerCase(); - return fromLabel - ? 'Matched by the label the analysis gave this file' - : 'Matched by folder name'; - } - return null; - }, [routing, engine, category]); + const routingNote = useMemo(() => describeRouting(routing, category), [routing, category]); // A changed suggestion is learned from the move, but only for files of the // same type and label, and only after a few (the engine weighs it against // what the file contains). Say so, so the user knows what to expect. diff --git a/src/renderer/components/organize/StaleSuggestionsNotice.jsx b/src/renderer/components/organize/StaleSuggestionsNotice.jsx new file mode 100644 index 00000000..29ded857 --- /dev/null +++ b/src/renderer/components/organize/StaleSuggestionsNotice.jsx @@ -0,0 +1,69 @@ +import React, { memo } from 'react'; +import PropTypes from 'prop-types'; +import { AlertTriangle } from 'lucide-react'; +import { Button } from '../ui'; +import { Text } from '../ui/Typography'; + +/** + * Files whose folder still comes from Smart Folder definitions that have + * since changed (organizeDestination isStaleDestination). One button analyzes + * them again with the current folders. While the re-rank with the new + * folders is still running, it says so instead. + */ +function StaleSuggestionsNotice({ + count, + isRanking = false, + isAnalyzing = false, + onRefresh, + className = '' +}) { + if (!count) return null; + const one = count === 1; + const files = `${count} ${one ? 'file' : 'files'}`; + return ( +
+ +
+ + {`Your Smart Folders changed after ${one ? 'this file was' : 'these files were'} analyzed`} + + + {isRanking + ? `Checking ${files} against your current Smart Folders.` + : `${files} still ${one ? 'goes' : 'go'} where your old Smart Folders sent ${ + one ? 'it' : 'them' + }. Analyze ${one ? 'it' : 'them'} again to sort ${ + one ? 'it' : 'them' + } with the current folders.`} + +
+ {!isRanking && ( + + )} +
+ ); +} + +StaleSuggestionsNotice.propTypes = { + count: PropTypes.number.isRequired, + isRanking: PropTypes.bool, + isAnalyzing: PropTypes.bool, + onRefresh: PropTypes.func.isRequired, + className: PropTypes.string +}; + +export default memo(StaleSuggestionsNotice); diff --git a/src/renderer/phases/OrganizePhase.jsx b/src/renderer/phases/OrganizePhase.jsx index 15e344eb..a260cfa8 100644 --- a/src/renderer/phases/OrganizePhase.jsx +++ b/src/renderer/phases/OrganizePhase.jsx @@ -58,7 +58,10 @@ import { } from './organize/useFileEditing'; import { useOrganization } from './organize/useOrganization'; import { useOrganizeSuggestions } from './organize/useOrganizeSuggestions'; -import { resolveOrganizeDestination } from './organize/organizeDestination'; +import { findStaleFiles, resolveOrganizeDestination } from './organize/organizeDestination'; +import { getFolderDefinitionsHash } from '../../shared/smartFolderDefinitions'; +import StaleSuggestionsNotice from '../components/organize/StaleSuggestionsNotice'; +import { useConfirmDialog } from '../hooks/useConfirmDialog'; import { useOrganizeNotificationFocus } from './organize/useOrganizeNotificationFocus'; import { setFileStates as setFileStatesAction } from '../store/slices/filesSlice'; import { updateAnalysisResult, updateAnalysisResultsByPaths } from '../store/slices/analysisSlice'; @@ -272,7 +275,16 @@ function OrganizePhase() { // D1: the suggestion engine ranks each ready file's destination; name // matching stays as the fallback when it has nothing. - const { suggestionsByPath } = useOrganizeSuggestions(baseUnprocessedFiles, safeSmartFolders); + const { suggestionsByPath, isRanking } = useOrganizeSuggestions( + baseUnprocessedFiles, + safeSmartFolders + ); + // The Smart Folder definitions now; each analysis carries the ones it was + // made under (analysis.smartFolderHash). See isStaleDestination. + const folderDefinitionsHash = useMemo( + () => getFolderDefinitionsHash(safeSmartFolders), + [safeSmartFolders] + ); const resolveDestination = useCallback( (file) => { const key = toCanonicalPath(file?.path); @@ -509,6 +521,44 @@ function OrganizePhase() { return undefined; }, [isOrganizing, isPaused]); + // Files whose folder still comes from Smart Folder definitions that have + // changed since they were analyzed (organizeDestination isStaleDestination). + // They are listed with a one-click Analyze again, and Move asks first. + const staleFiles = useMemo( + () => findStaleFiles(unprocessedFiles, resolveDestination, folderDefinitionsHash), + [unprocessedFiles, resolveDestination, folderDefinitionsHash] + ); + const { showConfirm, confirmDialog } = useConfirmDialog(); + const refreshStaleFiles = useCallback(() => { + reanalyzeFiles?.(staleFiles.map((file) => file.path)); + }, [reanalyzeFiles, staleFiles]); + + // Runs `proceed` at once when `files` holds no stale file; otherwise only + // after the user chooses to move anyway. + const moveAfterStaleCheck = useCallback( + (files, proceed) => { + const stale = findStaleFiles(files, resolveDestination, folderDefinitionsHash); + if (stale.length === 0) { + proceed(); + return; + } + const one = stale.length === 1; + void (async () => { + const moveAnyway = await showConfirm({ + title: 'Move files sorted by your old Smart Folders?', + message: `${stale.length} of these files ${one ? 'goes' : 'go'} where your old Smart Folders sent ${ + one ? 'it' : 'them' + }. To use your current folders, cancel and analyze ${one ? 'it' : 'them'} again.`, + confirmText: 'Move anyway', + cancelText: 'Cancel', + variant: 'warning' + }); + if (moveAnyway) proceed(); + })(); + }, + [resolveDestination, folderDefinitionsHash, showConfirm] + ); + const approveSelectedFiles = useCallback(() => { if (selectedFiles.size === 0) return; const filesToProcess = unprocessedFiles.filter( @@ -520,9 +570,18 @@ function OrganizePhase() { return; } - handleOrganizeFiles(filesToProcess); - setSelectedFiles(new Set()); - }, [selectedFiles, unprocessedFiles, addNotification, handleOrganizeFiles, setSelectedFiles]); + moveAfterStaleCheck(filesToProcess, () => { + handleOrganizeFiles(filesToProcess); + setSelectedFiles(new Set()); + }); + }, [ + selectedFiles, + unprocessedFiles, + addNotification, + handleOrganizeFiles, + setSelectedFiles, + moveAfterStaleCheck + ]); // Not license-gated (CP-19): moving already-analyzed files is the user's // own file operation, not new AI work. The gate applies to Analyze and @@ -531,9 +590,15 @@ function OrganizePhase() { if (selectedFiles.size > 0) { approveSelectedFiles(); } else { - handleOrganizeFiles(); + moveAfterStaleCheck(unprocessedFiles, () => handleOrganizeFiles()); } - }, [selectedFiles.size, approveSelectedFiles, handleOrganizeFiles]); + }, [ + selectedFiles.size, + approveSelectedFiles, + handleOrganizeFiles, + moveAfterStaleCheck, + unprocessedFiles + ]); // The plan checks every file on disk, so a large queue takes a moment: the // button shows it is working, and a second press waits for the first. @@ -811,6 +876,13 @@ function OrganizePhase() { ) : (
+
+ + {/* Asks before moving files sorted by old Smart Folder definitions */} + {confirmDialog} ); } diff --git a/src/renderer/phases/discover/AnalysisRunProvider.jsx b/src/renderer/phases/discover/AnalysisRunProvider.jsx index 71b22771..28dc6ab7 100644 --- a/src/renderer/phases/discover/AnalysisRunProvider.jsx +++ b/src/renderer/phases/discover/AnalysisRunProvider.jsx @@ -6,6 +6,8 @@ import { createLogger } from '../../../shared/logger'; import { useNotification } from '../../contexts/NotificationContext'; import { useDiscoverState } from './useDiscoverState'; import { useAnalysis } from './useAnalysis'; +import { useSampleRun } from './useSampleRun'; +import SampleRunDialog from '../../components/discover/SampleRunDialog'; const logger = createLogger('AnalysisRunProvider'); @@ -58,7 +60,7 @@ export function AnalysisRunProvider({ children }) { ); const { - analyzeFiles, + analyzeFiles: runAnalyzeFiles, cancelAnalysis, clearAnalysisQueue, retryFailedFiles, @@ -88,6 +90,16 @@ export function AnalysisRunProvider({ children }) { getCurrentPhase }); + // Before a large batch, offer to try a few files first. Every caller that + // starts analysis for files the user added goes through this analyzeFiles; + // runs the analysis starts itself (queued, resumed, retried) do not. + const sampleRun = useSampleRun({ + analyzeFiles: runAnalyzeFiles, + cancelAnalysis, + advancePhase: actions?.advancePhase + }); + const analyzeFiles = sampleRun.analyzeFiles; + // Stuck-run checks. They belong with the run, not with the Discover // screen: a run that keeps going while the user is on Organize still // needs its watchdog. They reset through the run, so a retry is not @@ -166,7 +178,23 @@ export function AnalysisRunProvider({ children }) { ] ); - return {children}; + return ( + + {children} + {sampleRun.session && ( + + )} + + ); } AnalysisRunProvider.propTypes = { diff --git a/src/renderer/phases/discover/sampleRun.js b/src/renderer/phases/discover/sampleRun.js new file mode 100644 index 00000000..21431d7e --- /dev/null +++ b/src/renderer/phases/discover/sampleRun.js @@ -0,0 +1,245 @@ +// @ts-check +/** + * Sample first: before a large batch, offer to analyze a few files and show + * where each would go, so vague Smart Folders show up after minutes instead + * of after the whole run. + * + * The sample uses the normal analysis run (origin 'sample') and the same + * ranking Organize uses (suggestions.getBatchSuggestions, then + * resolveOrganizeDestination), so what the sample shows is what Organize + * would do. Nothing is moved. Sample results are ordinary results: the rest + * of the batch does not analyze those files again. + * + * The offer is remembered per Smart Folder definition set + * (getFolderDefinitionsHash: names, descriptions and paths). Once the user + * has seen a sample, or chosen to skip it, for a set, it is not offered again + * until a folder's name, description or path changes. + * + * @module discover/sampleRun + */ + +import { getFolderDefinitionsHash } from '../../../shared/smartFolderDefinitions'; +import { checkFolderDescription } from '../../../shared/smartFolderDescriptionCheck'; +import { + findSmartFolderForFile, + findSmartFolderForLabel +} from '../../../shared/categoryFolderMatching'; +import { + countDestinationFolders, + isUncategorizedBucket +} from '../../utils/smartFolderDestinations'; +import { toCanonicalPath } from '../../utils/canonicalPath'; +import { RANKING_OPTIONS, toSuggestionFile } from '../organize/useOrganizeSuggestions'; +import { + describeRouting, + normalizeEngineEntry, + resolveOrganizeDestination, + routingScore +} from '../organize/organizeDestination'; + +/** A batch of this many files or more gets the sample offer. */ +export const SAMPLE_OFFER_MIN_FILES = 20; +/** How many files the sample analyzes. */ +export const SAMPLE_SIZE = 5; + +/** + * @typedef {Object} SampleRow + * @property {string} path + * @property {string} name + * @property {boolean} analyzed + * @property {string|null} folder - Smart Folder name, null for Uncategorized + * @property {number|null} score - 0..1, only where it measures the folder's fit + * @property {string} reason + */ + +/** + * The files in `files` with a path, once each (by canonical path). + * @param {any[]} files + * @returns {any[]} + */ +export function uniqueFiles(files) { + const seen = new Set(); + const out = []; + for (const file of Array.isArray(files) ? files : []) { + const key = toCanonicalPath(file?.path); + if (!key || seen.has(key)) continue; + seen.add(key); + out.push(file); + } + return out; +} + +/** + * Whether to offer a sample before analyzing `files`. + * + * @param {{ files: any[], smartFolders: any, reviewedHash: string|null|undefined, isAnalyzing: boolean }} input + * @returns {boolean} + */ +export function shouldOfferSample({ files, smartFolders, reviewedHash, isAnalyzing }) { + // Files added during a run wait for it; the offer would interrupt nothing. + if (isAnalyzing) return false; + if (uniqueFiles(files).length < SAMPLE_OFFER_MIN_FILES) return false; + // With only the Uncategorized catch-all there is nothing to try. + if (countDestinationFolders(smartFolders) === 0) return false; + const hash = getFolderDefinitionsHash(smartFolders); + return Boolean(hash) && hash !== reviewedHash; +} + +/** + * @param {any} file + * @returns {string} lowercase extension without the dot, '' when none + */ +function extensionOf(file) { + const fromField = typeof file?.extension === 'string' ? file.extension.trim() : ''; + if (fromField) return fromField.replace(/^\./, '').toLowerCase(); + const name = + (typeof file?.name === 'string' && file.name) || + String(file?.path || '') + .split(/[\\/]/) + .pop() || + ''; + const dot = name.lastIndexOf('.'); + return dot > 0 ? name.slice(dot + 1).toLowerCase() : ''; +} + +/** + * Up to `size` files spread across the file types in the batch: one of each + * type in turn, the most common types first, so a batch of PDFs, photos and + * spreadsheets samples all three. + * + * @param {any[]} files + * @param {number} [size] + * @returns {any[]} + */ +export function pickSampleFiles(files, size = SAMPLE_SIZE) { + /** @type {Map} */ + const byType = new Map(); + for (const file of uniqueFiles(files)) { + const ext = extensionOf(file); + const group = byType.get(ext); + if (group) group.push(file); + else byType.set(ext, [file]); + } + const groups = [...byType.entries()] + .sort((a, b) => b[1].length - a[1].length || a[0].localeCompare(b[0])) + .map(([, group]) => group); + const picked = []; + for (let round = 0; picked.length < size; round += 1) { + let tookAny = false; + for (const group of groups) { + if (picked.length >= size) break; + if (round < group.length) { + picked.push(group[round]); + tookAny = true; + } + } + if (!tookAny) break; + } + return picked; +} + +/** + * How many Smart Folders (not the Uncategorized catch-all) have a + * description hint from checkFolderDescription. + * + * @param {any} smartFolders + * @returns {number} + */ +export function countFoldersWithHints(smartFolders) { + const folders = Array.isArray(smartFolders) ? smartFolders.filter(Boolean) : []; + let count = 0; + for (const folder of folders) { + if (isUncategorizedBucket(folder)) continue; + const issues = checkFolderDescription({ + name: folder.name, + description: folder.description, + otherFolders: folders.filter((other) => other !== folder) + }); + if (issues.length > 0) count += 1; + } + return count; +} + +/** + * @param {any} file + * @returns {string} + */ +function displayName(file) { + return ( + (typeof file?.name === 'string' && file.name) || + String(file?.path || '') + .split(/[\\/]/) + .pop() || + '' + ); +} + +/** + * Where each sample file would go and why, the way Organize decides it: the + * suggestion engine's ranking first, name matching when it has nothing. + * + * @param {{ files: any[], smartFolders: any[], getBatchSuggestions?: Function|null }} input + * files: the sample files merged with their analysis results + * @returns {Promise} + */ +export async function rankSampleFiles({ files, smartFolders, getBatchSuggestions }) { + const folders = Array.isArray(smartFolders) ? smartFolders : []; + const list = Array.isArray(files) ? files : []; + const analyzed = list.filter((file) => file?.analysis && !file?.error); + + /** @type {Map} */ + const engineByPath = new Map(); + if (analyzed.length > 0 && typeof getBatchSuggestions === 'function') { + try { + const response = await getBatchSuggestions(analyzed.map(toSuggestionFile), RANKING_OPTIONS); + const groups = response?.success && Array.isArray(response.groups) ? response.groups : []; + for (const group of groups) { + for (const entry of Array.isArray(group?.files) ? group.files : []) { + const key = toCanonicalPath(entry?.path); + const normalized = normalizeEngineEntry(entry); + if (key && normalized) engineByPath.set(key, normalized); + } + } + } catch { + // The ranking is optional here as in Organize: name matching decides. + } + } + + const byLabel = (/** @type {any} */ label) => + typeof label === 'string' && label.trim() ? findSmartFolderForLabel(label, folders) : null; + const byFile = (/** @type {any} */ file) => findSmartFolderForFile(file, folders); + + return list.map((file) => { + const path = String(file?.path || ''); + const name = displayName(file); + if (!file?.analysis || file?.error) { + /** @type {SampleRow} */ + const notAnalyzed = { + path, + name, + analyzed: false, + folder: null, + score: null, + reason: 'This file could not be analyzed.' + }; + return notAnalyzed; + } + const destination = resolveOrganizeDestination({ + fileWithEdits: file, + edits: null, + engineSuggestion: engineByPath.get(toCanonicalPath(path)) || null, + smartFolders: folders, + findSmartFolderForCategory: byLabel, + findSmartFolderForFile: byFile + }); + return { + path, + name, + analyzed: true, + folder: destination.folder?.name || null, + score: routingScore(destination), + reason: + describeRouting(destination, file.analysis.category) || 'No Smart Folder fits this file.' + }; + }); +} diff --git a/src/renderer/phases/discover/useAnalysis.js b/src/renderer/phases/discover/useAnalysis.js index 858fbc13..19799ec3 100644 --- a/src/renderer/phases/discover/useAnalysis.js +++ b/src/renderer/phases/discover/useAnalysis.js @@ -45,6 +45,7 @@ import { runBatchPathWhenFree } from './batchBusy'; import { setAnalysisRunBase, clearAnalysisRunBase } from '../../store/analysisRunBase'; +import { getFolderDefinitionsHash } from '../../../shared/smartFolderDefinitions'; const logger = createLogger('DiscoverPhase:Analysis'); const BATCH_ANALYSIS_MIN_FILES = 40; @@ -543,6 +544,10 @@ export function useAnalysis(options = {}) { const lastProgressAtRef = useRef(0); const activeBatchIdRef = useRef(null); const cancelInFlightBatchIdRef = useRef(null); + // The Smart Folder definitions the current run analyzes under, stamped on + // each result as analysis.smartFolderHash. Organize compares it with the + // current folders to find results made under old definitions. + const runFolderHashRef = useRef(''); const invalidateEmbeddingStatsCache = useCallback(() => { try { @@ -910,8 +915,12 @@ export function useAnalysis(options = {}) { if (shouldTreatAsReady) { const baseSuggestedName = analysisForUi.llmSuggestedName || analysisForUi.suggestedName || fileName; + const smartFolderHash = + runFolderHashRef.current || + getFolderDefinitionsHash(/** @type {any} */ (store?.getState?.())?.files?.smartFolders); const enhancedAnalysis = { ...analysisForUi, + ...(smartFolderHash ? { smartFolderHash } : {}), originalSuggestedName: baseSuggestedName, suggestedName: resolveDiscoverSuggestedName({ originalFileName: fileName, @@ -953,7 +962,7 @@ export function useAnalysis(options = {}) { analyzedAt }); }, - [namingSettings, recordAnalysisResult, updateFileState] + [namingSettings, recordAnalysisResult, updateFileState, store] ); /** @@ -1061,11 +1070,15 @@ export function useAnalysis(options = {}) { const analyzeFiles = useCallback( async ( /** @type {any} */ files, - /** @type {{ bypassCache?: boolean, origin?: 'queued' | 'resume' | 'retry' }} */ runOptions = {} + /** @type {{ bypassCache?: boolean, origin?: 'queued' | 'resume' | 'retry' | 'sample' }} */ runOptions = {} ) => { // A reanalysis: every file goes through the per-file request, which // carries bypassCache, and the run ends without auto-advance. const bypassCache = runOptions?.bypassCache === true; + // A sample run (a few files tried before a large batch): its results + // are shown in the sample dialog, so it ends without the completion + // notice and without moving on to Organize. + const isSampleRun = !bypassCache && runOptions?.origin === 'sample'; // Why this run started, shown on the progress card (analysisSlice // runOrigin). A plain call is files the person just added. const runOrigin = bypassCache ? 'reanalyze' : runOptions?.origin || 'added'; @@ -1152,6 +1165,9 @@ export function useAnalysis(options = {}) { const runId = ++analysisRunIdRef.current; const isActiveRun = () => analysisRunIdRef.current === runId; + runFolderHashRef.current = getFolderDefinitionsHash( + /** @type {any} */ (store?.getState?.())?.files?.smartFolders + ); const runGeneration = ++analysisRunGeneration; cancelledBatchSnapshotRef.current = null; batchCompletedPathsRef.current.clear(); @@ -2009,20 +2025,22 @@ export function useAnalysis(options = {}) { ); const successCount = runResults.filter((/** @type {any} */ r) => r.analysis).length; const failureCount = runResults.length - successCount; - showAnalysisCompletionNotification({ - successCount, - failureCount, - addNotification, - actions, - getCurrentPhase, - autoAdvanceTimeoutRef, - // Read at fire time: by then a queued run has either started (store - // says analyzing) or is still waiting in pendingFilesRef. - isMoreAnalysisPending: () => - pendingFilesRef.current.length > 0 || - Boolean(/** @type {any} */ (store.getState())?.analysis?.isAnalyzing), - isReanalysis: bypassCache - }); + if (!isSampleRun) { + showAnalysisCompletionNotification({ + successCount, + failureCount, + addNotification, + actions, + getCurrentPhase, + autoAdvanceTimeoutRef, + // Read at fire time: by then a queued run has either started (store + // says analyzing) or is still waiting in pendingFilesRef. + isMoreAnalysisPending: () => + pendingFilesRef.current.length > 0 || + Boolean(/** @type {any} */ (store.getState())?.analysis?.isAnalyzing), + isReanalysis: bypassCache + }); + } } catch (error) { if (error.message !== 'Analysis cancelled by user') { const { message } = mapErrorToNotification({ error }); diff --git a/src/renderer/phases/discover/useSampleRun.js b/src/renderer/phases/discover/useSampleRun.js new file mode 100644 index 00000000..581749ad --- /dev/null +++ b/src/renderer/phases/discover/useSampleRun.js @@ -0,0 +1,214 @@ +// @ts-check +/** + * useSampleRun: the "try a few files first" step before a large batch. + * + * Wraps the analysis run's analyzeFiles. A plain call (files the user just + * added, or the Analyze button for files waiting in the queue) with + * SAMPLE_OFFER_MIN_FILES or more files, while the current Smart Folder + * definitions have not been tried on a sample, opens the offer instead of + * starting the run. Runs the analysis started itself (queued, resumed, + * retried, reanalyzed) are never held. + * + * Stages: 'offer' (sample or skip), 'running' (the sample is analyzing), + * 'review' (where each sample file would go; continue or edit folders). + * Skipping, or reaching the review, records the definitions hash in + * ui.smartFolderSampleHash (persisted), so the offer returns only after a + * folder's name, description or path changes. + * + * @module discover/useSampleRun + */ + +import { useCallback, useEffect, useRef, useState } from 'react'; +import { useStore } from 'react-redux'; +import { PHASES } from '../../../shared/constants'; +import { createLogger } from '../../../shared/logger'; +import { getFolderDefinitionsHash } from '../../../shared/smartFolderDefinitions'; +import { setSmartFolderSampleHash } from '../../store/slices/uiSlice'; +import { toCanonicalPath } from '../../utils/canonicalPath'; +import { + countFoldersWithHints, + pickSampleFiles, + rankSampleFiles, + shouldOfferSample, + uniqueFiles +} from './sampleRun'; + +const logger = createLogger('useSampleRun'); + +/** + * @typedef {Object} SampleSession + * @property {number} id + * @property {'offer'|'running'|'review'} stage + * @property {any[]} files - every file of the batch, the sample included + * @property {any[]} picked - the sample files + * @property {string} hash - the folder definitions the sample tries + * @property {number} hintCount - folders with a description hint + * @property {import('./sampleRun').SampleRow[]} rows - filled in at 'review' + */ + +/** @typedef {{ started: boolean, reason?: string }} RunStart */ + +/** + * @param {{ + * analyzeFiles: (files: any[], runOptions?: any) => Promise|RunStart, + * cancelAnalysis?: () => void, + * advancePhase?: (phase: string) => void + * }} params + */ +export function useSampleRun({ analyzeFiles, cancelAnalysis, advancePhase }) { + const store = useStore(); + const [session, setSessionState] = useState(/** @type {SampleSession|null} */ (null)); + const sessionRef = useRef(/** @type {SampleSession|null} */ (null)); + const nextIdRef = useRef(0); + const mountedRef = useRef(true); + + useEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + }; + }, []); + + const setSession = useCallback((/** @type {SampleSession|null} */ next) => { + sessionRef.current = next; + if (mountedRef.current) setSessionState(next); + }, []); + + const readState = useCallback(() => /** @type {any} */ (store?.getState?.()) || {}, [store]); + + const markReviewed = useCallback( + (/** @type {string} */ hash) => { + if (!hash || typeof store?.dispatch !== 'function') return; + store.dispatch(setSmartFolderSampleHash(hash)); + }, + [store] + ); + + const gatedAnalyzeFiles = useCallback( + async (/** @type {any[]} */ files, /** @type {any} */ runOptions) => { + if (runOptions && (runOptions.origin || runOptions.bypassCache)) { + return analyzeFiles(files, runOptions); + } + const current = sessionRef.current; + if (current) { + // The sample dialog is open: these files join the batch it holds. + setSession({ ...current, files: uniqueFiles([...current.files, ...(files || [])]) }); + return { started: false, reason: 'sample-offered' }; + } + const state = readState(); + const smartFolders = state.files?.smartFolders; + const offer = shouldOfferSample({ + files, + smartFolders, + reviewedHash: state.ui?.smartFolderSampleHash, + isAnalyzing: Boolean(state.analysis?.isAnalyzing) + }); + if (!offer) return analyzeFiles(files, runOptions); + + const batch = uniqueFiles(files); + nextIdRef.current += 1; + setSession({ + id: nextIdRef.current, + stage: 'offer', + files: batch, + picked: pickSampleFiles(batch), + hash: getFolderDefinitionsHash(smartFolders), + hintCount: countFoldersWithHints(smartFolders), + rows: [] + }); + return { started: false, reason: 'sample-offered' }; + }, + [analyzeFiles, readState, setSession] + ); + + /** Analyze the whole batch now; the offer stays away for these folders. */ + const skipSample = useCallback(async () => { + const current = sessionRef.current; + if (!current || current.stage !== 'offer') return { started: false, reason: 'no-offer' }; + markReviewed(current.hash); + setSession(null); + return analyzeFiles(current.files); + }, [analyzeFiles, markReviewed, setSession]); + + /** Analyze the sample, then show where each file would go. */ + const startSample = useCallback(async () => { + const current = sessionRef.current; + if (!current || current.stage !== 'offer') return; + const running = { ...current, stage: /** @type {const} */ ('running') }; + setSession(running); + const isCurrent = () => sessionRef.current?.id === running.id; + + let result; + try { + result = await analyzeFiles(running.picked, { origin: 'sample' }); + } catch (error) { + logger.warn('Sample run failed to start', { error: /** @type {any} */ (error)?.message }); + result = { started: false, reason: 'error' }; + } + if (!isCurrent()) return; + if (!result?.started) { + // Another run held the analysis (the sample waits in its queue): + // nothing to review here, and the rest stays in the queue. + setSession(null); + return; + } + + const state = readState(); + const results = Array.isArray(state.analysis?.results) ? state.analysis.results : []; + const byPath = new Map(results.map((/** @type {any} */ r) => [toCanonicalPath(r?.path), r])); + const sampleFiles = running.picked.map((file) => { + const done = byPath.get(toCanonicalPath(file?.path)); + return done ? { ...file, ...done } : file; + }); + const rows = await rankSampleFiles({ + files: sampleFiles, + smartFolders: Array.isArray(state.files?.smartFolders) ? state.files.smartFolders : [], + getBatchSuggestions: /** @type {any} */ (window)?.electronAPI?.suggestions + ?.getBatchSuggestions + }); + if (!isCurrent()) return; + markReviewed(running.hash); + setSession({ ...running, stage: 'review', rows }); + }, [analyzeFiles, markReviewed, readState, setSession]); + + /** After the review: analyze the files the sample did not cover. */ + const continueWithRest = useCallback(async () => { + const current = sessionRef.current; + if (!current || current.stage !== 'review') return { started: false, reason: 'no-review' }; + setSession(null); + const sampled = new Set(current.picked.map((file) => toCanonicalPath(file?.path))); + const rest = current.files.filter((file) => !sampled.has(toCanonicalPath(file?.path))); + if (rest.length === 0) return { started: false, reason: 'empty' }; + return analyzeFiles(rest); + }, [analyzeFiles, setSession]); + + /** After the review: go to Setup. The rest stays in the queue, unanalyzed. */ + const editFolders = useCallback(() => { + setSession(null); + advancePhase?.(PHASES?.SETUP ?? 'setup'); + }, [advancePhase, setSession]); + + /** Stop the sample while it runs. Every file stays in the queue. */ + const stopSample = useCallback(() => { + if (sessionRef.current?.stage !== 'running') return; + setSession(null); + cancelAnalysis?.(); + }, [cancelAnalysis, setSession]); + + /** Close the dialog. A running sample is stopped with stopSample instead. */ + const dismiss = useCallback(() => { + if (sessionRef.current?.stage === 'running') return; + setSession(null); + }, [setSession]); + + return { + session, + analyzeFiles: gatedAnalyzeFiles, + startSample, + skipSample, + continueWithRest, + editFolders, + stopSample, + dismiss + }; +} diff --git a/src/renderer/phases/organize/organizeDestination.js b/src/renderer/phases/organize/organizeDestination.js index 31a063bf..a1da4846 100644 --- a/src/renderer/phases/organize/organizeDestination.js +++ b/src/renderer/phases/organize/organizeDestination.js @@ -184,3 +184,107 @@ export function suggestionForOperation(destination, smartFolders) { source: 'engine' }; } + +/** + * Why a file goes where `resolveOrganizeDestination` sends it, in the words + * the Organize card shows. The sample run before a large batch uses it too, + * so both say the same thing about the same file. + * + * @param {OrganizeDestination|null|undefined} routing + * @param {unknown} [category] - the label the card shows (edit or analysis) + * @returns {string|null} + */ +export function describeRouting(routing, category) { + if (!routing) return null; + const engine = routing.engine; + if (routing.source === 'engine' && engine) { + return engine.explanation || 'Ranked by what the file contains'; + } + if (routing.source === 'user') { + return engine?.folder && engine.folder !== routing.folder?.name + ? `Your choice (suggested: ${engine.folder})` + : 'Your choice'; + } + if (routing.source === 'name') { + // A name match is either the analysis label or the file's own name/type. + const fromLabel = + typeof category === 'string' && + category.trim().toLowerCase() === + String(routing.folder?.name || '') + .trim() + .toLowerCase(); + return fromLabel + ? 'Matched by the label the analysis gave this file' + : 'Matched by folder name'; + } + return null; +} + +/** + * The engine's fit for the folder it picked, 0..1, or null where a number + * would not measure the folder: a name match, the user's own pick, or the + * catch-all default. + * + * @param {OrganizeDestination|null|undefined} routing + * @returns {number|null} + */ +export function routingScore(routing) { + if (routing?.source !== 'engine' || !routing.engine) return null; + if (routing.engine.source === 'default') return null; + const value = routing.engine.confidence; + return typeof value === 'number' && Number.isFinite(value) ? value : null; +} + +/** + * Stale results (decision, 2026-11-15): an explicit warning with a one-click + * refresh, not a silent automatic one. + * + * Every analysis is stamped with the Smart Folder definitions it was made + * under (`analysis.smartFolderHash`, see shared/smartFolderDefinitions). When + * the definitions change, Organize already re-ranks every file through the + * suggestion engine with the new folders (useOrganizeSuggestions), so an + * engine pick is current. What can still be old is the fallback: the label + * the analysis picked from the old folders (source 'name'), or no folder at + * all from that label (source 'none'). Those files are stale. While the + * re-rank is still running, files waiting for it count too. + * + * A stale file is not moved unnoticed: Organize lists them with one button + * that analyzes them again (the existing Reanalyze), and Move asks first. + * Refreshing automatically would mean analyzing again without asking, which + * on a large batch takes as long as the first run did. + * + * Analyses made before the stamp existed carry no hash and are not flagged. + * The user's own folder choice is never stale. Auto-organize (downloads and + * watched folders, in main) needs no check: it analyzes each new file with + * the folders current at that moment, and the analysis cache is keyed on the + * whole folder descriptions, so an edit never returns an old pick. + * + * @param {any} file - the file as Organize lists it (with `analysis`) + * @param {OrganizeDestination|null|undefined} destination + * @param {string} currentHash - getFolderDefinitionsHash of the current folders + * @returns {boolean} + */ +export function isStaleDestination(file, destination, currentHash) { + const stamped = file?.analysis?.smartFolderHash; + if (typeof stamped !== 'string' || !stamped || !currentHash) return false; + if (stamped === currentHash) return false; + return destination?.source === 'name' || destination?.source === 'none'; +} + +/** + * The files among `files` whose destination is stale (see isStaleDestination). + * + * @param {any[]} files + * @param {(file: any) => OrganizeDestination} resolveDestination + * @param {string} currentHash + * @returns {any[]} + */ +export function findStaleFiles(files, resolveDestination, currentHash) { + if (!currentHash || !Array.isArray(files) || typeof resolveDestination !== 'function') return []; + return files.filter((file) => { + const stamped = file?.analysis?.smartFolderHash; + // Only a file stamped with other definitions needs its destination worked out. + if (typeof stamped !== 'string' || !stamped || stamped === currentHash) return false; + return isStaleDestination(file, resolveDestination(file), currentHash); + }); +} diff --git a/src/renderer/store/index.js b/src/renderer/store/index.js index 14236475..572713b8 100644 --- a/src/renderer/store/index.js +++ b/src/renderer/store/index.js @@ -193,6 +193,7 @@ const loadState = () => { graphInsightsSection: 'insights', graphLegendHintDismissed: false, graphInsightsDockCollapsed: false, + smartFolderSampleHash: null, // Session-only flag: a fresh boot is never mid dashboard excursion. cameFromDashboard: false, viewMode: 'phased', @@ -362,6 +363,8 @@ const loadState = () => { graphInsightsSection: 'insights', graphLegendHintDismissed: parsed?.ui?.graphLegendHintDismissed === true, graphInsightsDockCollapsed: parsed?.ui?.graphInsightsDockCollapsed === true, + // Smart folders survive expiry, so which set was tried on a sample does too. + smartFolderSampleHash: readSmartFolderSampleHash(parsed?.ui), // Session-only flag: a fresh boot is never mid dashboard excursion. cameFromDashboard: false, // viewMode is a durable preference, not session state — preserve it @@ -438,7 +441,8 @@ const loadState = () => { selectedTheme: expiredState.ui.selectedTheme, graphTourDismissed: expiredState.ui.graphTourDismissed, graphLegendHintDismissed: expiredState.ui.graphLegendHintDismissed, - graphInsightsDockCollapsed: expiredState.ui.graphInsightsDockCollapsed + graphInsightsDockCollapsed: expiredState.ui.graphInsightsDockCollapsed, + smartFolderSampleHash: expiredState.ui.smartFolderSampleHash }, files: { selectedFiles: [], @@ -492,6 +496,7 @@ const loadState = () => { })(), graphLegendHintDismissed: parsed.ui?.graphLegendHintDismissed === true, graphInsightsDockCollapsed: parsed.ui?.graphInsightsDockCollapsed === true, + smartFolderSampleHash: readSmartFolderSampleHash(parsed.ui), // Session-only flag: a fresh boot is never mid dashboard excursion. cameFromDashboard: false, viewMode: parsed.ui?.viewMode === 'dashboard' ? 'dashboard' : 'phased', @@ -547,6 +552,17 @@ const isDev = process.env.NODE_ENV === 'development'; const DURABLE_THEME_VALUES = ['primary', 'light', 'dark']; +/** + * The Smart Folder set last tried on a sample (uiSlice smartFolderSampleHash). + * Blobs saved before the field existed have none: no sample tried yet. + * @param {any} parsedUi + * @returns {string|null} + */ +function readSmartFolderSampleHash(parsedUi) { + const value = parsedUi?.smartFolderSampleHash; + return typeof value === 'string' && value ? value : null; +} + /** * Resolve the palette to rehydrate. App.js mirrors the selected theme to a * dedicated `stratosort_theme` localStorage key synchronously on every change, diff --git a/src/renderer/store/middleware/persistenceMiddleware.js b/src/renderer/store/middleware/persistenceMiddleware.js index fee74e66..cb93e6e5 100644 --- a/src/renderer/store/middleware/persistenceMiddleware.js +++ b/src/renderer/store/middleware/persistenceMiddleware.js @@ -163,6 +163,8 @@ let lastSavedGraphLegendHintDismissed = null; /** @type {any} */ let lastSavedGraphInsightsDockCollapsed = null; /** @type {any} */ +let lastSavedSmartFolderSampleHash = null; +/** @type {any} */ let lastSavedViewMode = null; /** @type {any} */ let lastSavedSelectedTheme = null; @@ -414,6 +416,12 @@ function buildStateSnapshot(freshState, options = {}) { graphInsightsSection: freshState.ui.graphInsightsSection ?? null, graphLegendHintDismissed: Boolean(freshState.ui.graphLegendHintDismissed), graphInsightsDockCollapsed: Boolean(freshState.ui.graphInsightsDockCollapsed), + // Additive and optional: an older blob without it hydrates to null + // (no sample tried yet), so no migration step is needed. + smartFolderSampleHash: + typeof freshState.ui.smartFolderSampleHash === 'string' + ? freshState.ui.smartFolderSampleHash + : null, viewMode: freshState.ui.viewMode === 'dashboard' ? 'dashboard' : 'phased', selectedTheme: ['primary', 'light', 'dark'].includes(freshState.ui.selectedTheme) ? freshState.ui.selectedTheme @@ -496,6 +504,7 @@ function detectUnsavedChanges(state) { state.ui.graphInsightsSection !== lastSavedGraphInsightsSection || state.ui.graphLegendHintDismissed !== lastSavedGraphLegendHintDismissed || state.ui.graphInsightsDockCollapsed !== lastSavedGraphInsightsDockCollapsed || + (state.ui.smartFolderSampleHash ?? null) !== lastSavedSmartFolderSampleHash || state.ui.viewMode !== lastSavedViewMode || state.ui.selectedTheme !== lastSavedSelectedTheme || state.ui.settings !== lastSavedSettingsRef || @@ -707,6 +716,7 @@ const persistenceMiddleware = (/** @type {any} */ store) => { lastSavedGraphInsightsSection = freshState.ui.graphInsightsSection; lastSavedGraphLegendHintDismissed = freshState.ui.graphLegendHintDismissed; lastSavedGraphInsightsDockCollapsed = freshState.ui.graphInsightsDockCollapsed; + lastSavedSmartFolderSampleHash = freshState.ui.smartFolderSampleHash ?? null; lastSavedViewMode = freshState.ui.viewMode; lastSavedSelectedTheme = freshState.ui.selectedTheme; lastSavedSettingsRef = freshState.ui.settings; @@ -825,6 +835,7 @@ export const cancelPendingPersistence = () => { lastSavedGraphInsightsSection = null; lastSavedGraphLegendHintDismissed = null; lastSavedGraphInsightsDockCollapsed = null; + lastSavedSmartFolderSampleHash = null; lastSavedViewMode = null; lastSavedSelectedTheme = null; lastSavedSettingsRef = null; @@ -877,6 +888,7 @@ export const cleanupPersistence = (/** @type {any} */ store) => { lastSavedGraphInsightsSection = null; lastSavedGraphLegendHintDismissed = null; lastSavedGraphInsightsDockCollapsed = null; + lastSavedSmartFolderSampleHash = null; lastSavedViewMode = null; lastSavedSelectedTheme = null; lastSavedSettingsRef = null; diff --git a/src/renderer/store/slices/analysisSlice.js b/src/renderer/store/slices/analysisSlice.js index 81d294af..daf42883 100644 --- a/src/renderer/store/slices/analysisSlice.js +++ b/src/renderer/store/slices/analysisSlice.js @@ -44,9 +44,10 @@ function enforceResultsLimit(/** @type {any} */ results) { /** * Why the current run started, so the progress card can say so: files the * person added, files added while the last run was going, a run resumed after - * a restart, a retry of failed files, or a Reanalyze. Not persisted. + * a restart, a retry of failed files, a Reanalyze, or a few files tried + * before a large batch (the sample run). Not persisted. */ -const RUN_ORIGINS = ['added', 'queued', 'resume', 'retry', 'reanalyze']; +const RUN_ORIGINS = ['added', 'queued', 'resume', 'retry', 'reanalyze', 'sample']; /** @type {any} */ const initialState = { diff --git a/src/renderer/store/slices/uiSlice.js b/src/renderer/store/slices/uiSlice.js index ceeb5c35..8edf2cbf 100644 --- a/src/renderer/store/slices/uiSlice.js +++ b/src/renderer/store/slices/uiSlice.js @@ -276,7 +276,14 @@ const initialState = { * graph canvas. Defaults to expanded so the most useful insights stay * easy to find; users can hide the strip to reclaim vertical space. */ - graphInsightsDockCollapsed: false + graphInsightsDockCollapsed: false, + /** + * The Smart Folder definitions (getFolderDefinitionsHash) the user last + * tried on a sample, or chose to skip the sample for, before a large batch. + * Persisted so the offer comes back only after the folders change. + * @type {string|null} + */ + smartFolderSampleHash: null }; /** Valid accordion keys for the Knowledge OS graph sidebar (single-open). */ @@ -437,6 +444,9 @@ const uiSlice = createSlice({ selectedTheme: state?.selectedTheme ?? initialState.selectedTheme, graphInsightsDockCollapsed: state?.graphInsightsDockCollapsed ?? initialState.graphInsightsDockCollapsed, + // Which folder set was already tried on a sample: a "Start over" does + // not change the folders, so it must not bring the offer back. + smartFolderSampleHash: state?.smartFolderSampleHash ?? initialState.smartFolderSampleHash, // The settings cache mirrors what main has saved; it is not session // state. Nothing refetches it after a reset, so dropping it made // selectNamingDefaults fall back to the hard-coded defaults, which @@ -487,6 +497,10 @@ const uiSlice = createSlice({ setGraphInsightsDockCollapsed: (state, action) => { state.graphInsightsDockCollapsed = Boolean(action.payload); }, + setSmartFolderSampleHash: (state, action) => { + const value = action.payload; + state.smartFolderSampleHash = typeof value === 'string' && value ? value : null; + }, setViewMode: (state, action) => { const next = action.payload; state.viewMode = VALID_VIEW_MODES.includes(next) ? next : 'phased'; @@ -543,6 +557,7 @@ export const { setGraphInsightsSection, setGraphLegendHintDismissed, setGraphInsightsDockCollapsed, + setSmartFolderSampleHash, setViewMode, setCameFromDashboard, setSelectedTheme diff --git a/src/shared/smartFolderDefinitions.js b/src/shared/smartFolderDefinitions.js new file mode 100644 index 00000000..815c5e28 --- /dev/null +++ b/src/shared/smartFolderDefinitions.js @@ -0,0 +1,74 @@ +// @ts-check +/** + * One short hash of what the user's Smart Folders say: each folder's name, + * description and path. The AI reads exactly these to pick a folder, so two + * folder sets with the same hash route files the same way, and any edit to a + * name, a description or a location gives a new hash. + * + * Used by the renderer for two things: + * - the sample run before a large batch (remembered per hash, so the offer + * comes back only after the folders change), and + * - stale-result detection in Organize (each analysis is stamped with the + * hash it was made under; a different current hash means the folder it + * picked came from the old definitions). + * + * Order does not matter, and surrounding spaces, the path separator style + * and a trailing separator are ignored. Ids and keywords are left out: a + * folder recreated with the same words is the same definition. + */ + +/** + * @param {unknown} value + * @returns {string} + */ +const text = (value) => (typeof value === 'string' ? value.trim() : ''); + +/** + * @param {unknown} value + * @returns {string} + */ +function normalizePath(value) { + return text(value).replace(/\\/g, '/').replace(/\/+$/, ''); +} + +/** + * cyrb53: a small, fast 53-bit string hash. Not for security, only to tell + * two folder sets apart without storing their full text. + * @param {string} input + * @returns {string} 14 hex characters + */ +function cyrb53(input) { + let h1 = 0xdeadbeef; + let h2 = 0x41c6ce57; + for (let i = 0; i < input.length; i += 1) { + const ch = input.charCodeAt(i); + h1 = Math.imul(h1 ^ ch, 2654435761); + h2 = Math.imul(h2 ^ ch, 1597334677); + } + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507) ^ Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507) ^ Math.imul(h1 ^ (h1 >>> 13), 3266489909); + const value = 4294967296 * (2097151 & h2) + (h1 >>> 0); + return value.toString(16).padStart(14, '0'); +} + +/** + * @param {unknown} folders - the Smart Folder list + * @returns {string} '' when there are no folders, else 'sf-' and 14 hex characters + */ +function getFolderDefinitionsHash(folders) { + if (!Array.isArray(folders)) return ''; + const entries = []; + for (const folder of folders) { + if (!folder || typeof folder !== 'object') continue; + const record = /** @type {Record} */ (folder); + const name = text(record.name); + const path = normalizePath(record.path); + if (!name && !path) continue; + entries.push(JSON.stringify([name, text(record.description), path])); + } + if (entries.length === 0) return ''; + entries.sort(); + return `sf-${cyrb53(entries.join('\n'))}`; +} + +module.exports = { getFolderDefinitionsHash }; diff --git a/test/components/AnalysisRunProvider.sample.test.js b/test/components/AnalysisRunProvider.sample.test.js new file mode 100644 index 00000000..b7d56780 --- /dev/null +++ b/test/components/AnalysisRunProvider.sample.test.js @@ -0,0 +1,127 @@ +/** + * @jest-environment jsdom + * + * The provider hands Discover an analyzeFiles behind the sample offer: a + * large batch of added files opens the "Try a few files first?" dialog + * instead of starting a two-hour run on folders nobody has tried yet. + */ +import React from 'react'; +import { Provider } from 'react-redux'; +import { configureStore } from '@reduxjs/toolkit'; +import { render, screen, act, fireEvent } from '@testing-library/react'; +import '@testing-library/jest-dom'; +import uiReducer from '../../src/renderer/store/slices/uiSlice'; +import filesReducer, { setSmartFolders } from '../../src/renderer/store/slices/filesSlice'; +import analysisReducer from '../../src/renderer/store/slices/analysisSlice'; + +const mockAnalyzeFiles = jest.fn(async () => ({ started: true })); + +jest.mock('../../src/renderer/contexts/NotificationContext', () => ({ + useNotification: () => ({ addNotification: jest.fn() }) +})); + +jest.mock('../../src/renderer/phases/discover/useDiscoverState', () => { + const actions = { setPhaseData: jest.fn(), advancePhase: jest.fn() }; + return { + useDiscoverState: () => ({ + selectedFiles: [], + analysisResults: [], + isAnalyzing: false, + analysisProgress: { current: 0, total: 0 }, + fileStates: {}, + namingSettings: {}, + setIsAnalyzing: jest.fn(), + setAnalysisProgress: jest.fn(), + setCurrentAnalysisFile: jest.fn(), + setAnalysisResults: jest.fn(), + setFileStates: jest.fn(), + updateFileState: jest.fn(), + resetAnalysisState: jest.fn(), + actions, + getCurrentPhase: () => 'discover' + }) + }; +}); + +jest.mock('../../src/renderer/phases/discover/useAnalysis', () => ({ + useAnalysis: () => ({ + analyzeFiles: mockAnalyzeFiles, + cancelAnalysis: jest.fn(), + clearAnalysisQueue: jest.fn(), + retryFailedFiles: jest.fn(), + reanalyzeFiles: jest.fn(), + pruneBatchResultsForPath: jest.fn(), + resetAnalysisState: jest.fn() + }) +})); + +import { + AnalysisRunProvider, + useAnalysisRun +} from '../../src/renderer/phases/discover/AnalysisRunProvider'; + +const files = Array.from({ length: 25 }, (_, i) => ({ + path: `/Users/alex/Inbox/scan-${i}.pdf`, + name: `scan-${i}.pdf` +})); + +function setup() { + const store = configureStore({ + reducer: { + ui: uiReducer, + files: filesReducer, + analysis: analysisReducer, + system: () => ({ redactPaths: false, redactPathsLoaded: true }) + }, + middleware: (getDefault) => getDefault({ serializableCheck: false, immutableCheck: false }) + }); + store.dispatch( + setSmartFolders([ + { + id: 'f-inv', + name: 'Invoices', + description: 'Bills and invoices from suppliers', + path: '/Users/alex/Library/Invoices' + } + ]) + ); + let run; + function Probe() { + run = useAnalysisRun(); + return null; + } + render( + + + + + + ); + return { store, getRun: () => run }; +} + +afterEach(() => mockAnalyzeFiles.mockClear()); + +test('a large batch opens the sample offer instead of starting the run', async () => { + const { getRun } = setup(); + let result; + await act(async () => { + result = await getRun().analyzeFiles(files); + }); + expect(result).toEqual({ started: false, reason: 'sample-offered' }); + expect(mockAnalyzeFiles).not.toHaveBeenCalled(); + expect(screen.getByText('Try a few files first?')).toBeInTheDocument(); +}); + +test('skipping from the dialog starts the whole batch', async () => { + const { store, getRun } = setup(); + await act(async () => { + await getRun().analyzeFiles(files); + }); + await act(async () => { + fireEvent.click(screen.getByRole('button', { name: 'Analyze all 25 files' })); + }); + expect(mockAnalyzeFiles).toHaveBeenCalledWith(files); + expect(screen.queryByText('Try a few files first?')).not.toBeInTheDocument(); + expect(store.getState().ui.smartFolderSampleHash).toMatch(/^sf-/); +}); diff --git a/test/components/AnalysisRunProvider.test.js b/test/components/AnalysisRunProvider.test.js index 137eb89c..dfe79665 100644 --- a/test/components/AnalysisRunProvider.test.js +++ b/test/components/AnalysisRunProvider.test.js @@ -113,7 +113,7 @@ describe('AnalysisRunProvider', () => { expect(mockRunLifecycle.unmounts).toBe(0); }); - test('Discover reads the run actions from the provider', () => { + test('Discover reads the run actions from the provider', async () => { let seen; function Probe() { seen = useAnalysisRun(); @@ -124,7 +124,14 @@ describe('AnalysisRunProvider', () => { ); - expect(seen.analyzeFiles).toBe(mockAnalyzeFiles); + // analyzeFiles is the run's, behind the sample offer (useSampleRun), + // which lets a small batch straight through. + mockAnalyzeFiles.mockResolvedValueOnce({ started: true }); + const files = [{ path: '/Users/alex/in/a.pdf', name: 'a.pdf' }]; + await act(async () => { + await seen.analyzeFiles(files); + }); + expect(mockAnalyzeFiles).toHaveBeenCalledWith(files, undefined); }); test('a total analysis failure reported by the run reaches Discover', () => { diff --git a/test/components/OrganizePhase.test.js b/test/components/OrganizePhase.test.js index 674e88f0..e843f3d3 100644 --- a/test/components/OrganizePhase.test.js +++ b/test/components/OrganizePhase.test.js @@ -11,8 +11,10 @@ jest.mock('../../src/renderer/contexts/NotificationContext', () => ({ useNotification: jest.fn(() => ({ addNotification: mockAddNotification })) })); +const mockReanalyzeFiles = jest.fn(); + jest.mock('../../src/renderer/phases/discover/AnalysisRunProvider', () => ({ - useAnalysisRun: () => ({ reanalyzeFiles: jest.fn() }) + useAnalysisRun: () => ({ reanalyzeFiles: mockReanalyzeFiles }) })); jest.mock('../../src/renderer/components/UndoRedoSystem', () => ({ @@ -155,7 +157,20 @@ jest.mock('../../src/renderer/components/organize', () => ({ jest.mock('../../src/renderer/components/ui/Modal', () => ({ __esModule: true, - default: ({ isOpen, children }) => (isOpen ?
{children}
: null) + default: ({ isOpen, children }) => (isOpen ?
{children}
: null), + ConfirmModal: ({ isOpen, title, message, confirmText, cancelText, onConfirm, onClose }) => + isOpen ? ( +
+
{title}
+
{message}
+ + +
+ ) : null })); jest.mock('../../src/renderer/components/organize/OrganizePreviewModal', () => ({ @@ -783,3 +798,122 @@ describe('OrganizePhase: every file of the last run', () => { expect(reveal).toHaveBeenCalledWith('/dest/finance/GSC 2026.xlsx'); }); }); + +describe('OrganizePhase with results made under old Smart Folder definitions', () => { + const { getFolderDefinitionsHash } = require('../../src/shared/smartFolderDefinitions'); + const FOLDERS = [{ id: '1', name: 'Finance', path: '/dest/finance' }]; + const currentHash = getFolderDefinitionsHash(FOLDERS); + + let restoreState; + let handleOrganizeFiles; + + function renderWith(files) { + const { useOrganizeState } = require('../../src/renderer/phases/organize/useOrganizeState'); + const { useOrganization } = require('../../src/renderer/phases/organize/useOrganization'); + const { + useFileEditing, + useProcessedFiles + } = require('../../src/renderer/phases/organize/useFileEditing'); + const original = useOrganizeState.getMockImplementation(); + restoreState = () => useOrganizeState.mockImplementation(original); + useOrganizeState.mockImplementation(() => ({ ...original(), filesWithAnalysis: files })); + // No card edits (an edit is the user's choice, never stale) and every + // file still waiting to move. + useFileEditing.mockReturnValue({ + editingFiles: {}, + setEditingFiles: jest.fn(), + handleEditFile: jest.fn(), + getFileWithEdits: (file) => file + }); + useProcessedFiles.mockReturnValue({ + setProcessedFileIds: jest.fn(), + markFilesAsProcessed: jest.fn(), + unmarkFilesAsProcessed: jest.fn(), + getFilteredFiles: (list) => ({ unprocessedFiles: list, processedFiles: [] }), + normalizePath: (value) => value + }); + handleOrganizeFiles = jest.fn(); + useOrganization.mockReturnValue({ + isOrganizing: false, + batchProgress: null, + organizePreview: [], + handleOrganizeFiles, + organizeConflicts: [] + }); + render(); + } + + afterEach(() => { + restoreState?.(); + restoreState = null; + jest.clearAllMocks(); + }); + + const staleInvoice = { + path: '/tmp/invoice.pdf', + name: 'invoice.pdf', + analysis: { category: 'Finance', suggestedName: 'Invoice.pdf', smartFolderHash: 'sf-old' } + }; + const freshNotes = { + path: '/tmp/notes.txt', + name: 'notes.txt', + analysis: { category: 'Finance', suggestedName: 'Notes.txt', smartFolderHash: currentHash } + }; + const unstampedReport = { + path: '/tmp/report.pdf', + name: 'report.pdf', + analysis: { category: 'Finance', suggestedName: 'Report.pdf' } + }; + + test('lists a file whose folder came from old definitions, with one click to refresh', () => { + renderWith([staleInvoice, freshNotes, unstampedReport]); + + expect(screen.getByTestId('stale-suggestions-notice')).toBeInTheDocument(); + expect( + screen.getByText('Your Smart Folders changed after this file was analyzed') + ).toBeInTheDocument(); + expect( + screen.getByText( + '1 file still goes where your old Smart Folders sent it. Analyze it again to sort it with the current folders.' + ) + ).toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: 'Analyze 1 again' })); + expect(mockReanalyzeFiles).toHaveBeenCalledWith(['/tmp/invoice.pdf']); + }); + + test('Move asks first when a file would go where old definitions sent it', async () => { + renderWith([staleInvoice, freshNotes]); + + fireEvent.click(screen.getByRole('button', { name: /Move 2 files/i })); + expect(handleOrganizeFiles).not.toHaveBeenCalled(); + expect(screen.getByText('Move files sorted by your old Smart Folders?')).toBeInTheDocument(); + expect( + screen.getByText( + '1 of these files goes where your old Smart Folders sent it. To use your current folders, cancel and analyze it again.' + ) + ).toBeInTheDocument(); + + // Cancel: nothing moves. + await act(async () => { + fireEvent.click(screen.getByRole('button', { name: 'Cancel' })); + }); + expect(handleOrganizeFiles).not.toHaveBeenCalled(); + + // Move anyway: the move goes ahead as asked. + fireEvent.click(screen.getByRole('button', { name: /Move 2 files/i })); + await act(async () => { + fireEvent.click(screen.getByRole('button', { name: 'Move anyway' })); + }); + await waitFor(() => expect(handleOrganizeFiles).toHaveBeenCalledTimes(1)); + }); + + test('current results move at once, with no warning', () => { + renderWith([freshNotes, unstampedReport]); + + expect(screen.queryByTestId('stale-suggestions-notice')).not.toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: /Move 2 files/i })); + expect(handleOrganizeFiles).toHaveBeenCalledTimes(1); + expect(screen.queryByText('Move files sorted by your old Smart Folders?')).toBeNull(); + }); +}); diff --git a/test/components/SampleRunDialog.test.js b/test/components/SampleRunDialog.test.js new file mode 100644 index 00000000..fbc0cfe5 --- /dev/null +++ b/test/components/SampleRunDialog.test.js @@ -0,0 +1,141 @@ +/** + * @jest-environment jsdom + * + * The sample-first dialog: the offer (sample or skip), the running sample + * (stop only), and the review (where each file would go, with its score and + * reason; continue or edit the folders). + */ +import React from 'react'; +import { Provider } from 'react-redux'; +import { configureStore } from '@reduxjs/toolkit'; +import { render, screen, fireEvent } from '@testing-library/react'; +import '@testing-library/jest-dom'; +import uiReducer from '../../src/renderer/store/slices/uiSlice'; +import SampleRunDialog from '../../src/renderer/components/discover/SampleRunDialog'; + +const files = Array.from({ length: 25 }, (_, i) => ({ + path: `/Users/alex/Inbox/file-${i}.pdf`, + name: `file-${i}.pdf` +})); +const picked = files.slice(0, 5); + +function renderDialog(session, extra = {}) { + const store = configureStore({ + reducer: { + ui: uiReducer, + // Path privacy loaded and off, so file names show as they are. + system: () => ({ redactPaths: false, redactPathsLoaded: true }) + } + }); + const handlers = { + onSample: jest.fn(), + onSkip: jest.fn(), + onStop: jest.fn(), + onContinue: jest.fn(), + onEditFolders: jest.fn(), + onClose: jest.fn() + }; + render( + + + + ); + return handlers; +} + +describe('SampleRunDialog', () => { + test('the offer: try a few files first, or analyze them all', () => { + const handlers = renderDialog({ + id: 1, + stage: 'offer', + files, + picked, + hash: 'sf-1', + hintCount: 2, + rows: [] + }); + + expect(screen.getByText('Try a few files first?')).toBeInTheDocument(); + expect( + screen.getByText( + '25 files are ready to analyze. Analyze 5 of them first to see where your Smart Folders send them. No files are moved.' + ) + ).toBeInTheDocument(); + expect( + screen.getByText( + '2 Smart Folders have a description hint in Setup. The sample shows whether it matters.' + ) + ).toBeInTheDocument(); + expect(screen.getByText('file-0.pdf')).toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: 'Analyze 5 files first' })); + expect(handlers.onSample).toHaveBeenCalledTimes(1); + fireEvent.click(screen.getByRole('button', { name: 'Analyze all 25 files' })); + expect(handlers.onSkip).toHaveBeenCalledTimes(1); + }); + + test('no hint line when every folder description is clear', () => { + renderDialog({ id: 1, stage: 'offer', files, picked, hash: 'sf-1', hintCount: 0, rows: [] }); + expect(screen.queryByText(/description hint/)).not.toBeInTheDocument(); + }); + + test('while the sample runs, the only way out is Stop', () => { + const handlers = renderDialog( + { id: 1, stage: 'running', files, picked, hash: 'sf-1', hintCount: 0, rows: [] }, + { progress: { current: 2, total: 5 } } + ); + expect(screen.getByText('Trying your Smart Folders')).toBeInTheDocument(); + expect(screen.getByText('2 of 5 done')).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Stop' })); + expect(handlers.onStop).toHaveBeenCalledTimes(1); + }); + + test('the review shows each file with its folder, score and reason', () => { + const handlers = renderDialog({ + id: 1, + stage: 'review', + files, + picked, + hash: 'sf-1', + hintCount: 0, + rows: [ + { + path: picked[0].path, + name: 'file-0.pdf', + analyzed: true, + folder: 'Invoices', + score: 0.82, + reason: 'Reads like a supplier invoice' + }, + { + path: picked[1].path, + name: 'file-1.pdf', + analyzed: true, + folder: null, + score: null, + reason: 'No Smart Folder fits this file.' + }, + { + path: picked[2].path, + name: 'file-2.pdf', + analyzed: false, + folder: null, + score: null, + reason: 'This file could not be analyzed.' + } + ] + }); + + expect(screen.getByText('Where these files would go')).toBeInTheDocument(); + expect(screen.getByText('Invoices')).toBeInTheDocument(); + expect(screen.getByText('82% confidence')).toBeInTheDocument(); + expect(screen.getByText('Reads like a supplier invoice')).toBeInTheDocument(); + expect(screen.getByText('Uncategorized')).toBeInTheDocument(); + expect(screen.getByText('Not analyzed')).toBeInTheDocument(); + + fireEvent.click(screen.getByRole('button', { name: 'Analyze the other 20 files' })); + expect(handlers.onContinue).toHaveBeenCalledTimes(1); + fireEvent.click(screen.getByRole('button', { name: 'Edit Smart Folders' })); + expect(handlers.onEditFolders).toHaveBeenCalledTimes(1); + }); +}); diff --git a/test/components/StaleSuggestionsNotice.test.js b/test/components/StaleSuggestionsNotice.test.js new file mode 100644 index 00000000..f3e715c9 --- /dev/null +++ b/test/components/StaleSuggestionsNotice.test.js @@ -0,0 +1,46 @@ +/** + * @jest-environment jsdom + * + * The stale-result notice in Organize: files whose folder still comes from + * Smart Folder definitions that changed, with one click to analyze them + * again. While the re-rank with the new folders runs, it says so instead. + */ +import React from 'react'; +import { render, screen, fireEvent } from '@testing-library/react'; +import '@testing-library/jest-dom'; +import StaleSuggestionsNotice from '../../src/renderer/components/organize/StaleSuggestionsNotice'; + +describe('StaleSuggestionsNotice', () => { + test('nothing to say with no stale files', () => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); + + test('names the files and refreshes them in one click', () => { + const onRefresh = jest.fn(); + render(); + expect( + screen.getByText('Your Smart Folders changed after these files were analyzed') + ).toBeInTheDocument(); + expect( + screen.getByText( + '3 files still go where your old Smart Folders sent them. Analyze them again to sort them with the current folders.' + ) + ).toBeInTheDocument(); + fireEvent.click(screen.getByRole('button', { name: 'Analyze 3 again' })); + expect(onRefresh).toHaveBeenCalledTimes(1); + }); + + test('while the re-rank runs it says so and offers no button yet', () => { + render(); + expect( + screen.getByText('Checking 3 files against your current Smart Folders.') + ).toBeInTheDocument(); + expect(screen.queryByRole('button')).not.toBeInTheDocument(); + }); + + test('the refresh waits while another analysis runs', () => { + render(); + expect(screen.getByRole('button', { name: 'Analyze 1 again' })).toBeDisabled(); + }); +}); diff --git a/test/organizeDestination.test.js b/test/organizeDestination.test.js index 0d0a5c03..0bf99b76 100644 --- a/test/organizeDestination.test.js +++ b/test/organizeDestination.test.js @@ -7,7 +7,11 @@ import { normalizeEngineEntry, findFolderForCandidate, resolveOrganizeDestination, - suggestionForOperation + suggestionForOperation, + describeRouting, + routingScore, + isStaleDestination, + findStaleFiles } from '../src/renderer/phases/organize/organizeDestination'; const FINANCE = { id: 'f-fin', name: 'Finance', path: '/library/Finance' }; @@ -178,3 +182,79 @@ describe('suggestionForOperation', () => { ).toBe(undefined); }); }); + +describe('describeRouting and routingScore', () => { + test('an engine pick gives its reason and its score', () => { + const routing = resolve({ analysis: { category: 'Taxes' }, engine: engineFor(FINANCE) }); + expect(describeRouting(routing, 'Taxes')).toBe('This file\'s meaning matches "Finance"'); + expect(routingScore(routing)).toBe(0.82); + }); + + test('a label match says so, with no score', () => { + const routing = resolve({ analysis: { category: 'Taxes' } }); + expect(describeRouting(routing, 'Taxes')).toBe( + 'Matched by the label the analysis gave this file' + ); + expect(routingScore(routing)).toBeNull(); + }); + + test('the user choice names the engine pick it overrode, with no score', () => { + const routing = resolve({ edits: { category: 'Taxes' }, engine: engineFor(FINANCE) }); + expect(describeRouting(routing, 'Taxes')).toBe('Your choice (suggested: Finance)'); + expect(routingScore(routing)).toBeNull(); + }); + + test('no destination has no reason', () => { + expect(describeRouting(resolve({ analysis: { category: 'Nonsense' } }), 'Nonsense')).toBe(null); + }); +}); + +describe('stale results: picked under Smart Folder definitions that changed', () => { + const NOW = 'sf-current'; + const file = (analysis) => ({ path: '/Users/alex/in/doc.pdf', name: 'doc.pdf', analysis }); + + test('a label match from old definitions is stale', () => { + const stale = file({ category: 'Taxes', smartFolderHash: 'sf-old' }); + const routing = resolve({ analysis: stale.analysis }); + expect(routing.source).toBe('name'); + expect(isStaleDestination(stale, routing, NOW)).toBe(true); + }); + + test('no folder from an old label is stale too: the new folders may place it', () => { + const stale = file({ category: 'Nonsense', smartFolderHash: 'sf-old' }); + const routing = resolve({ analysis: stale.analysis }); + expect(routing.source).toBe('none'); + expect(isStaleDestination(stale, routing, NOW)).toBe(true); + }); + + test('an engine pick is current: it was ranked with the current folders', () => { + const stale = file({ category: 'Taxes', smartFolderHash: 'sf-old' }); + const routing = resolve({ analysis: stale.analysis, engine: engineFor(FINANCE) }); + expect(isStaleDestination(stale, routing, NOW)).toBe(false); + }); + + test('the user choice is never stale', () => { + const stale = file({ category: 'Taxes', userCategory: 'Taxes', smartFolderHash: 'sf-old' }); + const routing = resolve({ analysis: stale.analysis }); + expect(routing.source).toBe('user'); + expect(isStaleDestination(stale, routing, NOW)).toBe(false); + }); + + test('analysis under the current definitions, or with no stamp, is not stale', () => { + const current = file({ category: 'Taxes', smartFolderHash: NOW }); + const unstamped = file({ category: 'Taxes' }); + expect(isStaleDestination(current, resolve({ analysis: current.analysis }), NOW)).toBe(false); + expect(isStaleDestination(unstamped, resolve({ analysis: unstamped.analysis }), NOW)).toBe( + false + ); + }); + + test('findStaleFiles keeps only the stale ones', () => { + const a = { ...file({ category: 'Taxes', smartFolderHash: 'sf-old' }), path: '/a.pdf' }; + const b = { ...file({ category: 'Taxes', smartFolderHash: NOW }), path: '/b.pdf' }; + const c = { path: '/c.pdf', name: 'c.pdf' }; // not analyzed + const resolveFile = (f) => resolve({ analysis: f.analysis || {} }); + expect(findStaleFiles([a, b, c], resolveFile, NOW)).toEqual([a]); + expect(findStaleFiles([a, b, c], resolveFile, '')).toEqual([]); + }); +}); diff --git a/test/sampleRun.test.js b/test/sampleRun.test.js new file mode 100644 index 00000000..b796073e --- /dev/null +++ b/test/sampleRun.test.js @@ -0,0 +1,256 @@ +/** + * Sample first, the pure parts: when the offer applies, which files the + * sample takes, how many folders have description hints, and where each + * sample file would go (the same ranking Organize uses). + */ +import { + SAMPLE_OFFER_MIN_FILES, + SAMPLE_SIZE, + countFoldersWithHints, + pickSampleFiles, + rankSampleFiles, + shouldOfferSample +} from '../src/renderer/phases/discover/sampleRun'; +import { RANKING_OPTIONS } from '../src/renderer/phases/organize/useOrganizeSuggestions'; + +const { getFolderDefinitionsHash } = require('../src/shared/smartFolderDefinitions'); + +const INVOICES = { + id: 'f-inv', + name: 'Invoices', + description: 'Bills and invoices from suppliers, not receipts', + path: '/Users/alex/Library/Invoices' +}; +const PHOTOS = { + id: 'f-photo', + name: 'Photos', + description: 'Pictures taken with the phone or camera', + path: '/Users/alex/Library/Photos' +}; +const UNCATEGORIZED = { + id: 'f-unc', + name: 'Uncategorized', + description: '', + path: '/Users/alex/Library/Uncategorized', + isDefault: true +}; +const FOLDERS = [INVOICES, PHOTOS, UNCATEGORIZED]; + +const makeFiles = (count, ext = 'pdf') => + Array.from({ length: count }, (_, i) => ({ + path: `/Users/alex/Inbox/file-${ext}-${i}.${ext}`, + name: `file-${ext}-${i}.${ext}` + })); + +describe('shouldOfferSample', () => { + const base = { + smartFolders: FOLDERS, + reviewedHash: null, + isAnalyzing: false + }; + + test('a batch at the threshold is offered a sample', () => { + expect(SAMPLE_OFFER_MIN_FILES).toBe(20); + expect(shouldOfferSample({ ...base, files: makeFiles(SAMPLE_OFFER_MIN_FILES) })).toBe(true); + }); + + test('a smaller batch is not', () => { + expect(shouldOfferSample({ ...base, files: makeFiles(SAMPLE_OFFER_MIN_FILES - 1) })).toBe( + false + ); + }); + + test('the same file listed twice counts once', () => { + const files = makeFiles(SAMPLE_OFFER_MIN_FILES - 1); + expect(shouldOfferSample({ ...base, files: [...files, files[0]] })).toBe(false); + }); + + test('not while another run is going: the files wait for it', () => { + expect(shouldOfferSample({ ...base, files: makeFiles(30), isAnalyzing: true })).toBe(false); + }); + + test('not with only the Uncategorized catch-all: there is nothing to try', () => { + expect( + shouldOfferSample({ ...base, smartFolders: [UNCATEGORIZED], files: makeFiles(30) }) + ).toBe(false); + }); + + test('not again for folder definitions already tried, but again once they change', () => { + const reviewedHash = getFolderDefinitionsHash(FOLDERS); + expect(shouldOfferSample({ ...base, reviewedHash, files: makeFiles(30) })).toBe(false); + + const edited = [{ ...INVOICES, description: 'Supplier invoices only' }, PHOTOS, UNCATEGORIZED]; + expect( + shouldOfferSample({ ...base, smartFolders: edited, reviewedHash, files: makeFiles(30) }) + ).toBe(true); + }); +}); + +describe('pickSampleFiles', () => { + test('takes SAMPLE_SIZE files spread across the file types in the batch', () => { + const files = [...makeFiles(20, 'pdf'), ...makeFiles(6, 'jpg'), ...makeFiles(2, 'xlsx')]; + const picked = pickSampleFiles(files); + expect(picked).toHaveLength(SAMPLE_SIZE); + const types = picked.map((f) => f.name.split('.').pop()); + // One of each type first, the most common type first; then a second round. + expect(types).toEqual(['pdf', 'jpg', 'xlsx', 'pdf', 'jpg']); + }); + + test('a batch of one type takes its first files', () => { + const files = makeFiles(30, 'pdf'); + expect(pickSampleFiles(files)).toEqual(files.slice(0, SAMPLE_SIZE)); + }); + + test('reads the type from the extension field when there is one', () => { + const files = [ + { path: '/Users/alex/a', name: 'a', extension: '.PDF' }, + { path: '/Users/alex/b', name: 'b', extension: 'pdf' }, + { path: '/Users/alex/c.png', name: 'c.png' } + ]; + expect(pickSampleFiles(files, 2).map((f) => f.path)).toEqual([ + '/Users/alex/a', + '/Users/alex/c.png' + ]); + }); + + test('never more files than the batch has', () => { + expect(pickSampleFiles(makeFiles(3))).toHaveLength(3); + expect(pickSampleFiles([])).toEqual([]); + }); +}); + +describe('countFoldersWithHints', () => { + test('counts folders with a missing, short or look-alike description, not the catch-all', () => { + const folders = [ + INVOICES, + PHOTOS, + UNCATEGORIZED, // empty description, but it is the catch-all + { id: 'a', name: 'Misc', description: '', path: '/Users/alex/Misc' }, // missing + { id: 'b', name: 'Work', description: 'Work stuff', path: '/Users/alex/Work' } // short + ]; + expect(countFoldersWithHints(folders)).toBe(2); + }); + + test('two folders that read alike both count', () => { + const folders = [ + { + id: 'a', + name: 'Reports', + description: 'Monthly search reports keyword rankings traffic', + path: '/Users/alex/Reports' + }, + { + id: 'b', + name: 'Analytics', + description: 'Monthly search reports keyword rankings traffic exports', + path: '/Users/alex/Analytics' + } + ]; + expect(countFoldersWithHints(folders)).toBe(2); + }); + + test('clear, distinct descriptions count none', () => { + expect(countFoldersWithHints(FOLDERS)).toBe(0); + }); +}); + +describe('rankSampleFiles', () => { + const invoice = { + path: '/Users/alex/Inbox/march.pdf', + name: 'march.pdf', + analysis: { category: 'Photos', summary: 'Supplier invoice for March' } + }; + const photo = { + path: '/Users/alex/Inbox/beach.jpg', + name: 'beach.jpg', + analysis: { category: 'Photos' } + }; + const failed = { path: '/Users/alex/Inbox/broken.docx', name: 'broken.docx', error: 'Timed out' }; + + test('the engine ranking decides, as in Organize, with its score and reason', async () => { + const getBatchSuggestions = jest.fn().mockResolvedValue({ + success: true, + groups: [ + { + files: [ + { + path: invoice.path, + suggestion: { + folder: 'Invoices', + path: INVOICES.path, + folderId: INVOICES.id, + isSmartFolder: true + }, + confidence: 0.82, + explanation: 'This file reads like "Invoices"', + alternatives: [] + } + ] + } + ] + }); + + const rows = await rankSampleFiles({ + files: [invoice, photo, failed], + smartFolders: FOLDERS, + getBatchSuggestions + }); + + expect(getBatchSuggestions).toHaveBeenCalledTimes(1); + const [sent, options] = getBatchSuggestions.mock.calls[0]; + expect(options).toBe(RANKING_OPTIONS); + // Only analyzed files are ranked. + expect(sent.map((f) => f.path)).toEqual([invoice.path, photo.path]); + + expect(rows).toEqual([ + { + path: invoice.path, + name: 'march.pdf', + analyzed: true, + folder: 'Invoices', + score: 0.82, + reason: 'This file reads like "Invoices"' + }, + { + // No engine answer: the analysis label picks the folder, without a score. + path: photo.path, + name: 'beach.jpg', + analyzed: true, + folder: 'Photos', + score: null, + reason: 'Matched by the label the analysis gave this file' + }, + { + path: failed.path, + name: 'broken.docx', + analyzed: false, + folder: null, + score: null, + reason: 'This file could not be analyzed.' + } + ]); + }); + + test('a failed ranking falls back to name matching; no fit means Uncategorized', async () => { + const getBatchSuggestions = jest.fn().mockRejectedValue(new Error('engine offline')); + const unknown = { + path: '/Users/alex/Inbox/notes.txt', + name: 'notes.txt', + analysis: { category: 'Recipes' } + }; + + const rows = await rankSampleFiles({ + files: [photo, unknown], + smartFolders: FOLDERS, + getBatchSuggestions + }); + + expect(rows[0]).toMatchObject({ folder: 'Photos', score: null }); + expect(rows[1]).toMatchObject({ + analyzed: true, + folder: null, + score: null, + reason: 'No Smart Folder fits this file.' + }); + }); +}); diff --git a/test/smartFolderDefinitions.test.js b/test/smartFolderDefinitions.test.js new file mode 100644 index 00000000..0a5ca63f --- /dev/null +++ b/test/smartFolderDefinitions.test.js @@ -0,0 +1,73 @@ +/** + * getFolderDefinitionsHash: one hash of what the Smart Folders say (names, + * descriptions, paths). The sample offer is remembered per hash, and each + * analysis is stamped with it, so it must change exactly when a definition + * the AI reads changes. + */ +const { getFolderDefinitionsHash } = require('../src/shared/smartFolderDefinitions'); + +const INVOICES = { + id: 'f1', + name: 'Invoices', + description: 'Bills and invoices from suppliers', + path: '/Users/alex/Library/Invoices' +}; +const PHOTOS = { + id: 'f2', + name: 'Photos', + description: 'Pictures from the phone and camera', + path: '/Users/alex/Library/Photos' +}; + +describe('getFolderDefinitionsHash', () => { + test('no folders give no hash', () => { + expect(getFolderDefinitionsHash([])).toBe(''); + expect(getFolderDefinitionsHash(undefined)).toBe(''); + expect(getFolderDefinitionsHash([null, {}])).toBe(''); + }); + + test('the same definitions give the same hash in any order', () => { + const hash = getFolderDefinitionsHash([INVOICES, PHOTOS]); + expect(hash).toMatch(/^sf-[0-9a-f]{14}$/); + expect(getFolderDefinitionsHash([PHOTOS, INVOICES])).toBe(hash); + }); + + test('spaces around a value, separator style and a trailing separator do not count', () => { + const hash = getFolderDefinitionsHash([INVOICES]); + expect( + getFolderDefinitionsHash([ + { + ...INVOICES, + name: ' Invoices ', + description: 'Bills and invoices from suppliers ', + path: '\\Users\\alex\\Library\\Invoices\\' + } + ]) + ).toBe(hash); + }); + + test('ids and keywords do not count: the AI does not read them', () => { + const hash = getFolderDefinitionsHash([INVOICES]); + expect(getFolderDefinitionsHash([{ ...INVOICES, id: 'other', keywords: ['tax'] }])).toBe(hash); + }); + + test('a new name, description or path, or another folder, changes the hash', () => { + const hash = getFolderDefinitionsHash([INVOICES, PHOTOS]); + const variants = [ + [{ ...INVOICES, name: 'Bills' }, PHOTOS], + [{ ...INVOICES, description: 'Supplier invoices, not receipts' }, PHOTOS], + [{ ...INVOICES, path: '/Users/alex/Finance/Invoices' }, PHOTOS], + [INVOICES], + [INVOICES, PHOTOS, { name: 'Taxes', description: 'Tax forms', path: '/Users/alex/Taxes' }] + ]; + const hashes = variants.map((folders) => getFolderDefinitionsHash(folders)); + for (const other of hashes) expect(other).not.toBe(hash); + expect(new Set(hashes).size).toBe(hashes.length); + }); + + test('moving text between name and description is a different definition', () => { + const a = getFolderDefinitionsHash([{ name: 'Tax', description: 'forms', path: '/p' }]); + const b = getFolderDefinitionsHash([{ name: 'Tax forms', description: '', path: '/p' }]); + expect(a).not.toBe(b); + }); +}); diff --git a/test/smartFolderSampleHash.persistence.test.js b/test/smartFolderSampleHash.persistence.test.js new file mode 100644 index 00000000..d43b6113 --- /dev/null +++ b/test/smartFolderSampleHash.persistence.test.js @@ -0,0 +1,149 @@ +/** + * @jest-environment jsdom + * + * ui.smartFolderSampleHash remembers which Smart Folder definitions were + * already tried on a sample (or skipped), so the offer before a large batch + * does not come back after a restart or a "Start over", only after the + * folders change. It is saved with the session, restored on launch, and + * blobs saved before it existed restore as "no sample tried yet". + */ + +const REDUX_STATE_KEY = 'stratosort_redux_state'; + +function installStorage(storage) { + const localStorageMock = { + getItem: jest.fn((key) => + Object.prototype.hasOwnProperty.call(storage, key) ? storage[key] : null + ), + setItem: jest.fn((key, value) => { + storage[key] = String(value); + }), + removeItem: jest.fn((key) => { + delete storage[key]; + }), + clear: jest.fn(() => { + Object.keys(storage).forEach((key) => delete storage[key]); + }) + }; + Object.defineProperty(window, 'localStorage', { + value: localStorageMock, + configurable: true, + writable: true + }); + Object.defineProperty(global, 'localStorage', { + value: localStorageMock, + configurable: true, + writable: true + }); +} + +function bootWithUi(ui, { timestamp = Date.now() } = {}) { + jest.resetModules(); + jest.doMock('../src/renderer/store/middleware/ipcMiddleware', () => ({ + __esModule: true, + default: () => (next) => (action) => next(action), + markStoreReady: jest.fn() + })); + jest.doMock('../src/renderer/store/middleware/persistenceMiddleware', () => ({ + __esModule: true, + default: () => (next) => (action) => next(action) + })); + installStorage({ + [REDUX_STATE_KEY]: JSON.stringify({ + _version: 4, + timestamp, + ui, + files: { + selectedFiles: [], + smartFolders: [{ id: '1', name: 'Invoices', path: '/Users/alex/Invoices' }], + fileStates: {} + }, + analysis: { results: [] } + }) + }); + const storeModule = require('../src/renderer/store/index'); + return (storeModule.default || storeModule).getState(); +} + +describe('smartFolderSampleHash', () => { + afterEach(() => { + jest.dontMock('../src/renderer/store/middleware/ipcMiddleware'); + jest.dontMock('../src/renderer/store/middleware/persistenceMiddleware'); + }); + + test('the reducer keeps a hash and clears anything else', () => { + const reducer = require('../src/renderer/store/slices/uiSlice').default; + const { setSmartFolderSampleHash } = require('../src/renderer/store/slices/uiSlice'); + let state = reducer(undefined, { type: '@@init' }); + expect(state.smartFolderSampleHash).toBeNull(); + state = reducer(state, setSmartFolderSampleHash('sf-0123456789abcd')); + expect(state.smartFolderSampleHash).toBe('sf-0123456789abcd'); + state = reducer(state, setSmartFolderSampleHash('')); + expect(state.smartFolderSampleHash).toBeNull(); + }); + + test('"Start over" keeps it: the folders did not change', () => { + const reducer = require('../src/renderer/store/slices/uiSlice').default; + const { setSmartFolderSampleHash, resetUi } = require('../src/renderer/store/slices/uiSlice'); + let state = reducer(undefined, setSmartFolderSampleHash('sf-0123456789abcd')); + state = reducer(state, resetUi()); + expect(state.smartFolderSampleHash).toBe('sf-0123456789abcd'); + }); + + test('it is saved with the session', () => { + jest.resetModules(); + const { + buildStateSnapshot + } = require('../src/renderer/store/middleware/persistenceMiddleware'); + const state = { + ui: { + currentPhase: 'discover', + sidebarOpen: true, + settings: null, + viewMode: 'phased', + selectedTheme: 'primary', + smartFolderSampleHash: 'sf-0123456789abcd' + }, + files: { + selectedFiles: [], + organizedFiles: [], + organizedPathIndex: [], + smartFolders: [], + namingConvention: null, + lastOrganizeRun: null, + fileStates: {} + }, + analysis: { + results: [], + isAnalyzing: false, + analysisProgress: { current: 0, total: 0, lastActivity: 0 }, + currentAnalysisFile: '' + } + }; + expect(buildStateSnapshot(state).ui.smartFolderSampleHash).toBe('sf-0123456789abcd'); + state.ui.smartFolderSampleHash = undefined; + expect(buildStateSnapshot(state).ui.smartFolderSampleHash).toBeNull(); + }); + + test('it is restored on launch', () => { + const state = bootWithUi({ + currentPhase: 'discover', + smartFolderSampleHash: 'sf-0123456789abcd' + }); + expect(state.ui.smartFolderSampleHash).toBe('sf-0123456789abcd'); + }); + + test('a session saved before it existed restores as no sample tried', () => { + const state = bootWithUi({ currentPhase: 'discover' }); + expect(state.ui.smartFolderSampleHash).toBeNull(); + }); + + test('it outlives an expired session, like the Smart Folders it describes', () => { + const state = bootWithUi( + { currentPhase: 'discover', smartFolderSampleHash: 'sf-0123456789abcd' }, + { timestamp: Date.now() - 365 * 24 * 60 * 60 * 1000 } + ); + expect(state.ui.currentPhase).toBe('welcome'); + expect(state.ui.smartFolderSampleHash).toBe('sf-0123456789abcd'); + }); +}); diff --git a/test/useAnalysis.sampleRun.test.js b/test/useAnalysis.sampleRun.test.js new file mode 100644 index 00000000..5413eb9a --- /dev/null +++ b/test/useAnalysis.sampleRun.test.js @@ -0,0 +1,195 @@ +/** + * useAnalysis and Smart Folder definitions: + * - each result is stamped with the definitions it was analyzed under + * (analysis.smartFolderHash), so Organize can tell a result made under + * old definitions from a current one; + * - a sample run (origin 'sample') shows its results in the sample dialog, + * so it ends without the completion notice and without moving on to + * Organize. A normal run still does both. + */ +import * as React from 'react'; +import { renderHook, act, cleanup } from '@testing-library/react'; + +const mockState = { + analysis: { isAnalyzing: false, progress: null }, + files: { smartFolders: [] } +}; + +jest.mock('react-redux', () => ({ + useStore: jest.fn(() => ({ getState: jest.fn(() => mockState) })) +})); + +jest.mock('../src/shared/logger', () => { + const logger = { + setContext: jest.fn(), + info: jest.fn(), + debug: jest.fn(), + warn: jest.fn(), + error: jest.fn() + }; + return { logger, createLogger: jest.fn(() => logger) }; +}); + +jest.mock('../src/shared/constants', () => ({ + PHASES: { DISCOVER: 'discover', ORGANIZE: 'organize', COMPLETE: 'complete' }, + FILE_STATES: { + PENDING: 'pending', + ANALYZING: 'analyzing', + CATEGORIZED: 'categorized', + ERROR: 'error' + }, + SUPPORTED_AUDIO_EXTENSIONS: ['.mp3', '.wav'], + SUPPORTED_VIDEO_EXTENSIONS: ['.mp4', '.mov'] +})); + +jest.mock('../src/renderer/phases/discover/namingUtils', () => ({ + validateProgressState: jest.fn( + (progress) => + progress && + typeof progress.current === 'number' && + typeof progress.total === 'number' && + progress.current >= 0 && + progress.total >= 0 + ), + generatePreviewName: jest.fn((name) => name), + generateSuggestedNameFromAnalysis: jest.fn(({ originalFileName }) => originalFileName), + makeUniqueFileName: jest.fn((candidate) => candidate), + extractFileName: jest.fn((filePath) => { + const parts = String(filePath || '').split(/[\\/]/); + return parts[parts.length - 1] || String(filePath || ''); + }) +})); + +import { useAnalysis } from '../src/renderer/phases/discover/useAnalysis'; + +const { getFolderDefinitionsHash } = require('../src/shared/smartFolderDefinitions'); + +const FOLDERS = [ + { + id: 'f-inv', + name: 'Invoices', + description: 'Bills and invoices from suppliers', + path: '/Users/alex/Library/Invoices' + } +]; + +function useStoreBackedAnalysis(baseOptions) { + const [analysisResults, setResults] = React.useState([]); + const [fileStates, setStates] = React.useState({}); + const setAnalysisResults = React.useCallback( + (value) => setResults((prev) => (typeof value === 'function' ? value(prev) : value)), + [] + ); + const setFileStates = React.useCallback( + (value) => setStates((prev) => (typeof value === 'function' ? value(prev) : value)), + [] + ); + const setters = React.useMemo( + () => ({ ...baseOptions.setters, setAnalysisResults, setFileStates }), + [baseOptions.setters, setAnalysisResults, setFileStates] + ); + const hook = useAnalysis({ ...baseOptions, analysisResults, fileStates, setters }); + return { hook, analysisResults }; +} + +describe('useAnalysis with Smart Folder definitions', () => { + let baseOptions; + + beforeEach(() => { + jest.useFakeTimers(); + mockState.files.smartFolders = FOLDERS; + global.window.electronAPI = { + files: { + analyze: jest.fn(async () => ({ category: 'Invoices', suggestedName: 'invoice.pdf' })) + }, + analysis: { + batch: jest.fn(), + cancelBatch: jest.fn().mockResolvedValue({ success: true }), + getReadyQueue: jest.fn().mockResolvedValue({ success: true, readyFiles: [] }), + clearReadyQueue: jest.fn().mockResolvedValue({ success: true }) + }, + settings: { get: jest.fn().mockResolvedValue({ maxConcurrentAnalysis: 3 }) } + }; + baseOptions = { + selectedFiles: [], + isAnalyzing: false, + analysisProgress: { current: 0, total: 0 }, + namingSettings: { + convention: 'keep-original', + separator: '-', + dateFormat: 'YYYY-MM-DD', + caseConvention: 'none' + }, + setters: { + setIsAnalyzing: jest.fn(), + setAnalysisProgress: jest.fn(), + setCurrentAnalysisFile: jest.fn(), + setAnalysisResults: jest.fn(), + setFileStates: jest.fn() + }, + updateFileState: jest.fn(), + addNotification: jest.fn(), + actions: { setPhaseData: jest.fn(), advancePhase: jest.fn() }, + getCurrentPhase: () => 'discover' + }; + }); + + afterEach(() => { + cleanup(); + jest.useRealTimers(); + }); + + async function runAndSettle(result, files, runOptions) { + await act(async () => { + const run = result.current.hook.analyzeFiles(files, runOptions); + await jest.advanceTimersByTimeAsync(1000); + await run; + // Past the 2 s auto-advance delay. + await jest.advanceTimersByTimeAsync(3000); + }); + } + + const files = [ + { path: '/Users/alex/Inbox/a.pdf', name: 'a.pdf' }, + { path: '/Users/alex/Inbox/b.pdf', name: 'b.pdf' } + ]; + + test('each result carries the folder definitions it was analyzed under', async () => { + const { result } = renderHook(() => useStoreBackedAnalysis(baseOptions)); + await runAndSettle(result, files); + + const expected = getFolderDefinitionsHash(FOLDERS); + expect(expected).toMatch(/^sf-/); + expect(result.current.analysisResults).toHaveLength(2); + for (const r of result.current.analysisResults) { + expect(r.analysis.smartFolderHash).toBe(expected); + } + }); + + test('a sample run ends without the completion notice or the move to Organize', async () => { + const { result } = renderHook(() => useStoreBackedAnalysis(baseOptions)); + await runAndSettle(result, files, { origin: 'sample' }); + + expect(result.current.analysisResults).toHaveLength(2); + expect(baseOptions.setters.setIsAnalyzing).toHaveBeenCalledWith( + true, + expect.objectContaining({ origin: 'sample' }) + ); + const completionCalls = baseOptions.addNotification.mock.calls.filter( + (call) => call[3] === 'analysis-complete' + ); + expect(completionCalls).toHaveLength(0); + expect(baseOptions.actions.advancePhase).not.toHaveBeenCalled(); + }); + + test('a normal run still reports completion and moves on to Organize', async () => { + const { result } = renderHook(() => useStoreBackedAnalysis(baseOptions)); + await runAndSettle(result, files); + + const completionCalls = baseOptions.addNotification.mock.calls.filter( + (call) => call[3] === 'analysis-complete' + ); + expect(completionCalls).toHaveLength(1); + expect(baseOptions.actions.advancePhase).toHaveBeenCalledWith('organize'); + }); +}); diff --git a/test/useSampleRun.test.js b/test/useSampleRun.test.js new file mode 100644 index 00000000..a3ffb73f --- /dev/null +++ b/test/useSampleRun.test.js @@ -0,0 +1,306 @@ +/** + * @jest-environment jsdom + * + * Sample first, the flow: a large batch is offered a sample before it runs; + * the user can skip it, or try a few files, see where they would go, then + * continue with the rest or go edit the folders. The offer does not come + * back for the same Smart Folder definitions, and does once they change. + */ +import React from 'react'; +import { Provider } from 'react-redux'; +import { configureStore } from '@reduxjs/toolkit'; +import { renderHook, act } from '@testing-library/react'; +import uiReducer from '../src/renderer/store/slices/uiSlice'; +import filesReducer, { setSmartFolders } from '../src/renderer/store/slices/filesSlice'; +import analysisReducer, { setAnalysisResults } from '../src/renderer/store/slices/analysisSlice'; +import { useSampleRun } from '../src/renderer/phases/discover/useSampleRun'; +import { SAMPLE_SIZE } from '../src/renderer/phases/discover/sampleRun'; + +const { getFolderDefinitionsHash } = require('../src/shared/smartFolderDefinitions'); + +const INVOICES = { + id: 'f-inv', + name: 'Invoices', + description: 'Bills and invoices from suppliers, not receipts', + path: '/Users/alex/Library/Invoices' +}; +const PHOTOS = { + id: 'f-photo', + name: 'Photos', + description: 'Pictures taken with the phone or camera', + path: '/Users/alex/Library/Photos' +}; +const FOLDERS = [INVOICES, PHOTOS]; + +const makeFiles = (count) => + Array.from({ length: count }, (_, i) => { + const ext = i % 2 === 0 ? 'pdf' : 'jpg'; + return { path: `/Users/alex/Inbox/file-${i}.${ext}`, name: `file-${i}.${ext}` }; + }); + +function makeStore() { + const store = configureStore({ + reducer: { ui: uiReducer, files: filesReducer, analysis: analysisReducer }, + middleware: (getDefault) => getDefault({ serializableCheck: false, immutableCheck: false }) + }); + store.dispatch(setSmartFolders(FOLDERS)); + return store; +} + +function setup(store = makeStore()) { + const analyzeFiles = jest.fn(async () => ({ started: true })); + const cancelAnalysis = jest.fn(); + const advancePhase = jest.fn(); + const wrapper = ({ children }) => {children}; + const hook = renderHook(() => useSampleRun({ analyzeFiles, cancelAnalysis, advancePhase }), { + wrapper + }); + return { store, hook, analyzeFiles, cancelAnalysis, advancePhase }; +} + +/** The sample run: record a result for each file, as the analysis run does. */ +function analyzeSampleInto(store, analyzeFiles) { + analyzeFiles.mockImplementationOnce(async (files) => { + store.dispatch( + setAnalysisResults( + files.map((file) => ({ + ...file, + analysis: { category: file.name.endsWith('.jpg') ? 'Photos' : 'Invoices' } + })) + ) + ); + return { started: true }; + }); +} + +beforeEach(() => { + window.electronAPI = { + suggestions: { + getBatchSuggestions: jest.fn().mockResolvedValue({ success: true, groups: [] }) + } + }; +}); + +afterEach(() => { + delete window.electronAPI; +}); + +describe('useSampleRun', () => { + test('a small batch starts at once, with no offer', async () => { + const { hook, analyzeFiles } = setup(); + const files = makeFiles(19); + let result; + await act(async () => { + result = await hook.result.current.analyzeFiles(files); + }); + expect(analyzeFiles).toHaveBeenCalledWith(files, undefined); + expect(result).toEqual({ started: true }); + expect(hook.result.current.session).toBeNull(); + }); + + test('a large batch is offered a sample and nothing runs yet', async () => { + const { hook, analyzeFiles } = setup(); + let result; + await act(async () => { + result = await hook.result.current.analyzeFiles(makeFiles(25)); + }); + expect(result).toEqual({ started: false, reason: 'sample-offered' }); + expect(analyzeFiles).not.toHaveBeenCalled(); + const { session } = hook.result.current; + expect(session.stage).toBe('offer'); + expect(session.files).toHaveLength(25); + expect(session.picked).toHaveLength(SAMPLE_SIZE); + expect(session.hash).toBe(getFolderDefinitionsHash(FOLDERS)); + }); + + test('runs the analysis starts itself are never held', async () => { + const { hook, analyzeFiles } = setup(); + const files = makeFiles(25); + await act(async () => { + await hook.result.current.analyzeFiles(files, { origin: 'queued' }); + await hook.result.current.analyzeFiles(files, { bypassCache: true }); + }); + expect(analyzeFiles).toHaveBeenCalledTimes(2); + expect(hook.result.current.session).toBeNull(); + }); + + test('skipped: the whole batch runs, and the offer stays away for these folders', async () => { + const { store, hook, analyzeFiles } = setup(); + const files = makeFiles(25); + await act(async () => { + await hook.result.current.analyzeFiles(files); + }); + await act(async () => { + await hook.result.current.skipSample(); + }); + expect(analyzeFiles).toHaveBeenCalledWith(files); + expect(hook.result.current.session).toBeNull(); + expect(store.getState().ui.smartFolderSampleHash).toBe(getFolderDefinitionsHash(FOLDERS)); + + // The next large batch with the same folders starts at once. + const more = makeFiles(30).map((f) => ({ ...f, path: f.path.replace('Inbox', 'More') })); + await act(async () => { + await hook.result.current.analyzeFiles(more); + }); + expect(analyzeFiles).toHaveBeenLastCalledWith(more, undefined); + expect(hook.result.current.session).toBeNull(); + }); + + test('continued: the sample is reviewed, then only the other files run', async () => { + const { store, hook, analyzeFiles } = setup(); + window.electronAPI.suggestions.getBatchSuggestions.mockResolvedValue({ + success: true, + groups: [ + { + files: [ + { + path: '/Users/alex/Inbox/file-0.pdf', + suggestion: { + folder: 'Invoices', + path: INVOICES.path, + folderId: INVOICES.id, + isSmartFolder: true + }, + confidence: 0.9, + explanation: 'Reads like a supplier invoice', + alternatives: [] + } + ] + } + ] + }); + const files = makeFiles(25); + await act(async () => { + await hook.result.current.analyzeFiles(files); + }); + const picked = hook.result.current.session.picked; + + analyzeSampleInto(store, analyzeFiles); + await act(async () => { + await hook.result.current.startSample(); + }); + + // The sample goes through the normal run, marked as a sample. + expect(analyzeFiles).toHaveBeenCalledWith(picked, { origin: 'sample' }); + const { session } = hook.result.current; + expect(session.stage).toBe('review'); + expect(session.rows).toHaveLength(SAMPLE_SIZE); + expect(session.rows[0]).toEqual({ + path: '/Users/alex/Inbox/file-0.pdf', + name: 'file-0.pdf', + analyzed: true, + folder: 'Invoices', + score: 0.9, + reason: 'Reads like a supplier invoice' + }); + expect(session.rows.find((row) => row.name.endsWith('.jpg'))).toMatchObject({ + folder: 'Photos', + score: null + }); + // Reviewing the sample is what is remembered. + expect(store.getState().ui.smartFolderSampleHash).toBe(getFolderDefinitionsHash(FOLDERS)); + + await act(async () => { + await hook.result.current.continueWithRest(); + }); + const sampled = new Set(picked.map((f) => f.path)); + const rest = files.filter((f) => !sampled.has(f.path)); + expect(analyzeFiles).toHaveBeenLastCalledWith(rest); + expect(rest).toHaveLength(25 - SAMPLE_SIZE); + expect(hook.result.current.session).toBeNull(); + }); + + test('not offered again until the definitions change, then offered again', async () => { + const { store, hook, analyzeFiles } = setup(); + await act(async () => { + await hook.result.current.analyzeFiles(makeFiles(25)); + }); + analyzeSampleInto(store, analyzeFiles); + await act(async () => { + await hook.result.current.startSample(); + }); + act(() => { + hook.result.current.dismiss(); + }); + + const second = makeFiles(25).map((f) => ({ ...f, path: f.path.replace('Inbox', 'Second') })); + await act(async () => { + await hook.result.current.analyzeFiles(second); + }); + expect(analyzeFiles).toHaveBeenLastCalledWith(second, undefined); + + // A reworded description is a new definition: the offer comes back. + act(() => { + store.dispatch( + setSmartFolders([{ ...INVOICES, description: 'Supplier invoices only' }, PHOTOS]) + ); + }); + const third = makeFiles(25).map((f) => ({ ...f, path: f.path.replace('Inbox', 'Third') })); + const callsBefore = analyzeFiles.mock.calls.length; + await act(async () => { + await hook.result.current.analyzeFiles(third); + }); + expect(analyzeFiles).toHaveBeenCalledTimes(callsBefore); + expect(hook.result.current.session.stage).toBe('offer'); + }); + + test('edit folders after the review: go to Setup, the rest stays unanalyzed', async () => { + const { store, hook, analyzeFiles, advancePhase } = setup(); + await act(async () => { + await hook.result.current.analyzeFiles(makeFiles(25)); + }); + analyzeSampleInto(store, analyzeFiles); + await act(async () => { + await hook.result.current.startSample(); + }); + act(() => { + hook.result.current.editFolders(); + }); + expect(advancePhase).toHaveBeenCalledWith('setup'); + expect(analyzeFiles).toHaveBeenCalledTimes(1); // the sample only + expect(hook.result.current.session).toBeNull(); + }); + + test('stop during the sample cancels the run and records nothing', async () => { + const { store, hook, analyzeFiles, cancelAnalysis } = setup(); + await act(async () => { + await hook.result.current.analyzeFiles(makeFiles(25)); + }); + let finish; + analyzeFiles.mockImplementationOnce( + () => + new Promise((resolve) => { + finish = resolve; + }) + ); + let sampleDone; + act(() => { + sampleDone = hook.result.current.startSample(); + }); + expect(hook.result.current.session.stage).toBe('running'); + + act(() => { + hook.result.current.stopSample(); + }); + expect(cancelAnalysis).toHaveBeenCalledTimes(1); + await act(async () => { + finish({ started: true }); + await sampleDone; + }); + expect(hook.result.current.session).toBeNull(); + expect(store.getState().ui.smartFolderSampleHash).toBeNull(); + }); + + test('files added while the offer is open join its batch', async () => { + const { hook, analyzeFiles } = setup(); + await act(async () => { + await hook.result.current.analyzeFiles(makeFiles(25)); + }); + const extra = [{ path: '/Users/alex/Inbox/late.pdf', name: 'late.pdf' }]; + await act(async () => { + await hook.result.current.analyzeFiles(extra); + }); + expect(analyzeFiles).not.toHaveBeenCalled(); + expect(hook.result.current.session.files).toHaveLength(26); + }); +}); From e15f1779496fa56e4d8d30e91f4759a71fb40a44 Mon Sep 17 00:00:00 2001 From: Levy Tate <78818969+iLevyTate@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:02:46 -0500 Subject: [PATCH 04/10] perf(analysis): send the head of each sheet for CSV and spreadsheet prompts A spreadsheet prompt carried about 3,700 characters sampled from the start, middle and end of the text, so most sheets lost their names and columns and the model read rows of numbers. Table text now gets each sheet's heading, its columns, its first 8 rows and a count of the rows left out, within 2,000 characters. A Search Console workbook goes from 3,739 to 1,517 characters with every sheet covered; CSV exports from about 3,800 to 500 to 750. --- src/main/analysis/contentSelector.js | 76 +++++++++++++++++++++++++++- src/shared/performanceConstants.js | 8 ++- test/contentSelector.test.js | 71 ++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 2 deletions(-) diff --git a/src/main/analysis/contentSelector.js b/src/main/analysis/contentSelector.js index 5377393c..42ccc614 100644 --- a/src/main/analysis/contentSelector.js +++ b/src/main/analysis/contentSelector.js @@ -268,13 +268,74 @@ function sampleMiddleContent(text, startOffset, endOffset, budget, sampleCount) return samples; } +// First line of a CSV or spreadsheet as the extractors write it +// (tableRowText in documentExtractors.js). +const TABLE_START_RE = /^(?:## Sheet: .+|Columns: .+\|.+)$/; +const SHEET_HEADING_RE = /^## Sheet: /; + +/** + * True for the text of a CSV or spreadsheet whose first table has a header + * row or a sheet heading. + * @param {string} cleaned + */ +function isTableText(cleaned) { + const firstLine = cleaned.slice(0, 400).split('\n', 1)[0].trim(); + return TABLE_START_RE.test(firstLine); +} + +/** + * The head of every table in a CSV or spreadsheet: its sheet heading, its + * columns, its first rows and how many rows were left out. Exports are told + * apart by their sheets and columns, not by rows of numbers, and the model + * reads every prompt token, so rows past the head of each sheet are dropped. + * Every sheet gets an equal share of the budget; a share a sheet does not use + * goes to the sheets after it. + * @param {string} cleaned + * @param {number} budget + * @returns {string} + */ +function selectTableHeads(cleaned, budget) { + /** @type {string[][]} */ + const tables = []; + for (const line of cleaned.split('\n')) { + if (SHEET_HEADING_RE.test(line) || tables.length === 0) tables.push([]); + if (line.trim()) tables[tables.length - 1].push(line); + } + const { TABLE_ROWS_PER_SHEET } = CONTENT_SELECTION; + /** @type {string[]} */ + const parts = []; + let left = budget; + tables.forEach((lines, i) => { + const share = Math.floor(left / (tables.length - i)); + const headerCount = lines.findIndex((l) => !SHEET_HEADING_RE.test(l) && !/^Columns: /.test(l)); + const header = headerCount === -1 ? lines : lines.slice(0, headerCount); + const rows = headerCount === -1 ? [] : lines.slice(headerCount); + const kept = [...header]; + let used = header.join('\n').length; + let shown = 0; + while (shown < rows.length && shown < TABLE_ROWS_PER_SHEET) { + const next = rows[shown]; + if (used + next.length + 1 > share) break; + kept.push(next); + used += next.length + 1; + shown += 1; + } + if (rows.length > shown) kept.push(`(${rows.length - shown} more rows)`); + const text = kept.join('\n').slice(0, Math.max(share, 0)); + parts.push(text); + left -= text.length + 1; + }); + return parts.join('\n').slice(0, budget); +} + /** * Select representative content from a document to maximize information * density within a fixed character budget. * * For documents that fit within the budget, returns the full cleaned text. * For larger documents, strategically samples beginning, middle, and end - * with a structural outline prepended. + * with a structural outline prepended. A CSV or spreadsheet gets the head of + * each table instead, within at most TABLE_MAX_CHARS. * * @param {string} text - Full extracted document text * @param {number} budget - Maximum character budget for the output @@ -289,6 +350,19 @@ function selectRepresentativeContent(text, budget) { const cleaned = lightClean(text); const totalLength = cleaned.length; + if (isTableText(cleaned)) { + const tableBudget = Math.min(budget, CONTENT_SELECTION.TABLE_MAX_CHARS); + if (totalLength > tableBudget) { + const content = selectTableHeads(cleaned, tableBudget); + return { + content, + strategy: 'table-heads', + coveragePercent: Math.min(100, Math.round((content.length / totalLength) * 100)), + totalLength + }; + } + } + // If text fits within budget, return it all — no selection needed if (totalLength <= budget) { return { diff --git a/src/shared/performanceConstants.js b/src/shared/performanceConstants.js index f23a923d..32bf7d01 100644 --- a/src/shared/performanceConstants.js +++ b/src/shared/performanceConstants.js @@ -386,7 +386,13 @@ const CONTENT_SELECTION = { // Map phase: temperature (0 = deterministic, fastest generation) MAP_TEMPERATURE: 0, // Map phase: chunk size in characters - MAP_CHUNK_SIZE: 8000 + MAP_CHUNK_SIZE: 8000, + // CSV and spreadsheet prompts: the head of each sheet (heading, columns, + // first rows) within this many characters. On 275 analytics exports the + // model call was about 18 s of each file's 19 s, with an 8,100-character + // prompt; sheets and columns tell exports apart, rows of numbers do not. + TABLE_MAX_CHARS: 2000, + TABLE_ROWS_PER_SHEET: 8 }; const LIMITS = { diff --git a/test/contentSelector.test.js b/test/contentSelector.test.js index dc79de73..e48e0df9 100644 --- a/test/contentSelector.test.js +++ b/test/contentSelector.test.js @@ -295,3 +295,74 @@ describe('selectRepresentativeContent', () => { expect(result.coveragePercent).toBe(10); }); }); + +// --------------------------------------------------------------------------- +// CSV and spreadsheet text (the extractors' "## Sheet:" and "Columns:" lines) +// --------------------------------------------------------------------------- + +describe('selectRepresentativeContent on table text', () => { + const sheet = (name, columns, rows) => + [ + `## Sheet: ${name}`, + `Columns: ${columns.join(' | ')}`, + ...Array.from({ length: rows }, (_, i) => `${name.toLowerCase()} row ${i} 12 345 6.7% 8.9`) + ].join('\n'); + const workbook = [ + sheet('Queries', ['Top queries', 'Clicks', 'Impressions'], 900), + sheet('Pages', ['Top pages', 'Clicks', 'Impressions'], 900), + sheet('Devices', ['Device', 'Clicks'], 3), + ['## Sheet: Filters', 'Columns: Filter | Value', 'Property sc-domain:example.com'].join('\n') + ].join('\n'); + + test('keeps every sheet heading and its columns, within the table limit', () => { + const result = selectRepresentativeContent(workbook, 30000); + expect(result.strategy).toBe('table-heads'); + expect(result.content.length).toBeLessThanOrEqual(2000); + for (const name of ['Queries', 'Pages', 'Devices', 'Filters']) { + expect(result.content).toContain(`## Sheet: ${name}`); + } + expect(result.content).toContain('Columns: Top pages | Clicks | Impressions'); + // The last sheet survives: head/middle/tail sampling kept it only by chance. + expect(result.content).toContain('Property sc-domain:example.com'); + }); + + test('keeps the first rows of each sheet and says how many were left out', () => { + const { content } = selectRepresentativeContent(workbook, 30000); + expect(content).toContain('queries row 0 '); + expect(content).not.toContain('queries row 8 '); + expect(content).toContain('(892 more rows)'); + // A short sheet is kept whole, with no count line. + expect(content).toContain('devices row 2 '); + expect(content).not.toMatch(/devices row 2 [^\n]*\n\(\d+ more rows\)/); + }); + + test('a single CSV table gets the same treatment', () => { + const csv = [ + 'Columns: Keyword | Impressions | Clicks', + ...Array.from({ length: 500 }, (_, i) => `keyword ${i} 1000 10`) + ].join('\n'); + const result = selectRepresentativeContent(csv, 30000); + expect(result.strategy).toBe('table-heads'); + expect(result.content.split('\n')[0]).toBe('Columns: Keyword | Impressions | Clicks'); + expect(result.content).toContain('(492 more rows)'); + }); + + test('a smaller budget than the table limit still holds', () => { + const result = selectRepresentativeContent(workbook, 500); + expect(result.content.length).toBeLessThanOrEqual(500); + expect(result.content).toContain('## Sheet: Queries'); + }); + + test('a table that fits is returned whole', () => { + const small = sheet('Devices', ['Device', 'Clicks'], 3); + expect(selectRepresentativeContent(small, 30000)).toMatchObject({ + strategy: 'full', + content: small + }); + }); + + test('prose that mentions columns is not treated as a table', () => { + const prose = `Columns: the report lists clicks and impressions.\n${'Body text. '.repeat(2000)}`; + expect(selectRepresentativeContent(prose, 3000).strategy).toBe('representative'); + }); +}); From 7fce387887816e2732f19747fbb0bf065793b762 Mon Sep 17 00:00:00 2001 From: Levy Tate <78818969+iLevyTate@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:03:18 -0500 Subject: [PATCH 05/10] chore: raise the checkjs floor to 512 and note the open-item fixes in the changelog --- CHANGELOG.md | 29 +++++++++++++++++++++++++++++ CLAUDE.md | 2 +- scripts/verify-checkjs-ratchet.js | 2 +- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 215a3a42..03992dab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,35 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- **Try a few files before a large batch.** When you start analyzing 20 or more files and your Smart + Folders changed since the last sample, StratoSort offers to analyze 5 of them first, picked across + the file types in the batch. It shows where each would go and why, without moving anything. You + can analyze the rest, or stop and edit your folders. Skipping counts as a choice, so the offer + comes back only after a folder's name, description or path changes. The user guide has a new + section on writing folder descriptions. + +### Changed + +- **A file no Smart Folder fits goes to review instead of the closest folder.** A file now goes to a + Smart Folder only when a learned correction points there, its content matches one folder clearly + (score 0.55 or more, and no other folder within 0.03), or the folder is named after its file type + (or, when the AI model does the routing, after its label). Otherwise the card says "Needs review", + names the closest folders, and the file goes to your default folder unless you pick one. Before, + any folder that reached 0.4 won, so an unrelated Markdown report could land in an SEO folder. + Auto-organize and the Downloads watcher hold such files instead of moving them. +- **A folder you sort that is also a Smart Folder now asks you to choose.** Picking or dropping a + folder on Discover that is, sits inside, or holds a Smart Folder opens a choice instead of a + warning. On Organize, files that would stay in their own folder need a choice (leave them there or + skip them) before Move turns on. Each choice is kept for that Smart Folder. +- **Changing a Smart Folder's location to a folder that does not exist yet asks first.** It moved + the old folder there, files included, without asking. You now choose: move the folder and its + files, start with a new empty folder, or cancel. +- **Suggestions made before you changed your Smart Folders are flagged.** Organize lists the files + that still go where your old folders sent them, with one button to analyze them again, and Move + asks before moving them. + ## [5.3.1] - 2026-11-14 ### Fixed diff --git a/CLAUDE.md b/CLAUDE.md index 97153e8b..7b3e8594 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -103,7 +103,7 @@ tree. `ci`) enforces a monotonic floor and blocks new .js under `COVERED_DIRS` (`src/shared`, `src/preload`, `src/main/{ipc,services,analysis,errors,utils,workers,transports,core}`, `src/renderer/{services,hooks,phases,store,utils,components}` today (every `.js` under `src/` is - now opted in — 507/507 as of 2026-11-14, and `GLOBAL_DIRECTIVE_FLOOR` matches at 507) — see + now opted in — 512/512 as of 2026-11-15, and `GLOBAL_DIRECTIVE_FLOOR` matches at 512) — see `scripts/verify-checkjs-ratchet.js` for the authoritative list) from landing without the directive. When adding a file to a covered dir: put `// @ts-check` on line 1 and fix the errors before commit. When opting in a new file elsewhere: bump `GLOBAL_DIRECTIVE_FLOOR` in diff --git a/scripts/verify-checkjs-ratchet.js b/scripts/verify-checkjs-ratchet.js index 25c7c07c..058171e3 100644 --- a/scripts/verify-checkjs-ratchet.js +++ b/scripts/verify-checkjs-ratchet.js @@ -43,7 +43,7 @@ const COVERED_DIRS = [ 'src/renderer/utils', 'src/renderer/components' ]; -const GLOBAL_DIRECTIVE_FLOOR = 507; +const GLOBAL_DIRECTIVE_FLOOR = 512; // ──────────────────────────────────────────────────────────────────────── const DIRECTIVE_SCAN_LINES = 5; From 4043c0015f5f5ebcb6e49f77a2d38910720e5434 Mon Sep 17 00:00:00 2001 From: Levy Tate <78818969+iLevyTate@users.noreply.github.com> Date: Sun, 15 Nov 2026 18:30:21 -0500 Subject: [PATCH 06/10] fix(memory): stop the search index save from leaking about 35 MB per analyzed file lz4-napi's async compress() and uncompress() never free their native work buffers. Compressing a 60 MB index 12 times grew the process from 186 MB to 1,322 MB; compressSync() stayed flat. The search index saves itself every few files during a batch, so a 275-file analysis grew the app from 5.1 GB to 14.6 GB of private memory while the JS heap stayed near 60 MB. Text generation and embeddings were measured flat on their own. The async wrappers now call the sync binding (the async one is kept only as a fallback when the sync API is missing). --- src/shared/lz4Codec.js | 19 +++++++++++++------ test/lz4Codec.test.js | 33 +++++++++++++++++++++++++++------ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/shared/lz4Codec.js b/src/shared/lz4Codec.js index b23524df..7997e70e 100644 --- a/src/shared/lz4Codec.js +++ b/src/shared/lz4Codec.js @@ -55,27 +55,34 @@ const uncompressSync = (input) => { return buffer; }; +// lz4-napi's async compress() and uncompress() never free their native work +// buffers: compressing a 60 MB index 12 times grew the process by 1.1 GB, +// while compressSync() stayed flat. The search index saves itself every few +// files during a batch, so the app grew by about 35 MB per file analyzed. +// The async wrappers below therefore call the sync binding. LZ4 runs at +// hundreds of MB/s, so a 200 MB index blocks for well under a second. + /** @param {Buffer | ArrayBufferView | ArrayBuffer | string | null | undefined} input */ const compress = async (input) => { const buffer = toBuffer(input); - if (lz4Binding && typeof lz4Binding.compress === 'function') { - return lz4Binding.compress(buffer); - } if (lz4Binding && typeof lz4Binding.compressSync === 'function') { return lz4Binding.compressSync(buffer); } + if (lz4Binding && typeof lz4Binding.compress === 'function') { + return lz4Binding.compress(buffer); + } return buffer; }; /** @param {Buffer | ArrayBufferView | ArrayBuffer | string | null | undefined} input */ const uncompress = async (input) => { const buffer = toBuffer(input); - if (lz4Binding && typeof lz4Binding.uncompress === 'function') { - return lz4Binding.uncompress(buffer); - } if (lz4Binding && typeof lz4Binding.uncompressSync === 'function') { return lz4Binding.uncompressSync(buffer); } + if (lz4Binding && typeof lz4Binding.uncompress === 'function') { + return lz4Binding.uncompress(buffer); + } return buffer; }; diff --git a/test/lz4Codec.test.js b/test/lz4Codec.test.js index e5969159..9710a79b 100644 --- a/test/lz4Codec.test.js +++ b/test/lz4Codec.test.js @@ -47,13 +47,34 @@ describe('shared/lz4Codec', () => { expect(codec.uncompressSync(Buffer.from('sync:payload', 'utf8')).toString('utf8')).toBe( 'payload' ); - await expect(codec.compress(input)).resolves.toEqual(Buffer.from('async:payload', 'utf8')); - await expect(codec.uncompress(Buffer.from('async:payload', 'utf8'))).resolves.toEqual( + // The async wrappers call the sync binding: lz4-napi's async compress() + // and uncompress() leak their native buffers (about 100 MB per call on a + // 60 MB search index), which grew the app by ~35 MB per analyzed file. + await expect(codec.compress(input)).resolves.toEqual(Buffer.from('sync:payload', 'utf8')); + await expect(codec.uncompress(Buffer.from('sync:payload', 'utf8'))).resolves.toEqual( + Buffer.from('payload', 'utf8') + ); + expect(mockBinding.compressSync).toHaveBeenCalledTimes(2); + expect(mockBinding.uncompressSync).toHaveBeenCalledTimes(2); + expect(mockBinding.compress).not.toHaveBeenCalled(); + expect(mockBinding.uncompress).not.toHaveBeenCalled(); + }); + + test('uses the async binding only when the sync one is missing', async () => { + const mockBinding = { + compress: jest.fn(async (buf) => Buffer.from(`async:${buf.toString('utf8')}`, 'utf8')), + uncompress: jest.fn(async (buf) => + Buffer.from(buf.toString('utf8').replace(/^async:/, ''), 'utf8') + ) + }; + jest.doMock('lz4-napi', () => mockBinding); + + const codec = require('../src/shared/lz4Codec'); + await expect(codec.compress(Buffer.from('payload'))).resolves.toEqual( + Buffer.from('async:payload', 'utf8') + ); + await expect(codec.uncompress(Buffer.from('async:payload'))).resolves.toEqual( Buffer.from('payload', 'utf8') ); - expect(mockBinding.compressSync).toHaveBeenCalledTimes(1); - expect(mockBinding.uncompressSync).toHaveBeenCalledTimes(1); - expect(mockBinding.compress).toHaveBeenCalledTimes(1); - expect(mockBinding.uncompress).toHaveBeenCalledTimes(1); }); }); From 8f013e8cb524562d650e80b9ae3e396a5e59fe63 Mon Sep 17 00:00:00 2001 From: Levy Tate <78818969+iLevyTate@users.noreply.github.com> Date: Sun, 15 Nov 2026 19:06:50 -0500 Subject: [PATCH 07/10] fix(analysis): keep analysis results for 12 hours instead of 30 minutes A 275-file batch takes about two hours. With a 30-minute expiry, an identical copy reached more than 30 minutes after its twin, or a file analyzed again later in the session, went through the model a second time (a measured second pass over 117 unchanged files hit the cache 0 times). Both caches stay capped (500 file results, 500 documents, 300 images), and their keys already change with the file's size and modified time, the model, the Smart Folders and the instructions, so the expiry is only a backstop. Changelog: the memory fix, this one and the spreadsheet prompt change. --- CHANGELOG.md | 13 ++++++++++++ src/main/analysis/documentAnalysis.js | 5 ++++- src/main/services/AnalysisCacheService.js | 10 +++++++-- ...lysisCacheService.characterization.test.js | 21 +++++++++++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03992dab..7e1cc4df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,16 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). comes back only after a folder's name, description or path changes. The user guide has a new section on writing folder descriptions. +### Fixed + +- **Memory no longer climbs during a long analysis.** Each save of the search index leaked its + compression buffers, so the app grew by about 35 MB for every file analyzed: from 5 GB to almost + 15 GB over 275 files, which pushed the system into swap. +- **A long batch no longer analyzes its early files again.** Analysis results were kept for 30 + minutes, so in a batch of a few hundred files, an identical copy or a file analyzed again later + went through the AI a second time. Results are now kept for 12 hours; they still change when the + file, the model or your Smart Folders change. + ### Changed - **A file no Smart Folder fits goes to review instead of the closest folder.** A file now goes to a @@ -37,6 +47,9 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - **Changing a Smart Folder's location to a folder that does not exist yet asks first.** It moved the old folder there, files included, without asking. You now choose: move the folder and its files, start with a new empty folder, or cancel. +- **Spreadsheets and CSVs send the AI the head of each sheet.** The AI saw a sample from the start, + middle and end of the text, so most sheets lost their names and columns. It now gets each sheet's + name, its columns and its first rows, in less than half the text. - **Suggestions made before you changed your Smart Folders are flagged.** Organize lists the files that still go where your old folders sent them, with one button to analyze them again, and Move asks before moving them. diff --git a/src/main/analysis/documentAnalysis.js b/src/main/analysis/documentAnalysis.js index bff7c04b..e04787ed 100644 --- a/src/main/analysis/documentAnalysis.js +++ b/src/main/analysis/documentAnalysis.js @@ -75,7 +75,10 @@ const CACHE_CONFIG = { MAX_FILE_CACHE: 500, // Maximum number of files to cache in memory FALLBACK_CONFIDENCE: 65, // Confidence score for fallback analysis DEFAULT_CONFIDENCE: 85, // Default confidence for successful analysis - CACHE_TTL_MS: 30 * 60 * 1000 // 30 minute TTL to prevent memory leaks + // MAX_FILE_CACHE bounds memory, and the key holds the file's size and + // modified time, so the TTL is a backstop: 30 minutes made a long batch + // (275 files take about two hours) analyze its early files again. + CACHE_TTL_MS: 12 * 60 * 60 * 1000 }; // v3: spreadsheet and CSV text now carries sheet names and a column line. const ANALYSIS_SIGNATURE_VERSION = 'v3'; diff --git a/src/main/services/AnalysisCacheService.js b/src/main/services/AnalysisCacheService.js index edc5a829..ce7d9265 100644 --- a/src/main/services/AnalysisCacheService.js +++ b/src/main/services/AnalysisCacheService.js @@ -6,13 +6,19 @@ const { getInstance: getCacheInvalidationBus } = require('../../shared/cacheInva const { LRUCache } = require('../../shared/LRUCache'); const logger = createLogger('AnalysisCacheService'); +// Entries are bounded by maxEntries, and their keys change when the content, +// model, Smart Folders or instructions change, so the TTL is only a backstop. +// It used to be 30 minutes: a 275-file batch takes about two hours, so a copy +// of a file analyzed more than 30 minutes earlier was analyzed again. +const ANALYSIS_RESULT_TTL_MS = 12 * 60 * 60 * 1000; + /** * Default configurations for different cache types * @type {Record} */ const CACHE_TYPE_DEFAULTS = { - document: { maxEntries: 500, ttlMs: 30 * 60 * 1000, name: 'documentCache' }, - image: { maxEntries: 300, ttlMs: 30 * 60 * 1000, name: 'imageCache' }, + document: { maxEntries: 500, ttlMs: ANALYSIS_RESULT_TTL_MS, name: 'documentCache' }, + image: { maxEntries: 300, ttlMs: ANALYSIS_RESULT_TTL_MS, name: 'imageCache' }, llm: { maxEntries: 200, ttlMs: 60 * 60 * 1000, name: 'llmCache' } }; diff --git a/test/AnalysisCacheService.characterization.test.js b/test/AnalysisCacheService.characterization.test.js index c217807f..57574898 100644 --- a/test/AnalysisCacheService.characterization.test.js +++ b/test/AnalysisCacheService.characterization.test.js @@ -279,6 +279,27 @@ describe('createAnalysisCache Factory', () => { cache.shutdown(); }); + test('a document or image result outlives a two-hour batch', () => { + // A 275-file batch takes about two hours. With the old 30-minute expiry, + // a copy reached later than that was analyzed again from scratch. + const nowSpy = jest.spyOn(Date, 'now'); + const start = 1_800_000_000_000; + nowSpy.mockReturnValue(start); + const documents = createAnalysisCache('document'); + const images = createAnalysisCache('image'); + try { + documents.set('doc-key', { category: 'GSC' }); + images.set('img-key', { category: 'Photos' }); + nowSpy.mockReturnValue(start + 2 * 60 * 60 * 1000); + expect(documents.get('doc-key')).toEqual({ category: 'GSC' }); + expect(images.get('img-key')).toEqual({ category: 'Photos' }); + } finally { + documents.shutdown(); + images.shutdown(); + nowSpy.mockRestore(); + } + }); + test('createAnalysisCache() allows option overrides', () => { const cache = createAnalysisCache('document', { maxEntries: 1000 }); expect(cache.getStats().maxEntries).toBe(1000); From 15832ebf2d28aa9b6291c27f3a1e1913282c76cb Mon Sep 17 00:00:00 2001 From: Levy Tate <78818969+iLevyTate@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:51:07 -0500 Subject: [PATCH 08/10] perf(search): keep 12 passages for a spreadsheet or CSV instead of up to 80 Rows of an export are nearly alike, so most spreadsheets reached the 80-passage limit: a 275-file batch of exports stored 14,473 passages, each about 9.5 KB in memory and about 30 KB on disk, and every save of the search index rewrote all of them. chunkOptionsFor() caps text that starts like a table (a sheet heading or a column line) at MAX_TABLE_CHUNKS and is shared by live analysis and both Settings rebuild paths. On 30 exports the passages went from 1,452 to 354 and the median save from 170 ms to 93 ms. --- src/main/analysis/chunkEmbedding.js | 27 +++++++++++++---- src/main/analysis/contentSelector.js | 1 + src/main/ipc/semantic.js | 13 ++------ src/shared/performanceConstants.js | 5 ++++ test/chunkEmbedding.test.js | 45 +++++++++++++++++++++++++++- 5 files changed, 74 insertions(+), 17 deletions(-) diff --git a/src/main/analysis/chunkEmbedding.js b/src/main/analysis/chunkEmbedding.js index 56c1dee4..04c1e52d 100644 --- a/src/main/analysis/chunkEmbedding.js +++ b/src/main/analysis/chunkEmbedding.js @@ -10,6 +10,7 @@ const path = require('path'); const { chunkText } = require('../utils/textChunking'); +const { isTableText } = require('./contentSelector'); const { CHUNKING } = require('../../shared/performanceConstants'); const { AI_DEFAULTS } = require('../../shared/constants'); const { isAutoChunkEnabled } = require('../../shared/experienceMode'); @@ -42,6 +43,23 @@ async function isChunkingEnabled(loadSettings) { } } +/** + * How `text` is cut into passages. A CSV or spreadsheet keeps only its first + * MAX_TABLE_CHUNKS: past the head of each sheet its rows add near-duplicate + * vectors that cost memory and save time for little recall. + * @param {string} text + * @returns {{ chunkSize: number, overlap: number, maxChunks: number }} + */ +function chunkOptionsFor(text) { + return { + chunkSize: CHUNKING.CHUNK_SIZE, + overlap: CHUNKING.OVERLAP, + maxChunks: isTableText(String(text || '').trimStart()) + ? CHUNKING.MAX_TABLE_CHUNKS + : CHUNKING.MAX_CHUNKS + }; +} + /** * Embed the chunks of `text` into `batchUpsertFileChunks` payloads. Pure with * respect to the store: nothing is written. A chunk whose embedding fails is @@ -58,11 +76,7 @@ async function isChunkingEnabled(loadSettings) { */ async function buildFileChunkPayloads({ fileId, filePath, text, embedText, signal }) { const fileName = path.basename(filePath); - const chunks = chunkText(text, { - chunkSize: CHUNKING.CHUNK_SIZE, - overlap: CHUNKING.OVERLAP, - maxChunks: CHUNKING.MAX_CHUNKS - }); + const chunks = chunkText(text, chunkOptionsFor(text)); /** @type {Array>} */ const payloads = []; for (const c of chunks) { @@ -179,5 +193,6 @@ async function embedFileChunks({ module.exports = { embedFileChunks, - buildFileChunkPayloads + buildFileChunkPayloads, + chunkOptionsFor }; diff --git a/src/main/analysis/contentSelector.js b/src/main/analysis/contentSelector.js index 42ccc614..55fd5301 100644 --- a/src/main/analysis/contentSelector.js +++ b/src/main/analysis/contentSelector.js @@ -441,6 +441,7 @@ function selectRepresentativeContent(text, budget) { } module.exports = { + isTableText, selectRepresentativeContent, extractDocumentOutline, sampleMiddleContent diff --git a/src/main/ipc/semantic.js b/src/main/ipc/semantic.js index eb4f6b49..def7adf2 100644 --- a/src/main/ipc/semantic.js +++ b/src/main/ipc/semantic.js @@ -46,6 +46,7 @@ const { schemas: validationSchemas } = require('./validationSchemas'); const { cosineSimilarity, padOrTruncateVector } = require('../../shared/vectorMath'); const { categorizeModel } = require('../../shared/modelCategorization'); const { chunkText } = require('../utils/textChunking'); +const { chunkOptionsFor } = require('../analysis/chunkEmbedding'); const { normalizeText } = require('../../shared/normalization'); const { getFileEmbeddingId } = require('../utils/fileIdUtils'); const { enrichFolderTextForEmbedding } = require('../analysis/semanticExtensionMap'); @@ -2818,11 +2819,7 @@ async function handleFullRebuild( // Also process chunks from extracted text const extractedText = analysis.extractedText || ''; if (shouldRebuildChunks && extractedText.length >= CHUNKING.MIN_TEXT_LENGTH) { - const chunks = chunkText(extractedText, { - chunkSize: CHUNKING.CHUNK_SIZE, - overlap: CHUNKING.OVERLAP, - maxChunks: CHUNKING.MAX_CHUNKS - }); + const chunks = chunkText(extractedText, chunkOptionsFor(extractedText)); for (const chunk of chunks) { throwIfInterrupted(); @@ -3563,11 +3560,7 @@ async function handleRebuildFiles( pendingChunkGroups.set(fileId, fileChunkPayloads); if (extractedText.trim().length >= CHUNKING.MIN_TEXT_LENGTH) { - const chunks = chunkText(extractedText, { - chunkSize: CHUNKING.CHUNK_SIZE, - overlap: CHUNKING.OVERLAP, - maxChunks: CHUNKING.MAX_CHUNKS - }); + const chunks = chunkText(extractedText, chunkOptionsFor(extractedText)); for (const c of chunks) { try { diff --git a/src/shared/performanceConstants.js b/src/shared/performanceConstants.js index 32bf7d01..3d772e3d 100644 --- a/src/shared/performanceConstants.js +++ b/src/shared/performanceConstants.js @@ -355,6 +355,11 @@ const CHUNKING = { OVERLAP: 200, // Maximum number of chunks per document (prevents runaway chunking) MAX_CHUNKS: 80, + // Maximum for a CSV or spreadsheet. Their rows are near-identical, so most + // exports hit MAX_CHUNKS: 192 files of a 275-file batch of exports stored + // 10,532 passages, each kept twice on disk (about 30 KB) and in memory + // (about 9.5 KB), and every save rewrote all of them. + MAX_TABLE_CHUNKS: 12, // Minimum text length to trigger chunking MIN_TEXT_LENGTH: 200, // Maximum chunks to return per file in search results diff --git a/test/chunkEmbedding.test.js b/test/chunkEmbedding.test.js index 367fba88..2943d653 100644 --- a/test/chunkEmbedding.test.js +++ b/test/chunkEmbedding.test.js @@ -6,7 +6,11 @@ jest.mock('../src/shared/logger', () => ({ createLogger: () => ({ info: jest.fn(), warn: jest.fn(), error: jest.fn(), debug: jest.fn() }) })); -const { embedFileChunks, buildFileChunkPayloads } = require('../src/main/analysis/chunkEmbedding'); +const { + embedFileChunks, + buildFileChunkPayloads, + chunkOptionsFor +} = require('../src/main/analysis/chunkEmbedding'); const { CHUNKING } = require('../src/shared/performanceConstants'); const LONG_TEXT = Array.from({ length: 60 }, (_, i) => `Sentence ${i} of a long document. `).join( @@ -54,6 +58,45 @@ describe('chunkEmbedding', () => { expect(payloads[1].id).toBe('chunk:file:/docs/a.pdf:1'); }); + describe('spreadsheets and CSVs', () => { + const rows = (n) => + Array.from( + { length: n }, + (_, i) => `buy bread near me ${i} | ${i * 7} | ${i * 70} | 1.${i}%` + ).join('\n'); + const SHEET_TEXT = `## Sheet: Queries\nColumns: Query | Clicks | Impressions | CTR\n${rows(2000)}`; + const CSV_TEXT = `Columns: Keyword | Clicks | Impressions\n${rows(2000)}`; + + test('keep at most MAX_TABLE_CHUNKS passages, far fewer than other documents', async () => { + expect(CHUNKING.MAX_TABLE_CHUNKS).toBeLessThan(CHUNKING.MAX_CHUNKS); + for (const text of [SHEET_TEXT, CSV_TEXT]) { + const { payloads } = await buildFileChunkPayloads({ + fileId: 'file:/exports/a.xlsx', + filePath: '/exports/a.xlsx', + text, + embedText + }); + expect(payloads).toHaveLength(CHUNKING.MAX_TABLE_CHUNKS); + expect(payloads[0].meta.text.startsWith(text.slice(0, 20))).toBe(true); + } + expect(embedText).toHaveBeenCalledTimes(2 * CHUNKING.MAX_TABLE_CHUNKS); + }); + + test('prose that only mentions a sheet keeps the full limit', () => { + expect(chunkOptionsFor(SHEET_TEXT).maxChunks).toBe(CHUNKING.MAX_TABLE_CHUNKS); + expect(chunkOptionsFor(` \n${CSV_TEXT}`).maxChunks).toBe(CHUNKING.MAX_TABLE_CHUNKS); + expect(chunkOptionsFor(`Notes on the export.\n${SHEET_TEXT}`).maxChunks).toBe( + CHUNKING.MAX_CHUNKS + ); + expect(chunkOptionsFor(LONG_TEXT).maxChunks).toBe(CHUNKING.MAX_CHUNKS); + expect(chunkOptionsFor('')).toMatchObject({ + chunkSize: CHUNKING.CHUNK_SIZE, + overlap: CHUNKING.OVERLAP, + maxChunks: CHUNKING.MAX_CHUNKS + }); + }); + }); + test('persists chunks, replacing the old ones only after the new set exists', async () => { const vectorDb = makeStore(); const order = []; From 450a1d319cb0166fb0585a71c01c4ff38f9b8851 Mon Sep 17 00:00:00 2001 From: Levy Tate <78818969+iLevyTate@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:51:20 -0500 Subject: [PATCH 09/10] perf(analysis): ask the model for shorter purpose, summary and reasoning Writing the answer is most of each file's model time. The schema now asks for at most 10, 25 and 15 words for purpose, summary and reasoning. On 30 exports the median answer went from 734 to 618 characters and the median model time from 13.9 s to 11.1 s, with every file in the same category. The changelog notes this and the spreadsheet passage cap. --- CHANGELOG.md | 9 +++++++++ src/shared/analysisSchema.js | 6 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e1cc4df..cfcb7ccd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,15 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - **Spreadsheets and CSVs send the AI the head of each sheet.** The AI saw a sample from the start, middle and end of the text, so most sheets lost their names and columns. It now gets each sheet's name, its columns and its first rows, in less than half the text. +- **Analysis is about a fifth faster.** Most of each file's time is the AI writing its answer, so + the purpose, summary and reasoning it writes are now shorter (at most 10, 25 and 15 words). On 30 + exports the median model time went from 13.9 s to 11.1 s per file, and every file went to the same + folder as before. +- **Spreadsheets and CSVs keep 12 search passages instead of up to 80.** Their rows are nearly + alike, so most exports reached the limit: a batch of 275 exports stored 14,473 passages, and every + save of the search index rewrote all of them. On 30 exports the passages went from 1,452 to 354 + and each save took about half as long. Search by meaning now covers about the first 10,000 + characters of a spreadsheet; other documents are unchanged. - **Suggestions made before you changed your Smart Folders are flagged.** Organize lists the files that still go where your old folders sent them, with one button to analyze them again, and Move asks before moving them. diff --git a/src/shared/analysisSchema.js b/src/shared/analysisSchema.js index f51784d2..de0d4bde 100644 --- a/src/shared/analysisSchema.js +++ b/src/shared/analysisSchema.js @@ -75,15 +75,15 @@ const ANALYSIS_SCHEMA_PROMPT = { category: 'The high-level category (must match one of the provided Smart Folder names if applicable).', project: 'The specific project, case, or subject matter (2-5 words).', - purpose: 'The primary purpose or intent of the document (1 sentence).', - summary: "A concise 1-2 sentence summary of the document's content.", + purpose: 'The primary purpose or intent of the document (a short phrase, at most 10 words).', + summary: "A concise 1 sentence summary of the document's content (at most 25 words).", keywords: ['keyword1', 'keyword2', 'keyword3', 'keyword4', 'keyword5'], keyEntities: ['person_or_org_1', 'person_or_org_2', 'location_or_product', 'project_or_event'], confidence: 'Number 0-100 indicating confidence in the extraction.', suggestedName: "A short, concise topic-based filename (max 3 words) using underscores (e.g., 'budget_report', 'sunset_beach', 'meeting_notes'). Do NOT include the extension.", reasoning: - 'Brief explanation (1 sentence) of why this category was chosen based on the content and folder description.' + 'Brief explanation (at most 15 words) of why this category was chosen based on the content and folder description.' }; /** From 64f5c2290c4ac24daed312fe44fb5f4fa9b69730 Mon Sep 17 00:00:00 2001 From: Levy Tate <78818969+iLevyTate@users.noreply.github.com> Date: Mon, 16 Nov 2026 13:02:59 -0500 Subject: [PATCH 10/10] fix(dashboard): a clear empty state, readable folders, and stats that load at startup - An empty board said "All caught up", "All clear" and "Nothing waiting". It now reads "Ready for files", explains the three steps, and the pipeline footer only appears when something is waiting or failed. - Stage counts use the sans tabular figures, and each icon chip sits on an opaque base so the connector rail no longer runs through it. The lone ring on the first chip is gone. - Smart folder chips use columns at least 11rem wide and skip the Uncategorized catch-all, so names read in full. - The knowledge card asked for stats once; during startup the handler answers with a bare envelope, which left the card on "unavailable" for the session. A bare answer is now asked again up to six times. A single year reads as "2026". - No row grows into the window's spare height. - The notification collapse button is drawn only when there are notifications; with none it floated over every screen. --- CHANGELOG.md | 8 ++ src/renderer/components/Toast.jsx | 79 +++++------ .../components/dashboard/DashboardView.jsx | 128 +++++++++++------- .../dashboard/KnowledgeStatsCard.jsx | 36 ++++- .../ToastContainer.emptyToggle.test.js | 24 ++++ .../dashboard/DashboardView.test.js | 52 ++++++- .../dashboard/KnowledgeStatsCard.test.js | 57 +++++++- 7 files changed, 276 insertions(+), 108 deletions(-) create mode 100644 test/components/ToastContainer.emptyToggle.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index cfcb7ccd..1c617f07 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,14 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixed +- **The Dashboard reads clearly before your first files.** It said "All caught up", "All clear" and + "Nothing waiting" on a board that had never had a file; it now says it is ready for files and + shows the three steps. Smart folder names are no longer cut short, and the Uncategorized catch-all + is no longer listed as one. The knowledge card no longer stays on "unavailable" when the Dashboard + opens during startup, and a single year reads as "2026" instead of "2026 to 2026". Cards keep + their natural height instead of stretching into empty space. +- **A stray close button no longer floats at the top right of every screen.** The notification + list's collapse button was drawn even with no notifications. - **Memory no longer climbs during a long analysis.** Each save of the search index leaked its compression buffers, so the app grew by about 35 MB for every file analyzed: from 5 GB to almost 15 GB over 275 files, which pushed the system into swap. diff --git a/src/renderer/components/Toast.jsx b/src/renderer/components/Toast.jsx index e7cf4f60..65b5254d 100644 --- a/src/renderer/components/Toast.jsx +++ b/src/renderer/components/Toast.jsx @@ -400,45 +400,48 @@ export function ToastContainer({ toasts = [], onRemoveToast, onActivateToast }) className="z-toast pointer-events-none" style={containerStyle()} > - {/* Compact toggle button */} -
-
-
+ )} {/* Toast stack */}
diff --git a/src/renderer/components/dashboard/DashboardView.jsx b/src/renderer/components/dashboard/DashboardView.jsx index dac52a1e..8479336d 100644 --- a/src/renderer/components/dashboard/DashboardView.jsx +++ b/src/renderer/components/dashboard/DashboardView.jsx @@ -45,7 +45,10 @@ import { Heading, Text, Caption } from '../ui/Typography'; import AnalysisProgress from '../discover/AnalysisProgress'; import OrganizeSummaryBanner from '../organize/OrganizeSummaryBanner'; import KnowledgeStatsCard from './KnowledgeStatsCard'; -import { countDestinationFolders } from '../../utils/smartFolderDestinations'; +import { + countDestinationFolders, + isUncategorizedBucket +} from '../../utils/smartFolderDestinations'; import { usePhaseJump } from '../../utils/phaseNavigation'; import { useNotification } from '../../contexts/NotificationContext'; import { useUndoRedo } from '../UndoRedoSystem'; @@ -239,7 +242,7 @@ AnimatedCount.propTypes = { value: PropTypes.number.isRequired }; -function PipelineStage({ icon: Icon, label, value, tone, isEntryHint, onClick }) { +function PipelineStage({ icon: Icon, label, value, tone, onClick }) { // Invert the emphasis: an empty stage recedes (the number drops to muted gray, // the chip to a calm neutral blue) while a live count lights up in the stage's // semantic tone. The big number is written with explicit utilities rather than @@ -261,15 +264,20 @@ function PipelineStage({ icon: Icon, label, value, tone, isEntryHint, onClick }) : '' }`.trim()} > + {/* The tinted chip sits on an opaque card-colored base: the chips are + translucent, and without the base the connector rail ran visibly + through every icon. */} -
@@ -285,7 +293,6 @@ PipelineStage.propTypes = { label: PropTypes.string.isRequired, value: PropTypes.number.isRequired, tone: PropTypes.oneOf(['warning', 'accent', 'success']), - isEntryHint: PropTypes.bool, onClick: PropTypes.func }; @@ -327,13 +334,20 @@ function PipelineCard({ const denom = stats.pending + stats.ready + organizedCount; const weightedProgress = stats.ready * 0.5 + organizedCount; const railPct = denom > 0 ? Math.round((weightedProgress / denom) * 100) : 0; + // The footer only speaks when it has news: files still waiting, failures, + // or files a run did not move. A calm board does not repeat the badge. + const hasFooter = inFlight > 0 || stats.failed > 0 || moveFailedCount > 0; return (
Organization pipeline - 0 ? 'info' : 'success'} size="sm"> - {waitingLabel} - + {/* An empty board has nothing to be clear of: the hero already says + no files are in yet. */} + {isEmpty ? null : ( + 0 ? 'info' : 'success'} size="sm"> + {waitingLabel} + + )}
{/* The three lifecycle stages sit on a single connector rail so the card reads as a left-to-right flow, not three disconnected boxes. The track @@ -363,7 +377,6 @@ function PipelineCard({ label="Awaiting analysis" value={stats.pending} tone="warning" - isEntryHint={isEmpty} onClick={onPendingStage} />
+ {isEmpty ? ( + + Files you add move through these three steps, left to right. + + ) : null}
-
- - {inFlight > 0 - ? `${inFlight} ${inFlight === 1 ? 'file' : 'files'} not organized yet` - : 'Nothing waiting'} - -
- {moveFailedCount > 0 && ( - - )} - {stats.failed > 0 ? ( - - ) : ( - 0 failed - )} + {hasFooter ? ( +
+ + {inFlight > 0 + ? `${inFlight} ${inFlight === 1 ? 'file' : 'files'} not organized yet` + : ''} + +
+ {moveFailedCount > 0 && ( + + )} + {stats.failed > 0 ? ( + + ) : null} +
-
+ ) : null} ); } @@ -432,8 +450,11 @@ PipelineCard.propTypes = { }; function SmartFoldersCard({ folders, onManage, className }) { - const visible = folders.slice(0, FOLDER_CHIP_LIMIT); - const overflow = folders.length - visible.length; + // The Uncategorized catch-all is where unmatched files wait, not a + // destination (the hero count leaves it out for the same reason). + const destinations = folders.filter((folder) => folder && !isUncategorizedBucket(folder)); + const visible = destinations.slice(0, FOLDER_CHIP_LIMIT); + const overflow = destinations.length - visible.length; return (
@@ -442,13 +463,9 @@ function SmartFoldersCard({ folders, onManage, className }) { Manage
- {/* Responsive tile grid instead of small pills: tiles grow to fill the - card width (2 cols, 3 on large) AND its height. auto-rows-fr makes the - rows share the card's stretched height (this card matches the taller - Quick-actions card via lg:items-stretch), so a handful of folders read - as substantial cards instead of short pills floated at the top. - min-h keeps a floor for the many-folders case. */} -
+ {/* Columns at least 11rem wide, so a name like "Client proposals" reads in + full instead of being cut short in a third of the card. */} +
{visible.map((folder) => (
0) { return { label: `${stats.ready} ready to organize`, variant: 'info', icon: ArrowRight }; } + if (stats.total === 0 && organizedCount === 0) { + // Not "no files yet": a returning user with an empty queue has + // analyzed files before, and the knowledge card below says so. + return { label: 'Ready for files', variant: 'info', icon: FilePlus }; + } return { label: 'All caught up', variant: 'success', icon: CheckCircle2 }; - }, [stats.failed, stats.pending, stats.ready, unmovedRun]); + }, [stats.failed, stats.pending, stats.ready, stats.total, organizedCount, unmovedRun]); const summary = useMemo(() => { // The Uncategorized catch-all main seeds on every profile is not a @@ -874,8 +896,10 @@ export default function DashboardView() { {/* Live-status zone — the two surfaces that answer "what's happening?". The pipeline holds the aggregate counts; the activity rail holds the individual items behind those counts, so they are paired and share a - height via items-stretch rather than living in separate columns. */} -
+ height via items-stretch rather than living in separate columns. + No row grows into the window's spare height: a stretched card + centered three small counts in a mostly empty panel. */} +
+
Quick actions
@@ -957,7 +981,7 @@ export default function DashboardView() { {/* Knowledge layer — a pride-surface summary of every file the local AI has processed. Sits at the foot of the board so it reads as a reward / closing thought after the pipeline and quick-action rows. */} - +
diff --git a/src/renderer/components/dashboard/KnowledgeStatsCard.jsx b/src/renderer/components/dashboard/KnowledgeStatsCard.jsx index 7b2e1d8d..87a4872d 100644 --- a/src/renderer/components/dashboard/KnowledgeStatsCard.jsx +++ b/src/renderer/components/dashboard/KnowledgeStatsCard.jsx @@ -16,6 +16,12 @@ function SectionLabel({ children }) { ); } +// How often a bare startup answer is asked again before the card says the +// stats are unavailable: about 9 s, longer than the history service takes to +// start on a slow disk. +const STATS_RETRY_ATTEMPTS = 6; +const STATS_RETRY_DELAY_MS = 1500; + // Note: AnimatedCount is module-private in DashboardView and not exported. // Plain numbers are used here intentionally — the card's data is fetched // async and the count-up animation would conflict with the loading transition. @@ -55,7 +61,7 @@ function isoToYear(iso) { /** * A single stat tile — icon chip + value + label — matching the visual * language of PipelineStage in DashboardView (icon in a rounded chip, - * large mono value, tiny label below). + * large tabular value, tiny label below). * * @param {{ icon: React.ElementType, value: string, label: string }} props */ @@ -69,7 +75,7 @@ function StatTile({ icon: Icon, value, label }) {