Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
54 changes: 46 additions & 8 deletions .github/workflows/mac-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,11 @@ env:
jobs:
resolve-release-target:
runs-on: ubuntu-latest
# The only output is the validated tag (v<major>.<minor>.<patch>, confirmed
# to exist). Every checkout below pins refs/tags/<tag>, so no job checks
# out a branch or any other caller-chosen ref.
outputs:
tag: ${{ steps.resolve.outputs.tag }}
ref: ${{ steps.resolve.outputs.ref }}
steps:
- name: Resolve and validate release target
id: resolve
Expand All @@ -63,7 +65,6 @@ jobs:
return;
}
core.setOutput("tag", tag);
core.setOutput("ref", `refs/tags/${tag}`);
return;
}

Expand All @@ -87,7 +88,6 @@ jobs:
}

core.setOutput("tag", requestedTag);
core.setOutput("ref", `refs/tags/${requestedTag}`);

build-macos:
needs: resolve-release-target
Expand All @@ -111,7 +111,7 @@ jobs:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ needs.resolve-release-target.outputs.ref }}
ref: refs/tags/${{ needs.resolve-release-target.outputs.tag }}
fetch-depth: 0
- name: Verify package version matches release tag
env:
Expand All @@ -123,7 +123,26 @@ jobs:
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version-file: '.nvmrc'
cache: npm
# Restore-only npm cache (next two steps): a release build reads the
# cache that ordinary CI writes on main but never saves one, so a
# release run cannot seed the default-branch cache scope. setup-node
# would save in its post step, so its own caching stays off.
package-manager-cache: false

- name: Locate npm cache directory
id: npm-cache-dir
run: echo "dir=$(npm config get cache)" >> "$GITHUB_OUTPUT"

- name: Restore npm cache (read-only)
# Same path and key format as setup-node's `cache: npm` (used by CI on
# main), so its entries match here. matrix.arch equals the runner's
# os.arch() on every leg of this job.
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ steps.npm-cache-dir.outputs.dir }}
key: node-cache-${{ runner.os }}-${{ matrix.arch }}-npm-${{ hashFiles('package-lock.json') }}
restore-keys: |
node-cache-${{ runner.os }}-${{ matrix.arch }}-npm-

- name: Verify macOS build assets
run: node scripts/verify-build-assets.js --platform=mac
Expand Down Expand Up @@ -383,7 +402,7 @@ jobs:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ needs.resolve-release-target.outputs.ref }}
ref: refs/tags/${{ needs.resolve-release-target.outputs.tag }}
fetch-depth: 0

- name: Free disk space on runner
Expand Down Expand Up @@ -423,7 +442,26 @@ jobs:
uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
node-version-file: '.nvmrc'
cache: npm
# Restore-only npm cache (next two steps): a release build reads the
# cache that ordinary CI writes on main but never saves one, so a
# release run cannot seed the default-branch cache scope. setup-node
# would save in its post step, so its own caching stays off.
package-manager-cache: false

- name: Locate npm cache directory
id: npm-cache-dir
run: echo "dir=$(npm config get cache)" >> "$GITHUB_OUTPUT"

- name: Restore npm cache (read-only)
# Same path and key format as setup-node's `cache: npm` (used by CI on
# main), so its entries match here. matrix.arch equals the runner's
# os.arch() on every leg of this job.
uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ${{ steps.npm-cache-dir.outputs.dir }}
key: node-cache-${{ runner.os }}-${{ matrix.arch }}-npm-${{ hashFiles('package-lock.json') }}
restore-keys: |
node-cache-${{ runner.os }}-${{ matrix.arch }}-npm-

- name: Install dependencies (with optional deps)
run: npm ci --include=optional
Expand Down Expand Up @@ -549,7 +587,7 @@ jobs:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ needs.resolve-release-target.outputs.ref }}
ref: refs/tags/${{ needs.resolve-release-target.outputs.tag }}
fetch-depth: 0

- name: Download macOS artifacts
Expand Down
2 changes: 1 addition & 1 deletion docs/MCP_SERVER.md
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ Search the indexed corpus by meaning, keyword, or both.

```json
{
"query": "string (1..1000)",
"query": "string (2..1000)",
"top_k": 10,
"mode": "hybrid",
"smart_folder_id": "optional-uuid"
Expand Down
10 changes: 6 additions & 4 deletions scripts/compose-walkthrough-media.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,6 @@ const ROOT = path.resolve(__dirname, '..');
const OUT_DIR = path.join(ROOT, 'docs', 'walkthrough-assets');
const RAW_DIR = path.join(OUT_DIR, 'tour-raw');
const SHORT_DIR = path.join(OUT_DIR, 'shorts');
const VIDEO = path.join(RAW_DIR, 'app.webm');
const ACTIONS = path.join(RAW_DIR, 'actions.json');
const LOGO = path.join(ROOT, 'assets', 'stratosort-logo.png');

const FPS = 25;
Expand Down Expand Up @@ -470,9 +468,13 @@ function probeVideo(file) {
};
}

/** Paths inside a filter graph: forward slashes, and the drive colon escaped. */
/**
* Paths inside a filter graph: forward slashes, and the drive colon escaped.
* One pass handles both characters, so every backslash in the input becomes
* '/' and the only backslashes in the result are the ones escaping ':'.
*/
function ffPath(file) {
return file.replace(/\\/g, '/').replace(/:/g, '\\:');
return file.replace(/[\\:]/g, (ch) => (ch === ':' ? '\\:' : '/'));
}

const even = (n) => Math.max(2, Math.floor(n / 2) * 2);
Expand Down
4 changes: 3 additions & 1 deletion scripts/verify-splash-csp-hash.js
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ function main() {
}
const html = fs.readFileSync(indexPath, 'utf8');
const inlineBodies = [];
const re = /<script([^>]*)>([\s\S]*?)<\/script>/gi;
// The end tag may carry whitespace or junk before '>' ("</script >"); the
// browser still ends the script there, so the hashed body must end there too.
const re = /<script([^>]*)>([\s\S]*?)<\/script\b[^>]*>/gi;
let m;
while ((m = re.exec(html)) !== null) {
const attrs = m[1];
Expand Down
22 changes: 20 additions & 2 deletions src/main/analysis/contentSelector.js
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,25 @@ const MAX_HEADINGS = 25;
// Internal helpers
// ---------------------------------------------------------------------------

/**
* Remove HTML tags, repeating until nothing changes so no tag-like text can
* survive (for example "<scr<script>ipt>" never leaves "<script" behind).
* For this pattern one pass is usually already stable, so the loop is cheap;
* it is the form static analysis recognises as a complete sanitizer.
*
* @param {string} text
* @returns {string}
*/
function stripHtmlTags(text) {
let result = text;
let previous;
do {
previous = result;
result = result.replace(/<[^>]+>/g, '');
} while (result !== previous);
return result;
}

/**
* Lightweight text cleaning that preserves paragraph structure.
*
Expand Down Expand Up @@ -165,8 +184,7 @@ function extractDocumentOutline(text, maxLength) {
let heading = (match[1] || match[0]).trim();

// Strip residual HTML tags and Markdown #
heading = heading
.replace(/<[^>]+>/g, '')
heading = stripHtmlTags(heading)
.replace(/^#+\s*/, '')
.trim();

Expand Down
6 changes: 4 additions & 2 deletions src/main/analysis/documentAnalysis.js
Original file line number Diff line number Diff line change
Expand Up @@ -1016,7 +1016,7 @@ async function analyzeDocumentStages(filePath, smartFolders = [], options = {},
? extractedText.slice(0, 8192) + '\0' + extractedText.length
: extractedText;
const contentHash = crypto
.createHash('md5')
.createHash('sha256')
.update(hashInput)
.update(safeModelName)
.digest('hex');
Expand Down Expand Up @@ -1292,7 +1292,9 @@ async function extractCadOrDesignText(filePath, fileExtension) {
return cad.extractTextFromEps(filePath);
case '.ai':
return cad.extractTextFromAi(filePath, {
extractTextFromPdf: (/** @type {string} */ p) => extractTextFromPdf(p, path.basename(p))
// No OCR follows on this route, so nothing would release a pinned buffer.
extractTextFromPdf: (/** @type {string} */ p) =>
extractTextFromPdf(p, path.basename(p), { pinForOcr: false })
});
default:
return '';
Expand Down
25 changes: 16 additions & 9 deletions src/main/analysis/documentExtractors.js
Original file line number Diff line number Diff line change
Expand Up @@ -244,9 +244,10 @@ const REGEX_PATTERNS = {
rtfBraces: /[{}]/g,
rtfControlWords: /\\[a-zA-Z]+-?\d* ?/g,

// HTML patterns
scriptTags: /<script[\s\S]*?<\/script>/gi,
styleTags: /<style[\s\S]*?<\/style>/gi,
// HTML patterns. End tags may carry whitespace or junk before '>'
// ("</script >", "</SCRIPT foo>"); browsers still close the element there.
scriptTags: /<script[\s\S]*?<\/script\b[^>]*>/gi,
styleTags: /<style[\s\S]*?<\/style\b[^>]*>/gi,
htmlTags: /<[^>]+>/g,

// Whitespace patterns
Expand Down Expand Up @@ -1100,12 +1101,16 @@ function csvRowCells(row) {
* head of a CSV over the text size limit, like the other text formats.
*/
async function extractTextFromCsv(filePath, options = {}) {
// Treat CSV as structured text; enforce file size to avoid memory blowups
const fileSize = await checkFileSize(filePath, filePath);
const maxBytes =
options && Number.isFinite(options.maxBytes) && Number(options.maxBytes) > 0
? Math.floor(Number(options.maxBytes))
: null;
// Treat CSV as structured text; enforce file size to avoid memory blowups.
// A capped read only loads maxBytes, so the document size limit does not
// apply (a finite override, since checkFileSize reads Infinity as "unset").
const fileSize = maxBytes
? await checkFileSize(filePath, filePath, Number.MAX_SAFE_INTEGER)
: await checkFileSize(filePath, filePath);
let raw = await readTextFileWithEncoding(filePath, { maxBytes });
if (maxBytes && fileSize > maxBytes) {
// The cap lands mid-row; a half row would parse as a short or broken
Expand Down Expand Up @@ -1231,10 +1236,12 @@ function isEncryptedPdfError(/** @type {any} */ err) {
/**
* @param {any} filePath
* @param {any} fileName
* @param {{ signal?: AbortSignal }} [options]
* @param {{ signal?: AbortSignal, pinForOcr?: boolean }} [options] pinForOcr:
* false for callers that never run ocrPdfIfNeeded (the .ai reader), so a
* text-less file is not left pinned in the buffer cache until the TTL sweep.
*/
async function extractTextFromPdf(filePath, fileName, options = {}) {
const { signal } = options;
const { signal, pinForOcr = true } = options;
await checkFileSize(filePath, fileName);
throwIfAborted(signal);

Expand Down Expand Up @@ -1302,7 +1309,7 @@ async function extractTextFromPdf(filePath, fileName, options = {}) {
// ocrPdfIfNeeded, which releases them in its finally. pdf.js may have
// transferred the ArrayBuffer to its worker, leaving a detached
// zero-length husk — never cache that.
if (rawBuffer.length > 0) _pdfBufferCache.set(filePath, rawBuffer);
if (pinForOcr && rawBuffer.length > 0) _pdfBufferCache.set(filePath, rawBuffer);
if (thinTextLayer) {
logger.info('[PDF] Text layer too thin to analyze; treating as a scan', {
fileName,
Expand Down Expand Up @@ -3493,7 +3500,7 @@ function chunkTextForAnalysis(
* cannot outlive its consumer and pin its buffer in `_pdfBufferCache`.
* @param {string} filePath - Path to PDF file
* @param {string} fileName - Name of file for error messages
* @param {{ signal?: AbortSignal }} [options]
* @param {{ signal?: AbortSignal, pinForOcr?: boolean }} [options]
* @returns {Promise<string>} Extracted text
*/
async function extractTextFromPdfWithTimeout(filePath, fileName, options = {}) {
Expand Down
12 changes: 7 additions & 5 deletions src/main/analysis/documentLlm.js
Original file line number Diff line number Diff line change
Expand Up @@ -674,11 +674,13 @@ ${truncated}
return inference();
}

return globalDeduplicator.deduplicate(
dedupeKey,
inference,
{ type: 'document', fileName: originalFileName } // Metadata for debugging cache hits
);
return globalDeduplicator.deduplicate(dedupeKey, inference, {
type: 'document',
fileName: originalFileName,
// Lets a caller that joined this call re-run it when the
// starter's abort (not its own) ends the shared inference.
signal: abortController.signal
});
},
AppConfig.ai.textAnalysis.timeout,
`Document analysis for ${originalFileName}`,
Expand Down
Loading
Loading