From 5b52a7e303c80100581b47950fa3787f46ccd71f Mon Sep 17 00:00:00 2001 From: rakshith48 <33714643+rakshith48@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:59:40 +0000 Subject: [PATCH 1/2] fix: follow the API's main-content default and send crawl --delay in seconds Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- README.md | 50 +++++++++++---------- src/__tests__/commands/batch-scrape.test.ts | 17 ++++++- src/__tests__/commands/crawl.test.ts | 13 +++++- src/__tests__/utils/options.test.ts | 25 +++++++++++ src/commands/batch-scrape.ts | 9 ++-- src/commands/crawl.ts | 2 +- src/commands/scrape.ts | 8 ++-- src/index.ts | 50 ++++++++++++++++++--- src/utils/options.ts | 16 ++++++- 9 files changed, 143 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index 339326b950..4a5443f5c9 100644 --- a/README.md +++ b/README.md @@ -263,29 +263,30 @@ firecrawl scrape https://firecrawl.dev https://firecrawl.dev/blog https://docs.f #### Scrape Options -| Option | Description | -| -------------------------- | ------------------------------------------------------- | -| `-f, --format ` | Output format(s), comma-separated | -| `-H, --html` | Shortcut for `--format html` | -| `-S, --summary` | Shortcut for `--format summary` | -| `--only-main-content` | Extract only main content (removes navs, footers, etc.) | -| `--wait-for ` | Wait time before scraping (for JS-rendered content) | -| `--screenshot` | Take a screenshot | -| `--full-page-screenshot` | Take a full page screenshot | -| `--include-tags ` | Only include specific HTML tags | -| `--exclude-tags ` | Exclude specific HTML tags | -| `--max-age ` | Maximum age of cached content in milliseconds | -| `--lockdown` | Enable lockdown mode for the scrape | -| `--redact-pii` | Redact personally identifiable information from output | -| `--schema ` | JSON schema for structured extraction | -| `--schema-file ` | Path to JSON schema file for structured extraction | -| `--actions ` | JSON actions array to run during scrape | -| `--actions-file ` | Path to JSON actions file | -| `--proxy ` | Proxy mode for scraping (for example, `auto`, `basic`) | -| `-o, --output ` | Save output to file | -| `--json` | Output as JSON format | -| `--pretty` | Pretty print JSON output | -| `--timing` | Show request timing info | +| Option | Description | +| -------------------------- | ------------------------------------------------------ | +| `-f, --format ` | Output format(s), comma-separated | +| `-H, --html` | Shortcut for `--format html` | +| `-S, --summary` | Shortcut for `--format summary` | +| `--only-main-content` | Extract only main content (the default) | +| `--full-page` | Keep the whole page, including navs and footers | +| `--wait-for ` | Wait time before scraping (for JS-rendered content) | +| `--screenshot` | Take a screenshot | +| `--full-page-screenshot` | Take a full page screenshot | +| `--include-tags ` | Only include specific HTML tags | +| `--exclude-tags ` | Exclude specific HTML tags | +| `--max-age ` | Maximum age of cached content in milliseconds | +| `--lockdown` | Enable lockdown mode for the scrape | +| `--redact-pii` | Redact personally identifiable information from output | +| `--schema ` | JSON schema for structured extraction | +| `--schema-file ` | Path to JSON schema file for structured extraction | +| `--actions ` | JSON actions array to run during scrape | +| `--actions-file ` | Path to JSON actions file | +| `--proxy ` | Proxy mode for scraping (for example, `auto`, `basic`) | +| `-o, --output ` | Save output to file | +| `--json` | Output as JSON format | +| `--pretty` | Pretty print JSON output | +| `--timing` | Show request timing info | #### Available Formats @@ -743,7 +744,8 @@ firecrawl batch-scrape --cancel | ------------------------------ | ----------------------------------------------- | | `--urls-file ` | File with one URL per line | | `-f, --format ` | Output format(s), comma-separated | -| `--only-main-content` | Extract only main content | +| `--only-main-content` | Extract only main content (the default) | +| `--full-page` | Keep the whole page, including navs and footers | | `--wait-for ` | Wait time before scraping each page | | `--max-age ` | Maximum age of cached content in milliseconds | | `--schema ` | JSON schema for extraction (adds `json` format) | diff --git a/src/__tests__/commands/batch-scrape.test.ts b/src/__tests__/commands/batch-scrape.test.ts index 88281c8227..b73bf3b274 100644 --- a/src/__tests__/commands/batch-scrape.test.ts +++ b/src/__tests__/commands/batch-scrape.test.ts @@ -87,6 +87,19 @@ describe('executeBatchScrape', () => { vi.clearAllMocks(); }); + describe('onlyMainContent', () => { + it('sends onlyMainContent: false for --full-page', async () => { + mockClient.startBatchScrape.mockResolvedValue({ id: JOB_ID, url: '' }); + + await executeBatchScrape({ urls: URLS, onlyMainContent: false }); + + expect(mockClient.startBatchScrape).toHaveBeenCalledWith(URLS, { + options: { onlyMainContent: false }, + integration: 'cli', + }); + }); + }); + describe('start (no wait)', () => { it('starts a batch job and returns the job ID', async () => { mockClient.startBatchScrape.mockResolvedValue({ @@ -97,7 +110,7 @@ describe('executeBatchScrape', () => { const result = await executeBatchScrape({ urls: URLS }); expect(mockClient.startBatchScrape).toHaveBeenCalledWith(URLS, { - options: { onlyMainContent: false }, + options: {}, integration: 'cli', }); expect(mockClient.batchScrape).not.toHaveBeenCalled(); @@ -317,7 +330,7 @@ describe('executeBatchScrape', () => { }); expect(mockClient.batchScrape).toHaveBeenCalledWith(URLS, { - options: { onlyMainContent: false }, + options: {}, integration: 'cli', pollInterval: 2, timeout: 60, diff --git a/src/__tests__/commands/crawl.test.ts b/src/__tests__/commands/crawl.test.ts index 2efdd73c52..ec82b3613e 100644 --- a/src/__tests__/commands/crawl.test.ts +++ b/src/__tests__/commands/crawl.test.ts @@ -243,11 +243,22 @@ describe('executeCrawl', () => { crawlEntireDomain: false, allowExternalLinks: false, allowSubdomains: true, - delay: 1000, + delay: 1, maxConcurrency: 5, } ); }); + + it('should send --delay to the API in seconds', async () => { + mockClient.startCrawl.mockResolvedValue({ id: 'job-1', url: '' }); + + await executeCrawl({ urlOrJobId: 'https://example.com', delay: 250 }); + + expect(mockClient.startCrawl).toHaveBeenCalledWith( + 'https://example.com', + expect.objectContaining({ delay: 0.25 }) + ); + }); }); describe('Check crawl status', () => { diff --git a/src/__tests__/utils/options.test.ts b/src/__tests__/utils/options.test.ts index c9362ffab5..a887b976a5 100644 --- a/src/__tests__/utils/options.test.ts +++ b/src/__tests__/utils/options.test.ts @@ -209,6 +209,31 @@ describe('Option Parsing Utilities', () => { expect(result.onlyMainContent).toBe(true); }); + it('should leave onlyMainContent unset so the API default applies', () => { + const result = parseScrapeOptions({ url: 'https://example.com' }); + + expect(result.onlyMainContent).toBeUndefined(); + }); + + it('should send onlyMainContent: false for --full-page', () => { + const result = parseScrapeOptions({ + url: 'https://example.com', + fullPage: true, + }); + + expect(result.onlyMainContent).toBe(false); + }); + + it('should reject --only-main-content together with --full-page', () => { + expect(() => + parseScrapeOptions({ + url: 'https://example.com', + onlyMainContent: true, + fullPage: true, + }) + ).toThrow('Use either --only-main-content or --full-page, not both'); + }); + it('should parse waitFor option', () => { const options = { url: 'https://example.com', diff --git a/src/commands/batch-scrape.ts b/src/commands/batch-scrape.ts index e627fb371d..6eaa151290 100644 --- a/src/commands/batch-scrape.ts +++ b/src/commands/batch-scrape.ts @@ -49,10 +49,7 @@ export function buildBatchScrapeOptions( ): Record { const { schema, prompt } = options; const wantsJson = schema !== undefined || prompt !== undefined; - const scrapeOptions: Record = { - onlyMainContent: false, - ...options.scrapeOptions, - }; + const scrapeOptions: Record = { ...options.scrapeOptions }; const baseFormats: unknown[] | undefined = options.format ? parseFormats(options.format) @@ -90,8 +87,8 @@ export function buildBatchScrapeOptions( } } - if (options.onlyMainContent) { - scrapeOptions.onlyMainContent = true; + if (options.onlyMainContent !== undefined) { + scrapeOptions.onlyMainContent = options.onlyMainContent; } if (options.waitFor !== undefined) { scrapeOptions.waitFor = options.waitFor; diff --git a/src/commands/crawl.ts b/src/commands/crawl.ts index d9107e3422..bdc9a37a23 100644 --- a/src/commands/crawl.ts +++ b/src/commands/crawl.ts @@ -131,7 +131,7 @@ export async function executeCrawl( crawlOptions.allowSubdomains = options.allowSubdomains; } if (options.delay !== undefined) { - crawlOptions.delay = options.delay; + crawlOptions.delay = options.delay / 1000; } if (options.maxConcurrency !== undefined) { crawlOptions.maxConcurrency = options.maxConcurrency; diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts index 96fd28078d..39b28d9cbc 100644 --- a/src/commands/scrape.ts +++ b/src/commands/scrape.ts @@ -439,11 +439,9 @@ async function runWizard( // 2. Main content only const mainContent = await confirm({ message: 'Only main content?', - default: false, + default: true, }); - if (mainContent) { - options = { ...options, onlyMainContent: true }; - } + options = { ...options, onlyMainContent: mainContent }; // 3. Filter by paths (spacebar multi-select from discovered paths) const topPaths = getTopPaths(urls); @@ -520,7 +518,7 @@ export async function handleAllScrapeCommand( options.formats[0] !== 'markdown') || options.screenshot || options.fullPageScreenshot || - options.onlyMainContent; + options.onlyMainContent !== undefined; if (!hasExplicitFlags && process.stdin.isTTY) { const result = await runWizard(urls, options, allOptions); diff --git a/src/index.ts b/src/index.ts index 76947d5e75..45ad9ec15c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -82,7 +82,11 @@ import { handleStatusCommand } from './commands/status'; import { handleDoctorCommand } from './commands/doctor'; import { isUrl, normalizeUrl } from './utils/url'; import { resolveScrapeTarget } from './utils/scrape-target'; -import { parseMaxPages, parseScrapeOptions } from './utils/options'; +import { + onlyMainContentFlag, + parseMaxPages, + parseScrapeOptions, +} from './utils/options'; import { isJobId } from './utils/job'; import { ensureAuthenticated, printBanner } from './utils/auth'; import { maybeShowUpdateNotice } from './utils/update-notice'; @@ -406,7 +410,14 @@ function createScrapeCommand(): Command { 'Maximum PDF pages to parse (1-10000). PDFs cost 1 credit per parsed page; extra options may cost more.', parseMaxPages ) - .option('--only-main-content', 'Include only main content', false) + .option( + '--only-main-content', + 'Include only the main content (the API default)' + ) + .option( + '--full-page', + 'Keep the whole page, including navigation, headers and footers' + ) .option( '--wait-for ', 'Wait time before scraping in milliseconds', @@ -585,7 +596,14 @@ function createDownloadCommand(): Command { '-S, --summary', 'Download as summary (shortcut for --format summary)' ) - .option('--only-main-content', 'Include only main content', false) + .option( + '--only-main-content', + 'Include only the main content (the API default)' + ) + .option( + '--full-page', + 'Keep the whole page, including navigation, headers and footers' + ) .option( '--wait-for ', 'Wait time before scraping in milliseconds', @@ -704,7 +722,11 @@ function createCrawlCommand(): Command { .option('--crawl-entire-domain', 'Crawl entire domain', false) .option('--allow-external-links', 'Allow external links', false) .option('--allow-subdomains', 'Allow subdomains', false) - .option('--delay ', 'Delay between requests in milliseconds', parseInt) + .option( + '--delay ', + 'Delay between requests in milliseconds (sent to the API in seconds)', + parseInt + ) .option( '--max-concurrency ', 'Maximum concurrent requests', @@ -816,7 +838,14 @@ function createBatchScrapeCommand(): Command { '-f, --format ', 'Output format(s), comma-separated (default: markdown). Available: markdown, html, rawHtml, links, images, screenshot, summary, changeTracking, json, attributes, branding' ) - .option('--only-main-content', 'Include only main content', false) + .option( + '--only-main-content', + 'Include only the main content (the API default)' + ) + .option( + '--full-page', + 'Keep the whole page, including navigation, headers and footers' + ) .option( '--wait-for ', 'Wait time before scraping in milliseconds', @@ -925,7 +954,7 @@ function createBatchScrapeCommand(): Command { apiKey: options.apiKey, apiUrl: options.apiUrl, format: options.format, - onlyMainContent: options.onlyMainContent, + onlyMainContent: onlyMainContentFlag(options), waitFor: options.waitFor, maxAge: options.maxAge, schema, @@ -1026,7 +1055,14 @@ function createParseCommand(): Command { '-f, --format ', 'Output format(s). Multiple formats can be specified with commas (e.g., "markdown,links"). Available: markdown, html, rawHtml, links, images, summary, json, attributes. Single format outputs raw content; multiple formats output JSON.' ) - .option('--only-main-content', 'Include only main content', false) + .option( + '--only-main-content', + 'Include only the main content (the API default)' + ) + .option( + '--full-page', + 'Keep the whole page, including navigation, headers and footers' + ) .option('-S, --summary', 'Output summary (shortcut for --format summary)') .option('--include-tags ', 'Comma-separated list of tags to include') .option('--exclude-tags ', 'Comma-separated list of tags to exclude') diff --git a/src/utils/options.ts b/src/utils/options.ts index 2bf2f70b57..2d4a663a40 100644 --- a/src/utils/options.ts +++ b/src/utils/options.ts @@ -83,6 +83,20 @@ export function parseMaxPages(value: string): number { /** * Convert commander options to ScrapeOptions */ +/** + * onlyMainContent for a scrape: true with --only-main-content, false with + * --full-page, and unset otherwise so the API default (main content) applies. + */ +export function onlyMainContentFlag(options: { + onlyMainContent?: boolean; + fullPage?: boolean; +}): boolean | undefined { + if (options.onlyMainContent && options.fullPage) { + throw new Error('Use either --only-main-content or --full-page, not both'); + } + return options.fullPage ? false : options.onlyMainContent; +} + export function parseScrapeOptions(options: any): ScrapeOptions { // Parse formats from comma-separated string let formats: ScrapeFormat[] | undefined; @@ -117,7 +131,7 @@ export function parseScrapeOptions(options: any): ScrapeOptions { domainTools: options.domainTools, toolDetail: options.toolDetail, formats, - onlyMainContent: options.onlyMainContent, + onlyMainContent: onlyMainContentFlag(options), waitFor: options.waitFor, screenshot: options.screenshot, fullPageScreenshot: options.fullPageScreenshot, From cd6903bae9a0ad1021abc38ccdd3451b3bb529c0 Mon Sep 17 00:00:00 2001 From: rakshith48 <33714643+rakshith48@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:04:05 +0000 Subject: [PATCH 2/2] fix(scrape): reject --only-main-content with --full-page for provider calls too Co-authored-by: capy-ai[bot] <230910855+capy-ai[bot]@users.noreply.github.com> --- src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/index.ts b/src/index.ts index 45ad9ec15c..47d87eafac 100644 --- a/src/index.ts +++ b/src/index.ts @@ -479,6 +479,7 @@ function createScrapeCommand(): Command { .option('--proxy ', 'Proxy mode for scraping (e.g., auto, basic)') .action(async (positionalArgs, options) => { + onlyMainContentFlag(options); const target = resolveScrapeTarget(positionalArgs ?? [], options); if (target.kind === 'alexandria') { await handleAlexandria(target.calls, options);