fix: follow the API's main-content default and send crawl --delay in seconds

README.md

@@ -263,29 +263,30 @@ firecrawl scrape https://firecrawl.dev https://firecrawl.dev/blog https://docs.f
 
 #### Scrape Options
 
-| Option                     | Description                                             |
-| -------------------------- | ------------------------------------------------------- |
-| `-f, --format <formats>`   | Output format(s), comma-separated                       |
-| `-H, --html`               | Shortcut for `--format html`                            |
-| `-S, --summary`            | Shortcut for `--format summary`                         |
-| `--only-main-content`      | Extract only main content (removes navs, footers, etc.) |
-| `--wait-for <ms>`          | Wait time before scraping (for JS-rendered content)     |
-| `--screenshot`             | Take a screenshot                                       |
-| `--full-page-screenshot`   | Take a full page screenshot                             |
-| `--include-tags <tags>`    | Only include specific HTML tags                         |
-| `--exclude-tags <tags>`    | Exclude specific HTML tags                              |
-| `--max-age <milliseconds>` | Maximum age of cached content in milliseconds           |
-| `--lockdown`               | Enable lockdown mode for the scrape                     |
-| `--redact-pii`             | Redact personally identifiable information from output  |
-| `--schema <json>`          | JSON schema for structured extraction                   |
-| `--schema-file <path>`     | Path to JSON schema file for structured extraction      |
-| `--actions <json>`         | JSON actions array to run during scrape                 |
-| `--actions-file <path>`    | Path to JSON actions file                               |
-| `--proxy <proxy>`          | Proxy mode for scraping (for example, `auto`, `basic`)  |
-| `-o, --output <path>`      | Save output to file                                     |
-| `--json`                   | Output as JSON format                                   |
-| `--pretty`                 | Pretty print JSON output                                |
-| `--timing`                 | Show request timing info                                |
+| Option                     | Description                                            |
+| -------------------------- | ------------------------------------------------------ |
+| `-f, --format <formats>`   | Output format(s), comma-separated                      |
+| `-H, --html`               | Shortcut for `--format html`                           |
+| `-S, --summary`            | Shortcut for `--format summary`                        |
+| `--only-main-content`      | Extract only main content (the default)                |
+| `--full-page`              | Keep the whole page, including navs and footers        |
+| `--wait-for <ms>`          | Wait time before scraping (for JS-rendered content)    |
+| `--screenshot`             | Take a screenshot                                      |
+| `--full-page-screenshot`   | Take a full page screenshot                            |
+| `--include-tags <tags>`    | Only include specific HTML tags                        |
+| `--exclude-tags <tags>`    | Exclude specific HTML tags                             |
+| `--max-age <milliseconds>` | Maximum age of cached content in milliseconds          |
+| `--lockdown`               | Enable lockdown mode for the scrape                    |
+| `--redact-pii`             | Redact personally identifiable information from output |
+| `--schema <json>`          | JSON schema for structured extraction                  |
+| `--schema-file <path>`     | Path to JSON schema file for structured extraction     |
+| `--actions <json>`         | JSON actions array to run during scrape                |
+| `--actions-file <path>`    | Path to JSON actions file                              |
+| `--proxy <proxy>`          | Proxy mode for scraping (for example, `auto`, `basic`) |
+| `-o, --output <path>`      | Save output to file                                    |
+| `--json`                   | Output as JSON format                                  |
+| `--pretty`                 | Pretty print JSON output                               |
+| `--timing`                 | Show request timing info                               |
 
 #### Available Formats
 
@@ -743,7 +744,8 @@ firecrawl batch-scrape <job-id> --cancel
 | ------------------------------ | ----------------------------------------------- |
 | `--urls-file <path>`           | File with one URL per line                      |
 | `-f, --format <formats>`       | Output format(s), comma-separated               |
-| `--only-main-content`          | Extract only main content                       |
+| `--only-main-content`          | Extract only main content (the default)         |
+| `--full-page`                  | Keep the whole page, including navs and footers |
 | `--wait-for <ms>`              | Wait time before scraping each page             |
 | `--max-age <milliseconds>`     | Maximum age of cached content in milliseconds   |
 | `--schema <json>`              | JSON schema for extraction (adds `json` format) |

src/__tests__/commands/batch-scrape.test.ts

@@ -87,6 +87,19 @@ describe('executeBatchScrape', () => {
     vi.clearAllMocks();
   });
 
+  describe('onlyMainContent', () => {
+    it('sends onlyMainContent: false for --full-page', async () => {
+      mockClient.startBatchScrape.mockResolvedValue({ id: JOB_ID, url: '' });
+
+      await executeBatchScrape({ urls: URLS, onlyMainContent: false });
+
+      expect(mockClient.startBatchScrape).toHaveBeenCalledWith(URLS, {
+        options: { onlyMainContent: false },
+        integration: 'cli',
+      });
+    });
+  });
+
   describe('start (no wait)', () => {
     it('starts a batch job and returns the job ID', async () => {
       mockClient.startBatchScrape.mockResolvedValue({
@@ -97,7 +110,7 @@ describe('executeBatchScrape', () => {
       const result = await executeBatchScrape({ urls: URLS });
 
       expect(mockClient.startBatchScrape).toHaveBeenCalledWith(URLS, {
-        options: { onlyMainContent: false },
+        options: {},
         integration: 'cli',
       });
       expect(mockClient.batchScrape).not.toHaveBeenCalled();
@@ -317,7 +330,7 @@ describe('executeBatchScrape', () => {
       });
 
       expect(mockClient.batchScrape).toHaveBeenCalledWith(URLS, {
-        options: { onlyMainContent: false },
+        options: {},
         integration: 'cli',
         pollInterval: 2,
         timeout: 60,

src/__tests__/commands/crawl.test.ts

@@ -243,11 +243,22 @@ describe('executeCrawl', () => {
           crawlEntireDomain: false,
           allowExternalLinks: false,
           allowSubdomains: true,
-          delay: 1000,
+          delay: 1,
           maxConcurrency: 5,
         }
       );
     });
+
+    it('should send --delay to the API in seconds', async () => {
+      mockClient.startCrawl.mockResolvedValue({ id: 'job-1', url: '' });
+
+      await executeCrawl({ urlOrJobId: 'https://example.com', delay: 250 });
+
+      expect(mockClient.startCrawl).toHaveBeenCalledWith(
+        'https://example.com',
+        expect.objectContaining({ delay: 0.25 })
+      );
+    });
   });
 
   describe('Check crawl status', () => {

src/__tests__/utils/options.test.ts

@@ -209,6 +209,31 @@ describe('Option Parsing Utilities', () => {
       expect(result.onlyMainContent).toBe(true);
     });
 
+    it('should leave onlyMainContent unset so the API default applies', () => {
+      const result = parseScrapeOptions({ url: 'https://example.com' });
+
+      expect(result.onlyMainContent).toBeUndefined();
+    });
+
+    it('should send onlyMainContent: false for --full-page', () => {
+      const result = parseScrapeOptions({
+        url: 'https://example.com',
+        fullPage: true,
+      });
+
+      expect(result.onlyMainContent).toBe(false);
+    });
+
+    it('should reject --only-main-content together with --full-page', () => {
+      expect(() =>
+        parseScrapeOptions({
+          url: 'https://example.com',
+          onlyMainContent: true,
+          fullPage: true,
+        })
+      ).toThrow('Use either --only-main-content or --full-page, not both');
+    });
+
     it('should parse waitFor option', () => {
       const options = {
         url: 'https://example.com',

src/commands/batch-scrape.ts

@@ -49,10 +49,7 @@ export function buildBatchScrapeOptions(
 ): Record<string, unknown> {
   const { schema, prompt } = options;
   const wantsJson = schema !== undefined || prompt !== undefined;
-  const scrapeOptions: Record<string, unknown> = {
-    onlyMainContent: false,
-    ...options.scrapeOptions,
-  };
+  const scrapeOptions: Record<string, unknown> = { ...options.scrapeOptions };
 
   const baseFormats: unknown[] | undefined = options.format
     ? parseFormats(options.format)
@@ -90,8 +87,8 @@ export function buildBatchScrapeOptions(
     }
   }
 
-  if (options.onlyMainContent) {
-    scrapeOptions.onlyMainContent = true;
+  if (options.onlyMainContent !== undefined) {
+    scrapeOptions.onlyMainContent = options.onlyMainContent;
   }
   if (options.waitFor !== undefined) {
     scrapeOptions.waitFor = options.waitFor;

src/commands/crawl.ts

@@ -131,7 +131,7 @@ export async function executeCrawl(
       crawlOptions.allowSubdomains = options.allowSubdomains;
     }
     if (options.delay !== undefined) {
-      crawlOptions.delay = options.delay;
+      crawlOptions.delay = options.delay / 1000;
     }
     if (options.maxConcurrency !== undefined) {
       crawlOptions.maxConcurrency = options.maxConcurrency;

src/commands/scrape.ts

@@ -439,11 +439,9 @@ async function runWizard(
   // 2. Main content only
   const mainContent = await confirm({
     message: 'Only main content?',
-    default: false,
+    default: true,
   });
-  if (mainContent) {
-    options = { ...options, onlyMainContent: true };
-  }
+  options = { ...options, onlyMainContent: mainContent };
 
   // 3. Filter by paths (spacebar multi-select from discovered paths)
   const topPaths = getTopPaths(urls);
@@ -520,7 +518,7 @@ export async function handleAllScrapeCommand(
       options.formats[0] !== 'markdown') ||
     options.screenshot ||
     options.fullPageScreenshot ||
-    options.onlyMainContent;
+    options.onlyMainContent !== undefined;
 
   if (!hasExplicitFlags && process.stdin.isTTY) {
     const result = await runWizard(urls, options, allOptions);

src/index.ts

@@ -82,7 +82,11 @@ import { handleStatusCommand } from './commands/status';
 import { handleDoctorCommand } from './commands/doctor';
 import { isUrl, normalizeUrl } from './utils/url';
 import { resolveScrapeTarget } from './utils/scrape-target';
-import { parseMaxPages, parseScrapeOptions } from './utils/options';
+import {
+  onlyMainContentFlag,
+  parseMaxPages,
+  parseScrapeOptions,
+} from './utils/options';
 import { isJobId } from './utils/job';
 import { ensureAuthenticated, printBanner } from './utils/auth';
 import { maybeShowUpdateNotice } from './utils/update-notice';
@@ -406,7 +410,14 @@ function createScrapeCommand(): Command {
       'Maximum PDF pages to parse (1-10000). PDFs cost 1 credit per parsed page; extra options may cost more.',
       parseMaxPages
     )
-    .option('--only-main-content', 'Include only main content', false)
+    .option(
+      '--only-main-content',
+      'Include only the main content (the API default)'
+    )
+    .option(
+      '--full-page',
+      'Keep the whole page, including navigation, headers and footers'
+    )
     .option(
       '--wait-for <ms>',
       'Wait time before scraping in milliseconds',
@@ -468,6 +479,7 @@ function createScrapeCommand(): Command {
     .option('--proxy <proxy>', 'Proxy mode for scraping (e.g., auto, basic)')
 
     .action(async (positionalArgs, options) => {
+      onlyMainContentFlag(options);
       const target = resolveScrapeTarget(positionalArgs ?? [], options);
       if (target.kind === 'alexandria') {
         await handleAlexandria(target.calls, options);
@@ -585,7 +597,14 @@ function createDownloadCommand(): Command {
       '-S, --summary',
       'Download as summary (shortcut for --format summary)'
     )
-    .option('--only-main-content', 'Include only main content', false)
+    .option(
+      '--only-main-content',
+      'Include only the main content (the API default)'
+    )
+    .option(
+      '--full-page',
+      'Keep the whole page, including navigation, headers and footers'
+    )
     .option(
       '--wait-for <ms>',
       'Wait time before scraping in milliseconds',
@@ -704,7 +723,11 @@ function createCrawlCommand(): Command {
     .option('--crawl-entire-domain', 'Crawl entire domain', false)
     .option('--allow-external-links', 'Allow external links', false)
     .option('--allow-subdomains', 'Allow subdomains', false)
-    .option('--delay <ms>', 'Delay between requests in milliseconds', parseInt)
+    .option(
+      '--delay <ms>',
+      'Delay between requests in milliseconds (sent to the API in seconds)',
+      parseInt
+    )
     .option(
       '--max-concurrency <number>',
       'Maximum concurrent requests',
@@ -816,7 +839,14 @@ function createBatchScrapeCommand(): Command {
       '-f, --format <formats>',
       'Output format(s), comma-separated (default: markdown). Available: markdown, html, rawHtml, links, images, screenshot, summary, changeTracking, json, attributes, branding'
     )
-    .option('--only-main-content', 'Include only main content', false)
+    .option(
+      '--only-main-content',
+      'Include only the main content (the API default)'
+    )
+    .option(
+      '--full-page',
+      'Keep the whole page, including navigation, headers and footers'
+    )
     .option(
       '--wait-for <ms>',
       'Wait time before scraping in milliseconds',
@@ -925,7 +955,7 @@ function createBatchScrapeCommand(): Command {
         apiKey: options.apiKey,
         apiUrl: options.apiUrl,
         format: options.format,
-        onlyMainContent: options.onlyMainContent,
+        onlyMainContent: onlyMainContentFlag(options),
         waitFor: options.waitFor,
         maxAge: options.maxAge,
         schema,
@@ -1026,7 +1056,14 @@ function createParseCommand(): Command {
       '-f, --format <formats>',
       'Output format(s). Multiple formats can be specified with commas (e.g., "markdown,links"). Available: markdown, html, rawHtml, links, images, summary, json, attributes. Single format outputs raw content; multiple formats output JSON.'
     )
-    .option('--only-main-content', 'Include only main content', false)
+    .option(
+      '--only-main-content',
+      'Include only the main content (the API default)'
+    )
+    .option(
+      '--full-page',
+      'Keep the whole page, including navigation, headers and footers'
+    )
     .option('-S, --summary', 'Output summary (shortcut for --format summary)')
     .option('--include-tags <tags>', 'Comma-separated list of tags to include')
     .option('--exclude-tags <tags>', 'Comma-separated list of tags to exclude')

src/utils/options.ts

@@ -83,6 +83,20 @@ export function parseMaxPages(value: string): number {
 /**
  * Convert commander options to ScrapeOptions
  */
+/**
+ * onlyMainContent for a scrape: true with --only-main-content, false with
+ * --full-page, and unset otherwise so the API default (main content) applies.
+ */
+export function onlyMainContentFlag(options: {
+  onlyMainContent?: boolean;
+  fullPage?: boolean;
+}): boolean | undefined {
+  if (options.onlyMainContent && options.fullPage) {
+    throw new Error('Use either --only-main-content or --full-page, not both');
+  }
+  return options.fullPage ? false : options.onlyMainContent;
+}
+
 export function parseScrapeOptions(options: any): ScrapeOptions {
   // Parse formats from comma-separated string
   let formats: ScrapeFormat[] | undefined;
@@ -117,7 +131,7 @@ export function parseScrapeOptions(options: any): ScrapeOptions {
     domainTools: options.domainTools,
     toolDetail: options.toolDetail,
     formats,
-    onlyMainContent: options.onlyMainContent,
+    onlyMainContent: onlyMainContentFlag(options),
     waitFor: options.waitFor,
     screenshot: options.screenshot,
     fullPageScreenshot: options.fullPageScreenshot,