diff --git a/src/__tests__/commands/crawl.test.ts b/src/__tests__/commands/crawl.test.ts index 830fd120d8..2efdd73c52 100644 --- a/src/__tests__/commands/crawl.test.ts +++ b/src/__tests__/commands/crawl.test.ts @@ -3,7 +3,7 @@ */ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { executeCrawl } from '../../commands/crawl'; +import { executeCrawl, handleCrawlCommand } from '../../commands/crawl'; import { getClient } from '../../utils/client'; import { initializeConfig } from '../../utils/config'; import { setupTest, teardownTest } from '../utils/mock-client'; @@ -271,17 +271,30 @@ describe('executeCrawl', () => { expect(mockClient.getCrawlStatus).toHaveBeenCalledWith( '550e8400-e29b-41d4-a716-446655440000' ); - expect(result).toEqual({ - success: true, - data: { - id: mockStatus.id, - status: mockStatus.status, - total: mockStatus.total, - completed: mockStatus.completed, - creditsUsed: mockStatus.creditsUsed, - expiresAt: mockStatus.expiresAt, - }, + expect(result).toEqual({ success: true, data: mockStatus }); + }); + + it('should return the pages collected so far with the status', async () => { + const mockStatus = { + id: '550e8400-e29b-41d4-a716-446655440000', + status: 'completed', + total: 1, + completed: 1, + data: [ + { + markdown: '# Page', + metadata: { sourceURL: 'https://example.com' }, + }, + ], + }; + mockClient.getCrawlStatus.mockResolvedValue(mockStatus); + + const result = await executeCrawl({ + urlOrJobId: '550e8400-e29b-41d4-a716-446655440000', }); + + expect(result.success).toBe(true); + expect(result.data.data).toEqual(mockStatus.data); }); it('should auto-detect job ID from UUID format', async () => { @@ -431,6 +444,189 @@ describe('executeCrawl', () => { }); }); + describe('Resume an existing job', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('should poll a job ID to a terminal status with --wait', async () => { + const jobId = '550e8400-e29b-41d4-a716-446655440000'; + mockClient.getCrawlStatus + .mockResolvedValueOnce({ + id: jobId, + status: 'scraping', + total: 2, + completed: 1, + data: [], + }) + .mockResolvedValueOnce({ + id: jobId, + status: 'completed', + total: 2, + completed: 2, + data: [{}, {}], + }); + + const crawlPromise = executeCrawl({ + urlOrJobId: jobId, + wait: true, + pollInterval: 0.001, + }); + await vi.advanceTimersByTimeAsync(1); + const result = await crawlPromise; + + expect(mockClient.startCrawl).not.toHaveBeenCalled(); + expect(mockClient.crawl).not.toHaveBeenCalled(); + expect(mockClient.getCrawlStatus).toHaveBeenCalledTimes(2); + expect(result).toEqual({ + success: true, + data: { + id: jobId, + status: 'completed', + total: 2, + completed: 2, + data: [{}, {}], + }, + }); + }); + + it('should name the resume command when waiting on a job ID times out', async () => { + const jobId = '550e8400-e29b-41d4-a716-446655440000'; + mockClient.getCrawlStatus.mockResolvedValue({ + id: jobId, + status: 'scraping', + total: 2, + completed: 1, + data: [], + }); + + const crawlPromise = executeCrawl({ + urlOrJobId: jobId, + wait: true, + pollInterval: 1, + timeout: 1, + }); + await vi.advanceTimersByTimeAsync(2500); + const result = await crawlPromise; + + expect(result.success).toBe(false); + expect(result.error).toContain(`firecrawl crawl ${jobId} --wait`); + }); + + it('should poll one status page at a time and fetch every page once the job ends', async () => { + const jobId = '550e8400-e29b-41d4-a716-446655440000'; + const allPages = { + id: jobId, + status: 'completed', + total: 2, + completed: 2, + data: [{}, {}], + }; + mockClient.getCrawlStatus + .mockResolvedValueOnce({ + ...allPages, + data: [{}], + next: 'https://api.firecrawl.dev/v2/crawl/x?skip=1', + }) + .mockResolvedValueOnce(allPages); + + const result = await executeCrawl({ urlOrJobId: jobId, wait: true }); + + expect(mockClient.getCrawlStatus).toHaveBeenNthCalledWith(1, jobId, { + autoPaginate: false, + }); + expect(mockClient.getCrawlStatus).toHaveBeenNthCalledWith(2, jobId); + expect(result).toEqual({ success: true, data: allPages }); + }); + + for (const [flag, options] of [ + ['--poll-interval', { pollInterval: 0 }], + ['--timeout', { timeout: Number.NaN }], + ] as const) { + it(`should reject an invalid ${flag} before polling`, async () => { + const result = await executeCrawl({ + urlOrJobId: '550e8400-e29b-41d4-a716-446655440000', + wait: true, + ...options, + }); + + expect(result).toEqual({ + success: false, + error: `${flag} must be a positive number of seconds`, + }); + expect(mockClient.getCrawlStatus).not.toHaveBeenCalled(); + }); + } + }); + + describe('Exit status', () => { + afterEach(() => { + process.exitCode = undefined; + vi.restoreAllMocks(); + }); + + for (const status of ['failed', 'cancelled']) { + it(`should print the job and exit 1 when a waited crawl ends ${status}`, async () => { + const stdout = vi + .spyOn(process.stdout, 'write') + .mockImplementation(() => true); + const stderr = vi.spyOn(console, 'error').mockImplementation(() => {}); + mockClient.crawl.mockResolvedValue({ + id: 'job-1', + status, + total: 1, + completed: 0, + data: [], + }); + + await handleCrawlCommand({ + urlOrJobId: 'https://example.com', + wait: true, + }); + + expect( + stdout.mock.calls.map(([chunk]) => String(chunk)).join('') + ).toContain(`"status":"${status}"`); + expect(stderr).toHaveBeenCalledWith(`Error: Crawl ${status}.`); + expect(process.exitCode).toBe(1); + }); + } + + it('should leave the exit status alone when reading a failed job without --wait', async () => { + vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + mockClient.getCrawlStatus.mockResolvedValue({ + id: 'job-1', + status: 'failed', + total: 1, + completed: 0, + data: [], + }); + + await handleCrawlCommand({ + urlOrJobId: '550e8400-e29b-41d4-a716-446655440000', + }); + + expect(process.exitCode).toBeUndefined(); + }); + + it('should leave the exit status alone after a successful cancel with --wait', async () => { + vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + mockClient.cancelCrawl = vi.fn().mockResolvedValue(true); + + await handleCrawlCommand({ + urlOrJobId: '550e8400-e29b-41d4-a716-446655440000', + cancel: true, + wait: true, + }); + + expect(process.exitCode).toBeUndefined(); + }); + }); + describe('Progress mode', () => { beforeEach(() => { // Mock process.stderr.write to avoid console output during tests diff --git a/src/commands/crawl.ts b/src/commands/crawl.ts index 6ef47f16e7..d9107e3422 100644 --- a/src/commands/crawl.ts +++ b/src/commands/crawl.ts @@ -2,52 +2,20 @@ * Crawl command implementation */ -import type { - CrawlOptions, - CrawlResult, - CrawlStatusResult, -} from '../types/crawl'; +import type { CrawlOptions, CrawlResult } from '../types/crawl'; import { getClient } from '../utils/client'; import { isJobId } from '../utils/job'; import { writeOutput } from '../utils/output'; import { createProgressLine } from '../utils/spinner'; -/** - * Execute crawl status check - */ -async function checkCrawlStatus( - jobId: string, - options: CrawlOptions -): Promise { - try { - const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); - const status = await app.getCrawlStatus(jobId); - - return { - success: true, - data: { - id: status.id, - status: status.status, - total: status.total, - completed: status.completed, - creditsUsed: status.creditsUsed, - expiresAt: status.expiresAt, - }, - }; - } catch (error) { - return { - success: false, - error: error instanceof Error ? error.message : 'Unknown error occurred', - }; - } -} +const TERMINAL_STATUSES = new Set(['completed', 'failed', 'cancelled']); /** * Execute crawl command */ export async function executeCrawl( options: CrawlOptions -): Promise { +): Promise { try { const app = getClient({ apiKey: options.apiKey, apiUrl: options.apiUrl }); const { urlOrJobId, status, wait, pollInterval, timeout, cancel } = options; @@ -79,9 +47,55 @@ export async function executeCrawl( }; } - // If status flag is set or input looks like a job ID, check status + // A job ID returns the job's status and pages; with --wait, poll it to the end if (status || isJobId(urlOrJobId)) { - return await checkCrawlStatus(urlOrJobId, options); + if (!wait) { + return { success: true, data: await app.getCrawlStatus(urlOrJobId) }; + } + + for (const [name, value] of [ + ['--poll-interval', pollInterval], + ['--timeout', timeout], + ] as const) { + if (value !== undefined && !(Number.isFinite(value) && value > 0)) { + return { + success: false, + error: `${name} must be a positive number of seconds`, + }; + } + } + + const pollMs = (pollInterval ?? 5) * 1000; + const progress = options.progress ? createProgressLine() : undefined; + const deadline = + timeout !== undefined ? Date.now() + timeout * 1000 : undefined; + while (true) { + const job = await app.getCrawlStatus(urlOrJobId, { + autoPaginate: false, + }); + progress?.update( + `Progress: ${job.completed}/${job.total} pages (${job.status})` + ); + if (TERMINAL_STATUSES.has(job.status)) { + progress?.end(); + return { + success: true, + data: job.next ? await app.getCrawlStatus(urlOrJobId) : job, + }; + } + const remainingMs = + deadline !== undefined ? deadline - Date.now() : undefined; + if (remainingMs !== undefined && remainingMs <= 0) { + progress?.end(); + return { + success: false, + error: `Timeout after ${timeout} seconds. Crawl still in progress. Resume with: firecrawl crawl ${urlOrJobId} --wait`, + }; + } + await new Promise((resolve) => + setTimeout(resolve, Math.min(pollMs, remainingMs ?? pollMs)) + ); + } } // Build crawl options @@ -162,11 +176,7 @@ export async function executeCrawl( `Progress: ${status.completed}/${status.total} pages (${status.status})` ); - if ( - status.status === 'completed' || - status.status === 'failed' || - status.status === 'cancelled' - ) { + if (TERMINAL_STATUSES.has(status.status)) { progress.end(); return { success: true, @@ -215,37 +225,6 @@ export async function executeCrawl( } } -/** - * Format crawl status in human-readable way - */ -function formatCrawlStatus(data: CrawlStatusResult['data']): string { - if (!data) return ''; - - const lines: string[] = []; - lines.push(`Job ID: ${data.id}`); - lines.push(`Status: ${data.status}`); - lines.push(`Progress: ${data.completed}/${data.total} pages`); - - if (data.creditsUsed !== undefined) { - lines.push(`Credits Used: ${data.creditsUsed}`); - } - - if (data.expiresAt) { - const expiresDate = new Date(data.expiresAt); - lines.push( - `Expires: ${expiresDate.toLocaleString('en-US', { - year: 'numeric', - month: 'short', - day: 'numeric', - hour: '2-digit', - minute: '2-digit', - })}` - ); - } - - return lines.join('\n') + '\n'; -} - /** * Handle crawl command output */ @@ -257,29 +236,8 @@ export async function handleCrawlCommand(options: CrawlOptions): Promise { process.exit(1); } - // Handle status check result - if ('status' in result && result.data && 'status' in result.data) { - const statusResult = result as CrawlStatusResult; - if (statusResult.data) { - let outputContent: string; - - if (options.pretty || !options.output) { - // Human-readable format for status - outputContent = formatCrawlStatus(statusResult.data); - } else { - // JSON format - outputContent = options.pretty - ? JSON.stringify({ success: true, data: statusResult.data }, null, 2) - : JSON.stringify({ success: true, data: statusResult.data }); - } - - writeOutput(outputContent, options.output, !!options.output); - return; - } - } - - // Handle crawl result (job ID or completed crawl) - const crawlResult = result as CrawlResult; + // Handle crawl result (job ID, job status, or completed crawl) + const crawlResult = result; if (!crawlResult.data) { return; } @@ -306,4 +264,14 @@ export async function handleCrawlCommand(options: CrawlOptions): Promise { } writeOutput(outputContent, options.output, !!options.output); + + const finalStatus = crawlResult.data.status; + if ( + options.wait && + !options.cancel && + (finalStatus === 'failed' || finalStatus === 'cancelled') + ) { + console.error(`Error: Crawl ${finalStatus}.`); + process.exitCode = 1; + } } diff --git a/src/index.ts b/src/index.ts index 0b20816b7f..76947d5e75 100644 --- a/src/index.ts +++ b/src/index.ts @@ -656,15 +656,22 @@ function createDownloadCommand(): Command { function createCrawlCommand(): Command { const crawlCmd = new Command('crawl') .description('Crawl a website using Firecrawl') - .argument('[url-or-job-id]', 'URL to crawl or job ID to check status') + .argument( + '[url-or-job-id]', + 'URL to crawl, or job ID to read its status and pages' + ) .option( '-u, --url ', 'URL to crawl (alternative to positional argument)' ) - .option('--status', 'Check status of existing crawl job', false) + .option( + '--status', + 'Read the status and pages of an existing crawl job', + false + ) .option( '--wait', - 'Wait for crawl to complete before returning results', + 'Wait for the crawl (new or by job ID) to finish before returning results; exits 1 if it fails or is cancelled', false ) .option( diff --git a/src/types/crawl.ts b/src/types/crawl.ts index 84b0599fa3..144f1aee7e 100644 --- a/src/types/crawl.ts +++ b/src/types/crawl.ts @@ -58,16 +58,3 @@ export interface CrawlResult { data?: any; error?: string; } - -export interface CrawlStatusResult { - success: boolean; - data?: { - id: string; - status: 'scraping' | 'completed' | 'failed' | 'cancelled'; - total: number; - completed: number; - creditsUsed?: number; - expiresAt?: string; - }; - error?: string; -}