fix(firecrawl): read a scheme-less start URL as https when counting depth

Author: waleedlatif1

.github/CONTRIBUTING.md

@@ -148,7 +148,7 @@ The easiest way to run Sim locally is the interactive setup wizard:
 npx sim-setup
 ```
 
-When it finishes, open [http://localhost:3000/](http://localhost:3000/) in your browser. Inside a cloned Sim repository, run `bun run sim-setup` for the local development mode.
+When it finishes, open [http://localhost:3000/](http://localhost:3000/) in your browser. Inside a cloned Sim repository, run `bun run sim-setup --mode dev` for the local development mode.
 
 #### Requirements
 

apps/sim/tools/firecrawl/crawl.test.ts

@@ -72,6 +72,7 @@ describe('Firecrawl legacy crawl depth', () => {
     ['https://example.com/docs/guide/', '/docs/guide/setup', true],
     ['https://example.com/docs/guide/index.html', '/docs/guide/setup', true],
     ['https://example.com/docs/guide', '/docs/guide/setup/advanced', false],
+    ['docs.example.com/guide', '/guide/setup', true],
   ])('counts depth from the start URL %s for %s', (url, path, accepted) => {
     const body = crawlBody({ maxDepth: 1, url })
     const patterns = (body.includePaths as string[]).map((pattern) => new RegExp(pattern))

apps/sim/tools/firecrawl/crawl.ts

@@ -18,10 +18,14 @@ const logger = createLogger('FirecrawlCrawlTool')
 const POLL_INTERVAL_MS = 5000
 const MAX_POLL_TIME_MS = DEFAULT_EXECUTION_TIMEOUT_MS
 
-/** Path segments of a URL as Firecrawl v1 counted them: index files add no depth. */
+/**
+ * Path segments of a URL as Firecrawl v1 counted them: index files add no depth. A URL without
+ * a scheme is read as https, as Firecrawl reads it.
+ */
 function urlPathDepth(url: string): number {
-  if (!URL.canParse(url)) return 0
-  return new URL(url).pathname
+  const absolute = /^[a-z][a-z\d+.-]*:\/\//i.test(url) ? url : `https://${url}`
+  if (!URL.canParse(absolute)) return 0
+  return new URL(absolute).pathname
     .split('/')
     .filter((segment) => segment !== '' && segment !== 'index.html' && segment !== 'index.php')
     .length