Skip to content

Commit d71e4d9

Browse files
docs: update examples for v2 API
- Remove old v1 examples (smartscraper, markdownify, searchscraper, sitemap, agenticscraper) - Add scrape examples (basic, multi-format, pdf, fetchConfig) - Add extract examples (basic, with-schema) - Add search examples (basic, with-extraction) - Add monitor examples (basic, with-webhook) - Update crawl examples for namespace API - Update schema examples for camelCase fields - Update utilities for v2 response shapes Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 6844d33 commit d71e4d9

37 files changed

Lines changed: 240 additions & 561 deletions

examples/agenticscraper/agenticscraper_ai_extraction.ts

Lines changed: 0 additions & 35 deletions
This file was deleted.

examples/agenticscraper/agenticscraper_basic.ts

Lines changed: 0 additions & 22 deletions
This file was deleted.

examples/crawl/crawl_basic.ts

Lines changed: 15 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,20 @@ import { crawl } from "scrapegraph-js";
22

33
const apiKey = process.env.SGAI_API_KEY!;
44

5-
const res = await crawl(
6-
apiKey,
7-
{
8-
url: "https://scrapegraphai.com",
9-
prompt: "Extract the main content from each page",
10-
max_pages: 5,
11-
depth: 2,
12-
sitemap: true,
13-
},
14-
(status) => console.log(`Poll: ${status}`),
15-
);
5+
const startRes = await crawl.start(apiKey, {
6+
url: "https://example.com",
7+
maxPages: 5,
8+
maxDepth: 2,
9+
});
1610

17-
if (res.status === "success") {
18-
console.log("Pages crawled:", res.data?.crawled_urls?.length);
19-
console.log("Result:", JSON.stringify(res.data?.llm_result, null, 2));
20-
console.log(`Took ${res.elapsedMs}ms`);
21-
} else {
22-
console.error("Failed:", res.error);
11+
if (startRes.status !== "success") {
12+
console.error("Failed to start:", startRes.error);
13+
process.exit(1);
2314
}
15+
16+
console.log("Crawl started:", startRes.data?.id);
17+
console.log("Status:", startRes.data?.status);
18+
19+
const getRes = await crawl.get(apiKey, startRes.data!.id);
20+
console.log("\nProgress:", getRes.data?.finished, "/", getRes.data?.total);
21+
console.log("Pages:", getRes.data?.pages.map((p) => p.url));

examples/crawl/crawl_markdown.ts

Lines changed: 0 additions & 28 deletions
This file was deleted.
Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import { crawl } from "scrapegraph-js";
2+
3+
const apiKey = process.env.SGAI_API_KEY!;
4+
5+
const res = await crawl.start(apiKey, {
6+
url: "https://example.com",
7+
formats: [
8+
{ type: "markdown", mode: "reader" },
9+
{ type: "screenshot", width: 1280, height: 720 },
10+
],
11+
maxPages: 10,
12+
maxDepth: 2,
13+
includePatterns: ["/blog/*", "/docs/*"],
14+
excludePatterns: ["/admin/*"],
15+
});
16+
17+
if (res.status === "success") {
18+
console.log("Crawl ID:", res.data?.id);
19+
console.log("Status:", res.data?.status);
20+
console.log("Total pages to crawl:", res.data?.total);
21+
} else {
22+
console.error("Failed:", res.error);
23+
}

examples/crawl/crawl_with_schema.ts

Lines changed: 0 additions & 50 deletions
This file was deleted.

examples/extract/extract_basic.ts

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { extract } from "scrapegraph-js";
2+
3+
const apiKey = process.env.SGAI_API_KEY!;
4+
5+
const res = await extract(apiKey, {
6+
url: "https://example.com",
7+
prompt: "What is this page about? Extract the main heading and description.",
8+
});
9+
10+
if (res.status === "success") {
11+
console.log("Extracted:", JSON.stringify(res.data?.json, null, 2));
12+
console.log("\nTokens used:", res.data?.usage);
13+
} else {
14+
console.error("Failed:", res.error);
15+
}
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import { extract } from "scrapegraph-js";
2+
3+
const apiKey = process.env.SGAI_API_KEY!;
4+
5+
const res = await extract(apiKey, {
6+
url: "https://example.com",
7+
prompt: "Extract the page title and description",
8+
schema: {
9+
type: "object",
10+
properties: {
11+
title: { type: "string" },
12+
description: { type: "string" },
13+
},
14+
required: ["title"],
15+
},
16+
});
17+
18+
if (res.status === "success") {
19+
console.log("Extracted:", JSON.stringify(res.data?.json, null, 2));
20+
} else {
21+
console.error("Failed:", res.error);
22+
}

examples/markdownify/markdownify_basic.ts

Lines changed: 0 additions & 13 deletions
This file was deleted.

examples/markdownify/markdownify_stealth.ts

Lines changed: 0 additions & 17 deletions
This file was deleted.

0 commit comments

Comments
 (0)