Skip to content

Commit b570a57

Browse files
VinciGit00claude
andcommitted
feat: replace render/stealth booleans with unified fetch mode enum
Aligns fetchConfig with sgai-stack development branch contract: - mode: "auto" | "fast" | "js" | "direct+stealth" | "js+stealth" - Removes render and stealth boolean fields - Updates timeout range to 1000-60000ms (default 30000) - Adds SGAI-APIKEY header to all requests - Fixes API URL paths (/v2 → /api/v2) - Exports ApiFetchMode type Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 4b86432 commit b570a57

File tree

7 files changed

+48
-30
lines changed

7 files changed

+48
-30
lines changed

src/client.ts

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ export function scrapegraphai(config: ClientConfig) {
3030
async scrape(url: string, scrapeOptions?: ApiScrapeOptions, requestOptions?: RequestOptions) {
3131
return request(
3232
"POST",
33-
buildUrl("/v2/scrape"),
33+
buildUrl("/api/v2/scrape"),
3434
key,
3535
{ url, ...scrapeOptions },
3636
mergeRequestOptions(requestOptions),
@@ -46,7 +46,7 @@ export function scrapegraphai(config: ClientConfig) {
4646
if (extractOptions.fetchConfig) body.fetchConfig = extractOptions.fetchConfig;
4747
return request(
4848
"POST",
49-
buildUrl("/v2/extract"),
49+
buildUrl("/api/v2/extract"),
5050
key,
5151
body,
5252
mergeRequestOptions(requestOptions),
@@ -56,7 +56,7 @@ export function scrapegraphai(config: ClientConfig) {
5656
async search(query: string, searchOptions?: ApiSearchOptions, requestOptions?: RequestOptions) {
5757
return request(
5858
"POST",
59-
buildUrl("/v2/search"),
59+
buildUrl("/api/v2/search"),
6060
key,
6161
{ query, ...searchOptions },
6262
mergeRequestOptions(requestOptions),
@@ -66,7 +66,7 @@ export function scrapegraphai(config: ClientConfig) {
6666
async credits(requestOptions?: RequestOptions) {
6767
return request(
6868
"GET",
69-
buildUrl("/v2/credits"),
69+
buildUrl("/api/v2/credits"),
7070
key,
7171
undefined,
7272
mergeRequestOptions(requestOptions),
@@ -81,7 +81,7 @@ export function scrapegraphai(config: ClientConfig) {
8181
const query = qs.toString();
8282
return request(
8383
"GET",
84-
buildUrl(`/v2/history${query ? `?${query}` : ""}`),
84+
buildUrl(`/api/v2/history${query ? `?${query}` : ""}`),
8585
key,
8686
undefined,
8787
mergeRequestOptions(requestOptions),
@@ -92,7 +92,7 @@ export function scrapegraphai(config: ClientConfig) {
9292
async start(url: string, crawlOptions?: ApiCrawlOptions, requestOptions?: RequestOptions) {
9393
return request(
9494
"POST",
95-
buildUrl("/v2/crawl"),
95+
buildUrl("/api/v2/crawl"),
9696
key,
9797
{ url, ...crawlOptions },
9898
mergeRequestOptions(requestOptions),
@@ -102,7 +102,7 @@ export function scrapegraphai(config: ClientConfig) {
102102
async status(id: string, requestOptions?: RequestOptions) {
103103
return request(
104104
"GET",
105-
buildUrl(`/v2/crawl/${id}`),
105+
buildUrl(`/api/v2/crawl/${id}`),
106106
key,
107107
undefined,
108108
mergeRequestOptions(requestOptions),
@@ -112,7 +112,7 @@ export function scrapegraphai(config: ClientConfig) {
112112
async stop(id: string, requestOptions?: RequestOptions) {
113113
return request(
114114
"POST",
115-
buildUrl(`/v2/crawl/${id}/stop`),
115+
buildUrl(`/api/v2/crawl/${id}/stop`),
116116
key,
117117
{},
118118
mergeRequestOptions(requestOptions),
@@ -122,7 +122,7 @@ export function scrapegraphai(config: ClientConfig) {
122122
async resume(id: string, requestOptions?: RequestOptions) {
123123
return request(
124124
"POST",
125-
buildUrl(`/v2/crawl/${id}/resume`),
125+
buildUrl(`/api/v2/crawl/${id}/resume`),
126126
key,
127127
{},
128128
mergeRequestOptions(requestOptions),
@@ -134,7 +134,7 @@ export function scrapegraphai(config: ClientConfig) {
134134
async create(monitorCreateInput: ApiMonitorCreateInput, requestOptions?: RequestOptions) {
135135
return request(
136136
"POST",
137-
buildUrl("/v2/monitor"),
137+
buildUrl("/api/v2/monitor"),
138138
key,
139139
{ ...monitorCreateInput },
140140
mergeRequestOptions(requestOptions),
@@ -144,7 +144,7 @@ export function scrapegraphai(config: ClientConfig) {
144144
async list(requestOptions?: RequestOptions) {
145145
return request(
146146
"GET",
147-
buildUrl("/v2/monitor"),
147+
buildUrl("/api/v2/monitor"),
148148
key,
149149
undefined,
150150
mergeRequestOptions(requestOptions),
@@ -154,7 +154,7 @@ export function scrapegraphai(config: ClientConfig) {
154154
async get(id: string, requestOptions?: RequestOptions) {
155155
return request(
156156
"GET",
157-
buildUrl(`/v2/monitor/${id}`),
157+
buildUrl(`/api/v2/monitor/${id}`),
158158
key,
159159
undefined,
160160
mergeRequestOptions(requestOptions),
@@ -164,7 +164,7 @@ export function scrapegraphai(config: ClientConfig) {
164164
async pause(id: string, requestOptions?: RequestOptions) {
165165
return request(
166166
"POST",
167-
buildUrl(`/v2/monitor/${id}/pause`),
167+
buildUrl(`/api/v2/monitor/${id}/pause`),
168168
key,
169169
{},
170170
mergeRequestOptions(requestOptions),
@@ -174,7 +174,7 @@ export function scrapegraphai(config: ClientConfig) {
174174
async resume(id: string, requestOptions?: RequestOptions) {
175175
return request(
176176
"POST",
177-
buildUrl(`/v2/monitor/${id}/resume`),
177+
buildUrl(`/api/v2/monitor/${id}/resume`),
178178
key,
179179
{},
180180
mergeRequestOptions(requestOptions),
@@ -184,7 +184,7 @@ export function scrapegraphai(config: ClientConfig) {
184184
async delete(id: string, requestOptions?: RequestOptions) {
185185
return request(
186186
"DELETE",
187-
buildUrl(`/v2/monitor/${id}`),
187+
buildUrl(`/api/v2/monitor/${id}`),
188188
key,
189189
undefined,
190190
mergeRequestOptions(requestOptions),

src/http.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export async function request<T = unknown>(
2222
try {
2323
const headers: Record<string, string> = {
2424
Authorization: `Bearer ${apiKey}`,
25+
"SGAI-APIKEY": apiKey,
2526
"X-SDK-Version": `js@${VERSION}`,
2627
};
2728
if (body) headers["Content-Type"] = "application/json";

src/index.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ export type {
66
ApiExtractOptions,
77
ApiExtractRequestBase,
88
ApiFetchConfig,
9+
ApiFetchMode,
910
ApiHistoryFilterInput,
1011
ApiHistoryService,
1112
ApiHtmlMode,

src/schemas.ts

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ export const apiFetchContentTypeSchema = z.enum([
3232
"text/plain",
3333
"application/x-latex",
3434
]);
35+
export const apiFetchModeSchema = z.enum(["auto", "fast", "js", "direct+stealth", "js+stealth"]);
3536
export const apiUserPromptSchema = z.string().min(1).max(10_000);
3637

3738
export const apiUrlSchema = z.url().check(
@@ -56,16 +57,15 @@ export const apiUuidParamSchema = z.object({
5657
});
5758

5859
export const FETCH_CONFIG_DEFAULTS = {
59-
timeout: 15000,
60-
render: false,
60+
mode: "auto",
61+
timeout: 30000,
6162
wait: 0,
62-
stealth: false,
6363
scrolls: 0,
6464
} as const;
6565

6666
export const apiFetchConfigSchema = z.object({
67-
timeout: z.number().int().min(1000).max(30000).default(FETCH_CONFIG_DEFAULTS.timeout),
68-
render: z.boolean().default(FETCH_CONFIG_DEFAULTS.render),
67+
mode: apiFetchModeSchema.default(FETCH_CONFIG_DEFAULTS.mode),
68+
timeout: z.number().int().min(1000).max(60000).default(FETCH_CONFIG_DEFAULTS.timeout),
6969
wait: z.number().int().min(0).max(30000).default(FETCH_CONFIG_DEFAULTS.wait),
7070
headers: z.record(z.string(), z.string()).optional(),
7171
cookies: z.record(z.string(), z.string()).optional(),
@@ -74,7 +74,6 @@ export const apiFetchConfigSchema = z.object({
7474
.length(2)
7575
.transform((v) => v.toLowerCase())
7676
.optional(),
77-
stealth: z.boolean().default(FETCH_CONFIG_DEFAULTS.stealth),
7877
scrolls: z.number().int().min(0).max(100).default(FETCH_CONFIG_DEFAULTS.scrolls),
7978
mock: z
8079
.union([

src/types/index.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import type {
33
apiCrawlRequestSchema,
44
apiExtractRequestBaseSchema,
55
apiFetchConfigSchema,
6+
apiFetchModeSchema,
67
apiHistoryFilterSchema,
78
apiHtmlModeSchema,
89
apiLlmConfigSchema,
@@ -18,6 +19,7 @@ export const DEFAULT_BASE_URL = "https://api.scrapegraphai.com";
1819
export type { ApiModelName } from "../models.js";
1920

2021
export type ApiHtmlMode = z.infer<typeof apiHtmlModeSchema>;
22+
export type ApiFetchMode = z.infer<typeof apiFetchModeSchema>;
2123

2224
export interface ClientConfig {
2325
apiKey: string;

tests/client.test.ts

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ function mockApi(routes: Record<string, (req: Request) => Response | Promise<Res
1717
describe("scrapegraphai", () => {
1818
test("scrape returns markdown + requestId", async () => {
1919
const api = mockApi({
20-
"/v2/scrape": () =>
20+
"/api/v2/scrape": () =>
2121
Response.json({ markdown: "# Hello" }, { headers: { "x-request-id": "req-1" } }),
2222
});
2323
const sgai = scrapegraphai({ apiKey: "test", baseUrl: api.url });
@@ -30,7 +30,7 @@ describe("scrapegraphai", () => {
3030
test("extract sends prompt + schema", async () => {
3131
let body: any;
3232
const api = mockApi({
33-
"/v2/extract": async (req) => {
33+
"/api/v2/extract": async (req) => {
3434
body = await req.json();
3535
return Response.json({ json: { name: "Test" } }, { headers: { "x-request-id": "req-2" } });
3636
},
@@ -48,7 +48,7 @@ describe("scrapegraphai", () => {
4848
test("search sends query", async () => {
4949
let body: any;
5050
const api = mockApi({
51-
"/v2/search": async (req) => {
51+
"/api/v2/search": async (req) => {
5252
body = await req.json();
5353
return Response.json(
5454
{ results: [{ url: "https://a.com" }] },
@@ -65,7 +65,7 @@ describe("scrapegraphai", () => {
6565

6666
test("credits returns balance", async () => {
6767
const api = mockApi({
68-
"/v2/credits": () =>
68+
"/api/v2/credits": () =>
6969
Response.json(
7070
{ remainingCredits: 5000, totalCreditsUsed: 200 },
7171
{ headers: { "x-request-id": "req-4" } },
@@ -79,7 +79,7 @@ describe("scrapegraphai", () => {
7979

8080
test("crawl.start returns job id", async () => {
8181
const api = mockApi({
82-
"/v2/crawl": () =>
82+
"/api/v2/crawl": () =>
8383
Response.json(
8484
{ id: "crawl-123", status: "running" },
8585
{ headers: { "x-request-id": "req-5" } },
@@ -93,7 +93,7 @@ describe("scrapegraphai", () => {
9393

9494
test("crawl.status returns status", async () => {
9595
const api = mockApi({
96-
"/v2/crawl/crawl-123": () =>
96+
"/api/v2/crawl/crawl-123": () =>
9797
Response.json(
9898
{ id: "crawl-123", status: "completed", pages: [] },
9999
{ headers: { "x-request-id": "req-6" } },
@@ -108,7 +108,7 @@ describe("scrapegraphai", () => {
108108
test("crawl.stop sends POST", async () => {
109109
let called = false;
110110
const api = mockApi({
111-
"/v2/crawl/crawl-123/stop": () => {
111+
"/api/v2/crawl/crawl-123/stop": () => {
112112
called = true;
113113
return Response.json({ ok: true }, { headers: { "x-request-id": "req-7" } });
114114
},
@@ -122,7 +122,7 @@ describe("scrapegraphai", () => {
122122
test("monitor.create sends body", async () => {
123123
let body: any;
124124
const api = mockApi({
125-
"/v2/monitor": async (req) => {
125+
"/api/v2/monitor": async (req) => {
126126
body = await req.json();
127127
return Response.json({ id: "mon-1" }, { headers: { "x-request-id": "req-8" } });
128128
},
@@ -141,7 +141,7 @@ describe("scrapegraphai", () => {
141141
test("monitor.delete sends DELETE", async () => {
142142
let method: string | undefined;
143143
const api = mockApi({
144-
"/v2/monitor/mon-1": (req) => {
144+
"/api/v2/monitor/mon-1": (req) => {
145145
method = req.method;
146146
return Response.json({}, { headers: { "x-request-id": "req-9" } });
147147
},
@@ -152,6 +152,20 @@ describe("scrapegraphai", () => {
152152
api.stop();
153153
});
154154

155+
test("scrape forwards fetchConfig.mode", async () => {
156+
let body: any;
157+
const api = mockApi({
158+
"/api/v2/scrape": async (req) => {
159+
body = await req.json();
160+
return Response.json({ markdown: "ok" }, { headers: { "x-request-id": "req-fm" } });
161+
},
162+
});
163+
const sgai = scrapegraphai({ apiKey: "test", baseUrl: api.url });
164+
await sgai.scrape("https://example.com", { fetchConfig: { mode: "js+stealth" } });
165+
expect(body.fetchConfig.mode).toBe("js+stealth");
166+
api.stop();
167+
});
168+
155169
test("history appends query params", async () => {
156170
let requestUrl: string | undefined;
157171
const server = Bun.serve({

tests/http.test.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ describe("request", () => {
4141
await request("GET", `${server.url}/test`, "my-api-key");
4242
expect(capturedHeaders).toBeDefined();
4343
expect(capturedHeaders?.get("authorization")).toBe("Bearer my-api-key");
44+
expect(capturedHeaders?.get("sgai-apikey")).toBe("my-api-key");
4445
expect(capturedHeaders?.get("x-sdk-version")).toContain("js@");
4546
server.stop();
4647
});

0 commit comments

Comments
 (0)