Skip to content

Commit f7574ec

Browse files
lurensslurenss
committed
refactor: align client types with sgai stack contracts
1 parent 0447d3b commit f7574ec

7 files changed

Lines changed: 338 additions & 78 deletions

File tree

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,15 @@ const sgai = scrapegraphai({ apiKey: "your-api-key" });
2323
const result = await sgai.scrape("https://example.com", { format: "markdown" });
2424

2525
console.log(result.data);
26-
console.log(result._requestId);
26+
console.log(result.requestId);
2727
```
2828

2929
Every method returns:
3030

3131
```ts
3232
type ApiResult<T> = {
3333
data: T;
34-
_requestId: string;
34+
requestId: string;
3535
};
3636
```
3737

‎src/client.ts‎

Lines changed: 144 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,122 +1,209 @@
11
import { request } from "./http.js";
22
import type {
3+
ApiCrawlOptions,
4+
ApiExtractOptions,
5+
ApiGenerateSchemaOptions,
6+
ApiHistoryFilterInput,
7+
ApiMonitorCreateInput,
8+
ApiScrapeOptions,
9+
ApiSearchOptions,
310
ClientConfig,
4-
CrawlOptions,
5-
MonitorCreateOptions,
611
RequestOptions,
712
} from "./types/index.js";
813
import { DEFAULT_BASE_URL } from "./types/index.js";
914
import { toJsonSchema } from "./zod.js";
1015

11-
/** Create a ScrapeGraphAI client. All methods return `{ data, _requestId }`. */
16+
/** Create a ScrapeGraphAI client. All methods return `{ data, requestId }`. */
1217
export function scrapegraphai(config: ClientConfig) {
1318
const base = (config.baseUrl ?? DEFAULT_BASE_URL).replace(/\/$/, "");
1419
const key = config.apiKey;
1520
const defaults = { maxRetries: config.maxRetries, timeout: config.timeout };
1621

17-
function url(path: string) {
22+
function buildUrl(path: string) {
1823
return `${base}${path}`;
1924
}
2025

21-
function opts(ro?: RequestOptions) {
22-
return { ...defaults, ...ro };
26+
function mergeRequestOptions(requestOptions?: RequestOptions) {
27+
return { ...defaults, ...requestOptions };
2328
}
2429

2530
return {
26-
async scrape(
27-
targetUrl: string,
28-
options?: { format?: string; fetchConfig?: Record<string, unknown> },
29-
ro?: RequestOptions,
30-
) {
31-
return request("POST", url("/v2/scrape"), key, { url: targetUrl, ...options }, opts(ro));
31+
async scrape(url: string, scrapeOptions?: ApiScrapeOptions, requestOptions?: RequestOptions) {
32+
return request(
33+
"POST",
34+
buildUrl("/v2/scrape"),
35+
key,
36+
{ url, ...scrapeOptions },
37+
mergeRequestOptions(requestOptions),
38+
);
3239
},
3340

34-
async extract(
35-
targetUrl: string,
36-
options: { prompt: string; schema?: unknown; llmConfig?: Record<string, unknown> },
37-
ro?: RequestOptions,
38-
) {
39-
const body: Record<string, unknown> = { url: targetUrl, prompt: options.prompt };
40-
if (options.schema) body.schema = toJsonSchema(options.schema);
41-
if (options.llmConfig) body.llmConfig = options.llmConfig;
42-
return request("POST", url("/v2/extract"), key, body, opts(ro));
41+
async extract(url: string, extractOptions: ApiExtractOptions, requestOptions?: RequestOptions) {
42+
const body: Record<string, unknown> = { url, prompt: extractOptions.prompt };
43+
if (extractOptions.schema) body.schema = toJsonSchema(extractOptions.schema);
44+
if (extractOptions.llmConfig) body.llmConfig = extractOptions.llmConfig;
45+
if (extractOptions.mode) body.mode = extractOptions.mode;
46+
if (extractOptions.contentType) body.contentType = extractOptions.contentType;
47+
if (extractOptions.fetchConfig) body.fetchConfig = extractOptions.fetchConfig;
48+
return request(
49+
"POST",
50+
buildUrl("/v2/extract"),
51+
key,
52+
body,
53+
mergeRequestOptions(requestOptions),
54+
);
4355
},
4456

45-
async search(query: string, options?: { numResults?: number }, ro?: RequestOptions) {
46-
return request("POST", url("/v2/search"), key, { query, ...options }, opts(ro));
57+
async search(query: string, searchOptions?: ApiSearchOptions, requestOptions?: RequestOptions) {
58+
return request(
59+
"POST",
60+
buildUrl("/v2/search"),
61+
key,
62+
{ query, ...searchOptions },
63+
mergeRequestOptions(requestOptions),
64+
);
4765
},
4866

4967
async schema(
5068
prompt: string,
51-
options?: { existingSchema?: Record<string, unknown> },
52-
ro?: RequestOptions,
69+
schemaOptions?: ApiGenerateSchemaOptions,
70+
requestOptions?: RequestOptions,
5371
) {
54-
return request("POST", url("/v2/schema"), key, { prompt, ...options }, opts(ro));
72+
return request(
73+
"POST",
74+
buildUrl("/v2/schema"),
75+
key,
76+
{ prompt, ...schemaOptions },
77+
mergeRequestOptions(requestOptions),
78+
);
5579
},
5680

57-
async credits(ro?: RequestOptions) {
58-
return request("GET", url("/v2/credits"), key, undefined, opts(ro));
81+
async credits(requestOptions?: RequestOptions) {
82+
return request(
83+
"GET",
84+
buildUrl("/v2/credits"),
85+
key,
86+
undefined,
87+
mergeRequestOptions(requestOptions),
88+
);
5989
},
6090

61-
async history(
62-
options?: { page?: number; limit?: number; service?: string },
63-
ro?: RequestOptions,
64-
) {
91+
async history(historyFilter?: ApiHistoryFilterInput, requestOptions?: RequestOptions) {
6592
const qs = new URLSearchParams();
66-
if (options?.page != null) qs.set("page", String(options.page));
67-
if (options?.limit != null) qs.set("limit", String(options.limit));
68-
if (options?.service) qs.set("service", options.service);
93+
if (historyFilter?.page != null) qs.set("page", String(historyFilter.page));
94+
if (historyFilter?.limit != null) qs.set("limit", String(historyFilter.limit));
95+
if (historyFilter?.service) qs.set("service", historyFilter.service);
6996
const query = qs.toString();
7097
return request(
7198
"GET",
72-
url(`/v2/history${query ? `?${query}` : ""}`),
99+
buildUrl(`/v2/history${query ? `?${query}` : ""}`),
73100
key,
74101
undefined,
75-
opts(ro),
102+
mergeRequestOptions(requestOptions),
76103
);
77104
},
78105

79106
crawl: {
80-
async start(targetUrl: string, options?: CrawlOptions, ro?: RequestOptions) {
81-
return request("POST", url("/v2/crawl"), key, { url: targetUrl, ...options }, opts(ro));
107+
async start(url: string, crawlOptions?: ApiCrawlOptions, requestOptions?: RequestOptions) {
108+
return request(
109+
"POST",
110+
buildUrl("/v2/crawl"),
111+
key,
112+
{ url, ...crawlOptions },
113+
mergeRequestOptions(requestOptions),
114+
);
82115
},
83116

84-
async status(id: string, ro?: RequestOptions) {
85-
return request("GET", url(`/v2/crawl/${id}`), key, undefined, opts(ro));
117+
async status(id: string, requestOptions?: RequestOptions) {
118+
return request(
119+
"GET",
120+
buildUrl(`/v2/crawl/${id}`),
121+
key,
122+
undefined,
123+
mergeRequestOptions(requestOptions),
124+
);
86125
},
87126

88-
async stop(id: string, ro?: RequestOptions) {
89-
return request("POST", url(`/v2/crawl/${id}/stop`), key, {}, opts(ro));
127+
async stop(id: string, requestOptions?: RequestOptions) {
128+
return request(
129+
"POST",
130+
buildUrl(`/v2/crawl/${id}/stop`),
131+
key,
132+
{},
133+
mergeRequestOptions(requestOptions),
134+
);
90135
},
91136

92-
async resume(id: string, ro?: RequestOptions) {
93-
return request("POST", url(`/v2/crawl/${id}/resume`), key, {}, opts(ro));
137+
async resume(id: string, requestOptions?: RequestOptions) {
138+
return request(
139+
"POST",
140+
buildUrl(`/v2/crawl/${id}/resume`),
141+
key,
142+
{},
143+
mergeRequestOptions(requestOptions),
144+
);
94145
},
95146
},
96147

97148
monitor: {
98-
async create(options: MonitorCreateOptions, ro?: RequestOptions) {
99-
return request("POST", url("/v2/monitor"), key, { ...options }, opts(ro));
149+
async create(monitorCreateInput: ApiMonitorCreateInput, requestOptions?: RequestOptions) {
150+
return request(
151+
"POST",
152+
buildUrl("/v2/monitor"),
153+
key,
154+
{ ...monitorCreateInput },
155+
mergeRequestOptions(requestOptions),
156+
);
100157
},
101158

102-
async list(ro?: RequestOptions) {
103-
return request("GET", url("/v2/monitor"), key, undefined, opts(ro));
159+
async list(requestOptions?: RequestOptions) {
160+
return request(
161+
"GET",
162+
buildUrl("/v2/monitor"),
163+
key,
164+
undefined,
165+
mergeRequestOptions(requestOptions),
166+
);
104167
},
105168

106-
async get(id: string, ro?: RequestOptions) {
107-
return request("GET", url(`/v2/monitor/${id}`), key, undefined, opts(ro));
169+
async get(id: string, requestOptions?: RequestOptions) {
170+
return request(
171+
"GET",
172+
buildUrl(`/v2/monitor/${id}`),
173+
key,
174+
undefined,
175+
mergeRequestOptions(requestOptions),
176+
);
108177
},
109178

110-
async pause(id: string, ro?: RequestOptions) {
111-
return request("POST", url(`/v2/monitor/${id}/pause`), key, {}, opts(ro));
179+
async pause(id: string, requestOptions?: RequestOptions) {
180+
return request(
181+
"POST",
182+
buildUrl(`/v2/monitor/${id}/pause`),
183+
key,
184+
{},
185+
mergeRequestOptions(requestOptions),
186+
);
112187
},
113188

114-
async resume(id: string, ro?: RequestOptions) {
115-
return request("POST", url(`/v2/monitor/${id}/resume`), key, {}, opts(ro));
189+
async resume(id: string, requestOptions?: RequestOptions) {
190+
return request(
191+
"POST",
192+
buildUrl(`/v2/monitor/${id}/resume`),
193+
key,
194+
{},
195+
mergeRequestOptions(requestOptions),
196+
);
116197
},
117198

118-
async delete(id: string, ro?: RequestOptions) {
119-
return request("DELETE", url(`/v2/monitor/${id}`), key, undefined, opts(ro));
199+
async delete(id: string, requestOptions?: RequestOptions) {
200+
return request(
201+
"DELETE",
202+
buildUrl(`/v2/monitor/${id}`),
203+
key,
204+
undefined,
205+
mergeRequestOptions(requestOptions),
206+
);
120207
},
121208
},
122209
};

‎src/http.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,18 @@ import { VERSION } from "./types/index.js";
33

44
export interface ApiResult<T> {
55
data: T;
6-
_requestId: string;
6+
requestId: string;
77
}
88

99
export async function request<T = unknown>(
1010
method: "GET" | "POST" | "DELETE",
1111
url: string,
1212
apiKey: string,
1313
body?: Record<string, unknown>,
14-
opts?: { maxRetries?: number; timeout?: number } & RequestOptions,
14+
requestOptions?: { maxRetries?: number; timeout?: number } & RequestOptions,
1515
): Promise<ApiResult<T>> {
16-
const maxRetries = opts?.maxRetries ?? 2;
17-
const timeout = opts?.timeout ?? 30_000;
16+
const maxRetries = requestOptions?.maxRetries ?? 2;
17+
const timeout = requestOptions?.timeout ?? 30_000;
1818

1919
let lastError: Error | null = null;
2020

@@ -30,7 +30,7 @@ export async function request<T = unknown>(
3030
method,
3131
headers,
3232
body: body ? JSON.stringify(body) : undefined,
33-
signal: opts?.signal ?? AbortSignal.timeout(timeout),
33+
signal: requestOptions?.signal ?? AbortSignal.timeout(timeout),
3434
});
3535

3636
if ((res.status === 502 || res.status === 503) && attempt < maxRetries) {
@@ -46,7 +46,7 @@ export async function request<T = unknown>(
4646
const data = (method === "DELETE" ? null : await res.json()) as T;
4747
const requestId = res.headers.get("x-request-id") ?? "";
4848

49-
return { data, _requestId: requestId };
49+
return { data, requestId };
5050
} catch (e) {
5151
lastError = e instanceof Error ? e : new Error(String(e));
5252
if (e instanceof DOMException && e.name === "TimeoutError") throw lastError;

‎src/index.ts‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,15 @@
11
export { scrapegraphai } from "./client.js";
22
export type { ApiResult } from "./http.js";
33
export type {
4+
ApiCrawlOptions,
5+
ApiExtractOptions,
6+
ApiFetchConfig,
7+
ApiGenerateSchemaOptions,
8+
ApiHistoryFilterInput,
9+
ApiMonitorCreateInput,
10+
ApiScrapeOptions,
11+
ApiSearchOptions,
412
ClientConfig,
5-
CrawlOptions,
6-
MonitorCreateOptions,
713
RequestOptions,
814
} from "./types/index.js";
915
export { DEFAULT_BASE_URL, VERSION } from "./types/index.js";

0 commit comments

Comments
 (0)