feat: add wait_ms parameter to scrape, smartscraper, and markdownify endpoints

Adds a new `waitMs` option (sent as `wait_ms` in the API payload) to control
how long the scraping provider waits before capturing page content. Defaults
to 3000ms on the server side. Aligns with sgai-api PR #399.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: VinciGit00

package.json

@@ -1,7 +1,7 @@
 {
   "name": "scrapegraph-js",
   "author": "ScrapeGraphAI",
-  "version": "0.2.111",
+  "version": "0.2.112",
   "description": "Scrape and extract structured data from a webpage using ScrapeGraphAI's APIs. Supports cookies for authentication, infinite scrolling, and pagination.",
   "repository": {
     "type": "git",

src/markdownify.js

@@ -11,11 +11,12 @@ import { getMockResponse } from './utils/mockResponse.js';
  * @param {Object} options - Optional configuration options.
  * @param {boolean} options.mock - Override mock mode for this request
  * @param {boolean} [options.stealth=false] - Enable stealth mode to avoid bot detection
+ * @param {number} [options.waitMs] - Number of milliseconds to wait before scraping the website (default: 3000)
  * @returns {Promise<string>} A promise that resolves to the markdown representation of the webpage.
  * @throws {Error} Throws an error if the HTTP request fails.
  */
 export async function markdownify(apiKey, url, options = {}) {
-  const { mock = null, stealth = false } = options;
+  const { mock = null, stealth = false, waitMs = null } = options;
 
   // Check if mock mode is enabled
   const useMock = mock !== null ? mock : isMockEnabled();
@@ -41,6 +42,13 @@ export async function markdownify(apiKey, url, options = {}) {
     payload.stealth = stealth;
   }
 
+  if (waitMs !== null) {
+    if (!Number.isInteger(waitMs) || waitMs < 0) {
+      throw new Error('waitMs must be a positive integer');
+    }
+    payload.wait_ms = waitMs;
+  }
+
   try {
     const response = await axios.post(endpoint, payload, { headers });
     return response.data;

src/scrape.js

@@ -13,6 +13,7 @@ import { getMockResponse, createMockAxiosResponse } from './utils/mockResponse.j
  * @param {boolean} [options.branding=false] - Whether to include branding in the response (defaults to false).
  * @param {Object} options.headers - Optional custom headers to send with the request.
  * @param {boolean} [options.stealth=false] - Enable stealth mode to avoid bot detection
+ * @param {number} [options.waitMs] - Number of milliseconds to wait before scraping the website (default: 3000)
  * @returns {Promise<Object>} A promise that resolves to the HTML content and metadata.
  * @throws {Error} Throws an error if the HTTP request fails.
  *
@@ -57,7 +58,8 @@ export async function scrape(apiKey, url, options = {}) {
     branding = false,
     headers: customHeaders = {},
     mock = null,
-    stealth = false
+    stealth = false,
+    waitMs = null
   } = options;
 
   // Check if mock mode is enabled
@@ -91,6 +93,13 @@ export async function scrape(apiKey, url, options = {}) {
     payload.stealth = stealth;
   }
 
+  if (waitMs !== null) {
+    if (!Number.isInteger(waitMs) || waitMs < 0) {
+      throw new Error('waitMs must be a positive integer');
+    }
+    payload.wait_ms = waitMs;
+  }
+
   // Only include headers in payload if they are provided
   if (Object.keys(customHeaders).length > 0) {
     payload.headers = customHeaders;

src/smartScraper.js

@@ -26,10 +26,11 @@ import { getMockResponse, createMockAxiosResponse } from './utils/mockResponse.j
  * @param {boolean} [stealth] - Optional flag to enable stealth mode to avoid bot detection
  * @param {string} [websiteHtml] - Optional raw HTML content to process (max 2MB, mutually exclusive with url and websiteMarkdown)
  * @param {string} [websiteMarkdown] - Optional Markdown content to process (max 2MB, mutually exclusive with url and websiteHtml)
+ * @param {number} [waitMs] - Optional number of milliseconds to wait before scraping the website (default: 3000)
  * @returns {Promise<string>} Extracted data in JSON format matching the provided schema
  * @throws - Will throw an error in case of an HTTP failure or validation error.
  */
-export async function smartScraper(apiKey, url, prompt, schema = null, numberOfScrolls = null, totalPages = null, cookies = null, options = {}, plain_text = false, renderHeavyJs = false, stealth = false, websiteHtml = null, websiteMarkdown = null) {
+export async function smartScraper(apiKey, url, prompt, schema = null, numberOfScrolls = null, totalPages = null, cookies = null, options = {}, plain_text = false, renderHeavyJs = false, stealth = false, websiteHtml = null, websiteMarkdown = null, waitMs = null) {
   const { mock = null } = options;
 
   // Validate that exactly one of url, websiteHtml, or websiteMarkdown is provided
@@ -123,6 +124,13 @@ export async function smartScraper(apiKey, url, prompt, schema = null, numberOfS
     payload.stealth = stealth;
   }
 
+  if (waitMs !== null) {
+    if (!Number.isInteger(waitMs) || waitMs < 0) {
+      throw new Error('waitMs must be a positive integer');
+    }
+    payload.wait_ms = waitMs;
+  }
+
   try {
     const response = await axios.post(endpoint, payload, { headers });
     return response.data;