Crawl

Start a crawl job

post

Authorizations

Body

urlstringrequired

maxPagesinteger · min: 1optional

followLinksbooleanoptionalDefault: true

ignoreSitemapbooleanoptionalDefault: false

excludePatternsstring[]optional

includePatternsstring[]optional

sessionOptionsobjectoptionalDefault: {"useStealth":false,"useProxy":false,"acceptCookies":false}

scrapeOptionsobjectoptional

Responses

application/json

objectoptional

application/json

application/json

post

HTTPcURLJavaScriptPython

POST /api/crawl HTTP/1.1
Host: app.hyperbrowser.ai
x-api-key: YOUR_API_KEY
Content-Type: application/json
Accept: */*
Content-Length: 977

{
  "url": "text",
  "maxPages": 1,
  "followLinks": true,
  "ignoreSitemap": true,
  "excludePatterns": [
    "text"
  ],
  "includePatterns": [
    "text"
  ],
  "sessionOptions": {
    "useStealth": true,
    "useProxy": true,
    "proxyServer": "text",
    "proxyServerPassword": "text",
    "proxyServerUsername": "text",
    "proxyCountry": "US",
    "proxyState": "AL",
    "proxyCity": "new york",
    "operatingSystems": [
      "windows"
    ],
    "device": [
      "desktop"
    ],
    "platform": [
      "chrome"
    ],
    "locales": [
      "en"
    ],
    "screen": {
      "width": 1,
      "height": 1
    },
    "solveCaptchas": true,
    "adblock": true,
    "trackers": true,
    "annoyances": true,
    "enableWebRecording": true,
    "profile": {
      "id": "text",
      "persistChanges": true
    },
    "acceptCookies": true,
    "extensionIds": [
      "123e4567-e89b-12d3-a456-426614174000"
    ],
    "urlBlocklist": [
      "text"
    ],
    "browserArgs": [
      "text"
    ],
    "imageCaptchaParams": [
      {
        "imageSelector": "text",
        "inputSelector": "text"
      }
    ]
  },
  "scrapeOptions": {
    "formats": [
      "html"
    ],
    "includeTags": [
      "text"
    ],
    "excludeTags": [
      "text"
    ],
    "onlyMainContent": true,
    "waitFor": 1,
    "timeout": 1,
    "waitUntil": "load",
    "screenshotOptions": {
      "fullPage": true,
      "format": "jpeg"
    }
  }
}

{
  "jobId": "text"
}

Get crawl job status and results

get

Authorizations

Path parameters

idstringrequired

Query parameters

pageintegeroptional

batchSizeinteger · min: 1optional

Responses

application/json

objectoptional

application/json

application/json

get

HTTPcURLJavaScriptPython

GET /api/crawl/{id} HTTP/1.1
Host: app.hyperbrowser.ai
x-api-key: YOUR_API_KEY
Accept: */*

{
  "jobId": "123e4567-e89b-12d3-a456-426614174000",
  "status": "pending",
  "error": "text",
  "totalCrawledPages": 1,
  "totalPageBatches": 1,
  "currentPageBatch": 1,
  "batchSize": 1,
  "data": [
    {
      "url": "text",
      "status": "completed",
      "error": "text",
      "metadata": {
        "ANY_ADDITIONAL_PROPERTY": "text"
      },
      "markdown": "text",
      "html": "text",
      "links": [
        "text"
      ],
      "screenshot": "text"
    }
  ]
}