Parse a website
curl --request POST \
--url https://your-instance.example.com/api/parse-website \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"websiteUrl": "https://docs.example.com",
"maxPages": 50
}
'import requests
url = "https://your-instance.example.com/api/parse-website"
payload = {
"websiteUrl": "https://docs.example.com",
"maxPages": 50
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({websiteUrl: 'https://docs.example.com', maxPages: 50})
};
fetch('https://your-instance.example.com/api/parse-website', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://your-instance.example.com/api/parse-website",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'websiteUrl' => 'https://docs.example.com',
'maxPages' => 50
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://your-instance.example.com/api/parse-website"
payload := strings.NewReader("{\n \"websiteUrl\": \"https://docs.example.com\",\n \"maxPages\": 50\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://your-instance.example.com/api/parse-website")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"websiteUrl\": \"https://docs.example.com\",\n \"maxPages\": 50\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://your-instance.example.com/api/parse-website")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"websiteUrl\": \"https://docs.example.com\",\n \"maxPages\": 50\n}"
response = http.request(request)
puts response.read_body{
"message": "Parse queued",
"project": "/your-org/your-repo",
"queueId": 5,
"position": 1
}{
"error": "repoUrl is required"
}{
"error": "Authentication required"
}{
"error": "An unexpected error occurred"
}Parse
Parse a website
Crawl and index a public website starting from the given URL
POST
/
parse-website
Parse a website
curl --request POST \
--url https://your-instance.example.com/api/parse-website \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"websiteUrl": "https://docs.example.com",
"maxPages": 50
}
'import requests
url = "https://your-instance.example.com/api/parse-website"
payload = {
"websiteUrl": "https://docs.example.com",
"maxPages": 50
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({websiteUrl: 'https://docs.example.com', maxPages: 50})
};
fetch('https://your-instance.example.com/api/parse-website', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://your-instance.example.com/api/parse-website",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'websiteUrl' => 'https://docs.example.com',
'maxPages' => 50
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://your-instance.example.com/api/parse-website"
payload := strings.NewReader("{\n \"websiteUrl\": \"https://docs.example.com\",\n \"maxPages\": 50\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://your-instance.example.com/api/parse-website")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"websiteUrl\": \"https://docs.example.com\",\n \"maxPages\": 50\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://your-instance.example.com/api/parse-website")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"websiteUrl\": \"https://docs.example.com\",\n \"maxPages\": 50\n}"
response = http.request(request)
puts response.read_body{
"message": "Parse queued",
"project": "/your-org/your-repo",
"queueId": 5,
"position": 1
}{
"error": "repoUrl is required"
}{
"error": "Authentication required"
}{
"error": "An unexpected error occurred"
}Authorizations
API key generated from Personal Settings. See Authentication.
Body
application/json
Root URL to start crawling from
Display name for the project
Short description shown in the library list
Maximum number of pages to crawl
Maximum link depth from the root URL
URL path patterns to skip (e.g., /blog/*)
Re-crawl even if already indexed
Response
Parse job accepted and queued
Example:
"Parse queued"
Project identifier assigned to this library (e.g. /your-org/your-repo)
Example:
"/your-org/your-repo"
Numeric ID for this parse job
Example:
5
Position in the queue. null if the job started immediately
Example:
1
Was this page helpful?
⌘I