Scrape Web Page by URL

{
  "status": "Processing",
  "message": "Source processed successfully",
  "file_name": "https://example.com/",
  "file_size": 0,
  "file_type": "",
  "file_source": "url",
  "project_id": "550e8400-e29b-41d4-a716-446655440000",
  "project_name": "My Project",
  "partition_method": "basic"
}

import fetch from 'node-fetch';

const uploadUrlSource = async (apiToken, url, crawlUrls = false) => {
  const response = await fetch('https://sources.graphorlm.com/upload-url-source', {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${apiToken}`,
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({ url, crawlUrls })
  });

  if (!response.ok) {
    throw new Error(`Upload from URL failed: ${response.status} ${response.statusText}`);
  }

  const result = await response.json();
  console.log('URL upload accepted:', result);
  return result;
};

// Usage (scrapes the page content)
uploadUrlSource('grlm_your_api_token_here', 'https://example.com/');

import requests

def upload_url_source(api_token, url, crawl_urls=False):
    endpoint = "https://sources.graphorlm.com/upload-url-source"
    headers = {"Authorization": f"Bearer {api_token}", "Content-Type": "application/json"}
    payload = {"url": url, "crawlUrls": crawl_urls}

    response = requests.post(endpoint, headers=headers, json=payload, timeout=300)
    response.raise_for_status()
    return response.json()

# Usage (scrapes the page content)
result = upload_url_source("grlm_your_api_token_here", "https://example.com")
print("URL scraping accepted:", result["file_name"])  # typically echoes the URL

curl -X POST https://sources.graphorlm.com/upload-url-source \
  -H "Authorization: Bearer grlm_your_api_token_here" \
  -H "Content-Type: application/json" \
  -d '{"url":"https://example.com/","crawlUrls":false}'

{
  "detail": "Invalid input: URL is required"
}