Web Data Infrastructure for AI

1// npm i olostep
2import Olostep from 'olostep'
3
4const client = new Olostep({ apiKey: 'YOUR_REAL_KEY' })
5
6const crawl = await client.crawls.create({
7  url: 'https://olostep.com',
8  maxPages: 100,
9  includeUrls: ['/**'],
10  excludeUrls: ['/collections/**'],
11  includeExternal: false,
12})
13
14console.log(crawl.id, crawl.status)
15
16// Wait for completion and iterate pages
17for await (const page of crawl.pages()) {
18  console.log(page.url)
19  const content = await client.retrieve({ retrieveId: page.retrieve_id, formats: ['markdown'] })
20  console.log(content.markdown_content.slice(0, 200))
21}

1# Start crawl
2curl -s -X POST "https://api.olostep.com/v1/crawls" \
3  -H "Authorization: Bearer <YOUR_API_KEY>" \
4  -H "Content-Type: application/json" \
5  -d '{
6    "start_url": "https://olostep.com",
7    "max_pages": 100,
8    "include_urls": ["/**"],
9    "exclude_urls": ["/collections/**"],
10    "include_external": false
11  }'
12
13# Check status (replace <CRAWL_ID>)
14curl -s "https://api.olostep.com/v1/crawls/<CRAWL_ID>" \
15  -H "Authorization: Bearer <YOUR_API_KEY>"
16
17# Get pages (replace <CRAWL_ID>)
18curl -s "https://api.olostep.com/v1/crawls/<CRAWL_ID>/pages" \
19  -H "Authorization: Bearer <YOUR_API_KEY>"
20
21# Retrieve content (replace <RETRIEVE_ID>)
22curl -s -G "https://api.olostep.com/v1/retrieve" \
23  -H "Authorization: Bearer <YOUR_API_KEY>" \
24  --data-urlencode "retrieve_id=<RETRIEVE_ID>" \
25  --data-urlencode "formats=markdown"

1// npm i olostep
2import Olostep from 'olostep'
3
4const client = new Olostep({ apiKey: 'YOUR_REAL_KEY' })
5
6const map = await client.maps.create({
7  url: 'https://docs.olostep.com',
8  includeUrls: ['/features/**'],
9  topN: 100,
10})
11
12console.log(`Map ID: ${map.id}`)
13
14// Iterate all URLs (handles pagination automatically)
15for await (const url of map.urls()) {
16  console.log(url)
17}

1curl -s -X POST "https://api.olostep.com/v1/maps" \
2  -H "Authorization: Bearer <YOUR_API_KEY>" \
3  -H "Content-Type: application/json" \
4  -d '{
5    "url": "https://docs.olostep.com",
6    "include_urls": ["/features/**"],
7    "top_n": 100
8  }'

1// npm i olostep
2import Olostep from 'olostep'
3
4const client = new Olostep({ apiKey: 'YOUR_REAL_KEY' })
5
6const batch = await client.batches.create([
7  { url: 'https://www.google.com/search?q=stripe&gl=us&hl=en', customId: 'item-1' },
8  { url: 'https://www.google.com/search?q=paddle&gl=us&hl=en', customId: 'item-2' },
9], {
10  parser: '@olostep/google-search',
11})
12
13console.log(batch.id, batch.total_urls)
14
15// Wait and iterate results (auto-waits for completion)
16for await (const item of batch.items()) {
17  const content = await item.retrieve(['json'])
18  console.log(item.url, item.custom_id)
19  console.log(content.json_content)
20}

1curl -s -X POST "https://api.olostep.com/v1/batches" \
2  -H "Authorization: Bearer <YOUR_API_KEY>" \
3  -H "Content-Type: application/json" \
4  -d '{
5    "items": [
6      {"custom_id": "item-1", "url": "https://www.google.com/search?q=stripe&gl=us&hl=en"},
7      {"custom_id": "item-2", "url": "https://www.google.com/search?q=paddle&gl=us&hl=en"}
8    ],
9    "parser": {"id": "@olostep/google-search"}
10  }'

1// npm i olostep
2import Olostep from 'olostep'
3
4const client = new Olostep({ apiKey: 'YOUR_REAL_KEY' })
5
6const search = await client.searches.create('Latest updates with SpaceX')
7
8console.log(search.id, search.links.length)

1curl -s -X POST "https://api.olostep.com/v1/searches" \
2  -H "Authorization: Bearer <YOUR_API_KEY>" \
3  -H "Content-Type: application/json" \
4  -d '{
5    "query": "Latest updates with SpaceX"
6  }'

1const API_URL = 'https://api.olostep.com/v1'
2const headers = { 
3  'Authorization': 'Bearer <YOUR_API_KEY>', 
4  'Content-Type': 'application/json' 
5}
6
7// Create a monitor
8const res = await fetch(`${API_URL}/monitors`, {
9  method: 'POST',
10  headers,
11  body: JSON.stringify({
12    query: 'Alert me when Tesla stock price is above $500',
13    frequency: 'every hour',
14    email: 'alerts@example.com'
15  })
16})
17
18const monitor = await res.json()
19console.log(`Monitor created: ${monitor.id}`)
20console.log(`Status: ${monitor.status}`)
21
22// List all monitors
23const monitors = await fetch(`${API_URL}/monitors`, { headers }).then(r => r.json())
24monitors.monitors.forEach(m => console.log(`${m.id}: ${m.url} (${m.frequency})`))
25
26// Get monitor details
27const details = await fetch(`${API_URL}/monitors/${monitor.id}`, { headers }).then(r => r.json())
28console.log(details)
29
30// Delete a monitor
31await fetch(`${API_URL}/monitors/${monitor.id}`, { method: 'DELETE', headers })
32console.log(`Monitor ${monitor.id} deleted`)

1# Create a monitor
2curl -s -X POST "https://api.olostep.com/v1/monitors" \
3  -H "Authorization: Bearer <YOUR_API_KEY>" \
4  -H "Content-Type: application/json" \
5  -d '{
6    "query": "Track changes in product pricing and stock information",
7    "url": "https://example.com/products/widget-pro",
8    "frequency": "daily",
9    "email": "alerts@example.com"
10  }'
11
12# List all monitors
13curl -s "https://api.olostep.com/v1/monitors" \
14  -H "Authorization: Bearer <YOUR_API_KEY>"
15
16# Get monitor details (replace <MONITOR_ID>)
17curl -s "https://api.olostep.com/v1/monitors/<MONITOR_ID>" \
18  -H "Authorization: Bearer <YOUR_API_KEY>"
19
20# Delete a monitor (replace <MONITOR_ID>)
21curl -s -X DELETE "https://api.olostep.com/v1/monitors/<MONITOR_ID>" \
22  -H "Authorization: Bearer <YOUR_API_KEY>"