[dependencies]
spider = { version = "2", features = ["spider_cloud"] }

use spider::configuration::{SpiderCloudConfig, SpiderCloudMode};
use spider::website::Website;

let cloud = SpiderCloudConfig::new("sk-...")
    .with_mode(SpiderCloudMode::Smart); // proxy by default, auto-unblock when blocked

let mut website = Website::new("https://example.com")
    .with_spider_cloud_config(cloud)
    .build()?;

[dependencies]
spider = "2"

use spider::{tokio, website::Website};

#[tokio::main]
async fn main() {
    let mut website = Website::new("https://example.com");
    let mut rx = website.subscribe(16);

    tokio::spawn(async move {
        while let Ok(page) = rx.recv().await {
            println!("{}  {}", page.status_code, page.get_url());
        }
    });

    website.crawl().await;
    website.unsubscribe();
}

let mut website = Website::new("https://example.com")
    .with_limit(50)                    // concurrent requests
    .with_depth(10)                    // how deep to follow links
    .with_delay(500)                   // pause between requests (ms)
    .with_respect_robots_txt(true)
    .with_subdomains(true)
    .with_user_agent(Some("MyBot/1.0"))
    .with_stealth(true)
    .build()
    .unwrap();

cargo test -p spider                  # unit tests
RUN_LIVE_TESTS=1 cargo test           # live network tests