RUST: Web scraping template

// Cargo.toml

[package]
name = "web-scraper"
version = "0.1.0"
edition = "2024"

[dependencies]
reqwest = {version = "0.12"}
tokio = {version = "1.37.0", features = ["full"]}
scraper = "0.19.0"

// main.rs

// Web Scraping using rust
// Extract and print the URLs from a test link

use reqwest::Client;

#[tokio::main]

async fn main() {
    let client = Client::new();

    let response = client
        .get("https://scrapeme.live/shop/")
        .send()
        .await
        .unwrap();

    let html_content = response.text().await.unwrap();

    let document = scraper::Html::parse_document(&html_content);

    let html_product_selector = scraper::Selector::parse("li.product").unwrap();

    let html_products = document.select(&html_product_selector);

    for product in html_products {
        let url = product
            .select(&scraper::Selector::parse("a").unwrap())
            .next()
            .and_then(|a| a.value().attr("href"))
            .map(str::to_owned);

        println!("url = {:?}", url);
    }
}