Code shared from the Rust Playground

playground.rs

extern crate futures;
extern crate reqwest;
extern crate scraper;

use futures::{future, Future};
use scraper::{Html, Selector};
use std::time::Instant;

async fn get_doc(url: &str) -> Result<String, Box<dyn std::error::Error>> {
    let doc = reqwest::get(url).await?.text().await?;

    Ok(doc)
}

fn parse_links(doc: &str) -> Vec<String> {
    Html::parse_document(doc)
        .select(&Selector::parse("a").unwrap())
        .map(|elm| {
            if let Some(url) = elm.value().attr("href") {
                String::from(url)
            } else {
                String::new()
            }
        })
        .filter(|s| s.starts_with("http"))
        .collect()
}

async fn get_links(link: &str) -> Vec<String> {
    match get_doc(link).await {
        Ok(doc) => parse_links(&doc),
        Err(_) => vec![],
    }
}

#[tokio::main]
async fn main() {
    let mut links: Vec<_> = vec![String::from("http://jbadavis.github.io")];
    let mut link_store = links.clone();

    println!("Starting at {:?}\n", links[0]);

    for _i in 0..2 {
        let futures: Vec<_> = links.iter().map(|link| get_links(&link)).collect();

        let f = future::join_all(futures);

        f.map(|x| println!("{}", x));
    }

    println!("{:?}", link_store.len());
}