ChristianIvicevic/crawl.rs Secret

## crawl.rs
use std::collections::HashMap;

use s3::{creds::Credentials, Bucket, Region};
use scraper::{Html, Selector};
use serde::Serialize;
use vercel_runtime::{run, Body, Error, Request, Response, StatusCode};

const WOTC_URL: &str = "https://magic.wizards.com/en/rules";

#[derive(Serialize)]
struct Embed {
    title: Option<String>,
    description: Option<String>,
    color: Option<String>,
    url: Option<String>,
}

#[tokio::main]
async fn main() -> Result<(), Error> {
    dotenv::dotenv().ok();

    tracing_subscriber::fmt()
        .with_max_level(tracing::Level::INFO)
        .with_target(false)
        .init();

    run(handler).await
}

pub async fn handler(request: Request) -> Result<Response<Body>, Error> {
    let auth_bearer_token = format!("Bearer {}", std::env::var("CRON_SECRET")?);
    let header = request.headers().get("authorization");
    if header.is_none() || header.unwrap().to_str().unwrap() != auth_bearer_token {
        return Ok(Response::builder()
            .status(StatusCode::UNAUTHORIZED)
            .body(().into())?);
    }

    let http_response = reqwest::get(WOTC_URL).await?.text().await?;
    let document = Html::parse_document(&http_response);
    let anchors = document
        .select(&Selector::parse(r#"a[href$=".txt"]"#).unwrap())
        .filter_map(|it| it.attr("href"))
        .collect::<Vec<_>>();

    // We're assuming there is only a single .txt file linked
    let href = anchors.first().unwrap();
    let file_name = href.split('/').last().unwrap().replace(' ', "");

    let bucket = Bucket::new(
        &std::env::var("R2_BUCKET_NAME")?,
        Region::R2 {
            account_id: std::env::var("R2_ACCOUNT_ID")?,
        },
        Credentials::new(
            Some(&std::env::var("R2_ACCESS_KEY_ID")?),
            Some(&std::env::var("R2_SECRET_ACCESS_KEY")?),
            None,
            None,
            None,
        )?,
    )?;

    if bucket.head_object(&file_name).await.is_ok() {
        tracing::info!(
            "Bucket already contains file with name `{}`. Nothing to upload.",
            &file_name
        );
        return Ok(Response::builder().status(StatusCode::OK).body(().into())?);
    }

    let file_content = reqwest::get(*href).await?.text().await?;
    bucket.put_object(&file_name, file_content.as_ref()).await?;

    if let Ok(webhook_url) = std::env::var("DISCORD_WEBHOOK_URL") {
        reqwest::Client::new()
            .post(webhook_url)
            .json(&HashMap::from([(
                "embeds",
                [Embed {
                    title: Some("MTG Comprehensive Rules have been updated".to_string()),
                    description: Some(format!(
                        "The new rulings file `{}` has been successfully uploaded to the bucket.",
                        file_name
                    )),
                    color: None,
                    // For some reason WOTC uses spaces in their filenames and we need to manually escape them...
                    url: Some(href.replace(' ', "%20").to_string()),
                }],
            )]))
            .send()
            .await?;
    } else {
        tracing::warn!("The DISCORD_WEBHOOK_URL environment variable is missing.");
    }

    Ok(Response::builder().status(StatusCode::OK).body(().into())?)
}

## crawl.ts
import * as cheerio from 'cheerio'
import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'
import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
import { configDotenv } from 'dotenv'

const r2 = new S3Client({
  region: 'auto',
  endpoint: process.env.R2_ENDPOINT,
  credentials: {
    accessKeyId: process.env.R2_ACCESS_KEY_ID ?? '',
    secretAccessKey: process.env.R2_SECRET_ACCESS_KEY ?? ''
  }
})

async function main() {
  configDotenv()

  const httpResponse = await fetch('https://magic.wizards.com/en/rules')
  const httpContent = await httpResponse.text()

  const $ = cheerio.load(httpContent)

  const link = $('a')
    .filter((_, el) => $(el).attr('href')?.endsWith('.txt') ?? false)
    .attr('href')
  if (!link) return

  const fileName = link.split('/').at(-1)?.replace(' ', '-')
  if (!fileName) return

  const signedUrl = await getSignedUrl(
    r2,
    new PutObjectCommand({
      Bucket: process.env.R2_BUCKET_NAME,
      Key: fileName
    }),
    { expiresIn: 60 }
  )

  const fileResponse = await fetch(link)
  const fileContent = await fileResponse.text()

  await fetch(signedUrl, {
    method: 'PUT',
    body: fileContent
  })

  if (process.env.DISCORD_WEBHOOK_URL) {
    await fetch(process.env.DISCORD_WEBHOOK_URL, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({
        embeds: [
          {
            title: 'MTG Comprehensive Rules have been updated',
            description: `The new rulings file has been successfully uploaded to \`${fileName}\`.`,
            color: null,
            url: link
          }
        ]
      })
    })
  }
}

void main()
	use std::collections::HashMap;

	use s3::{creds::Credentials, Bucket, Region};
	use scraper::{Html, Selector};
	use serde::Serialize;
	use vercel_runtime::{run, Body, Error, Request, Response, StatusCode};

	const WOTC_URL: &str = "https://magic.wizards.com/en/rules";

	#[derive(Serialize)]
	struct Embed {
	title: Option<String>,
	description: Option<String>,
	color: Option<String>,
	url: Option<String>,
	}

	#[tokio::main]
	async fn main() -> Result<(), Error> {
	dotenv::dotenv().ok();

	tracing_subscriber::fmt()
	.with_max_level(tracing::Level::INFO)
	.with_target(false)
	.init();

	run(handler).await
	}

	pub async fn handler(request: Request) -> Result<Response<Body>, Error> {
	let auth_bearer_token = format!("Bearer {}", std::env::var("CRON_SECRET")?);
	let header = request.headers().get("authorization");
	if header.is_none() \|\| header.unwrap().to_str().unwrap() != auth_bearer_token {
	return Ok(Response::builder()
	.status(StatusCode::UNAUTHORIZED)
	.body(().into())?);
	}

	let http_response = reqwest::get(WOTC_URL).await?.text().await?;
	let document = Html::parse_document(&http_response);
	let anchors = document
	.select(&Selector::parse(r#"a[href$=".txt"]"#).unwrap())
	.filter_map(\|it\| it.attr("href"))
	.collect::<Vec<_>>();

	// We're assuming there is only a single .txt file linked
	let href = anchors.first().unwrap();
	let file_name = href.split('/').last().unwrap().replace(' ', "");

	let bucket = Bucket::new(
	&std::env::var("R2_BUCKET_NAME")?,
	Region::R2 {
	account_id: std::env::var("R2_ACCOUNT_ID")?,
	},
	Credentials::new(
	Some(&std::env::var("R2_ACCESS_KEY_ID")?),
	Some(&std::env::var("R2_SECRET_ACCESS_KEY")?),
	None,
	None,
	None,
	)?,
	)?;

	if bucket.head_object(&file_name).await.is_ok() {
	tracing::info!(
	"Bucket already contains file with name `{}`. Nothing to upload.",
	&file_name
	);
	return Ok(Response::builder().status(StatusCode::OK).body(().into())?);
	}

	let file_content = reqwest::get(*href).await?.text().await?;
	bucket.put_object(&file_name, file_content.as_ref()).await?;

	if let Ok(webhook_url) = std::env::var("DISCORD_WEBHOOK_URL") {
	reqwest::Client::new()
	.post(webhook_url)
	.json(&HashMap::from([(
	"embeds",
	[Embed {
	title: Some("MTG Comprehensive Rules have been updated".to_string()),
	description: Some(format!(
	"The new rulings file `{}` has been successfully uploaded to the bucket.",
	file_name
	)),
	color: None,
	// For some reason WOTC uses spaces in their filenames and we need to manually escape them...
	url: Some(href.replace(' ', "%20").to_string()),
	}],
	)]))
	.send()
	.await?;
	} else {
	tracing::warn!("The DISCORD_WEBHOOK_URL environment variable is missing.");
	}

	Ok(Response::builder().status(StatusCode::OK).body(().into())?)
	}
	import * as cheerio from 'cheerio'
	import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'
	import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
	import { configDotenv } from 'dotenv'

	const r2 = new S3Client({
	region: 'auto',
	endpoint: process.env.R2_ENDPOINT,
	credentials: {
	accessKeyId: process.env.R2_ACCESS_KEY_ID ?? '',
	secretAccessKey: process.env.R2_SECRET_ACCESS_KEY ?? ''
	}
	})

	async function main() {
	configDotenv()

	const httpResponse = await fetch('https://magic.wizards.com/en/rules')
	const httpContent = await httpResponse.text()

	const $ = cheerio.load(httpContent)

	const link = $('a')
	.filter((_, el) => $(el).attr('href')?.endsWith('.txt') ?? false)
	.attr('href')
	if (!link) return

	const fileName = link.split('/').at(-1)?.replace(' ', '-')
	if (!fileName) return

	const signedUrl = await getSignedUrl(
	r2,
	new PutObjectCommand({
	Bucket: process.env.R2_BUCKET_NAME,
	Key: fileName
	}),
	{ expiresIn: 60 }
	)

	const fileResponse = await fetch(link)
	const fileContent = await fileResponse.text()

	await fetch(signedUrl, {
	method: 'PUT',
	body: fileContent
	})

	if (process.env.DISCORD_WEBHOOK_URL) {
	await fetch(process.env.DISCORD_WEBHOOK_URL, {
	method: 'POST',
	headers: {
	'Content-Type': 'application/json'
	},
	body: JSON.stringify({
	embeds: [
	{
	title: 'MTG Comprehensive Rules have been updated',
	description: `The new rulings file has been successfully uploaded to \`${fileName}\`.`,
	color: null,
	url: link
	}
	]
	})
	})
	}
	}

	void main()