Leetcode simple bash crawler without login #post

README.md

I just wrote a simple bash code to crawling leetcode's

1. code definition
2. question frontend id
3. content
4. example testcases

and then template them into a rust file

I know there has much more I can enhanced, but I am just lazzzzzzzzy :lanzyface

for the newest version you may referenced to this file new

crawler_bash

#!/bin/bash
###################################
# just bash new {problem's name}  #
# example: bash new two-sum       #
###################################

set -euo pipefail

problem_name=$(bb "$1" -s -l -t -)

header=$(curl -sIL --max-redirs 1 https://leetcode.com/problems/$problem_name | rg -o -U --multiline-dotall -w '(HTT
P/2 200)(.)*')

csrf=$(echo $header | rg -o -w 'csrftoken=([^;]+);*?')
cookie=$(echo $header | rg -o -w '__cf_bm=([^;]+);*?')

# combine the csrf
cookie="${csrf}; ${cookie}"
# get rid of csrftoken= string
csrf=${csrf:10}

request_json=$(jq -r -n \
  --arg ts "$problem_name" \
'{
    "operationName": "questionData",
    "variables": {"titleSlug": $ts},
    "query": "query questionData($titleSlug: String!)
    {
      question(titleSlug: $titleSlug)
      {
        questionFrontendId
        codeDefinition
        content
        exampleTestcases
      }
    }"
}')

response_json=$(echo $request_json | curl -s -XPOST 'https://leetcode.com/graphql' \
  -H 'authority: leetcode.com' \
  -H 'pragma: no-cache' \
  -H 'cache-control: no-cache' \
  -H 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36' \
  -H 'content-type: application/json' \
  -H "x-csrftoken: ${csrf}" \
  -H "cookie: ${cookie}" \
  -H "referer: https://leetcode.com/problems/${problem_name}/" \
  --data-binary @- \
  --compressed)

response_json=$(echo $response_json | sd '<[^>]*>' '')
response_json=$(echo $response_json | sd '&[\w]+;' ' ')
response_json=$(echo $response_json | sd '\\r\\n' '\\\\n// ')

code_definition=$(echo $response_json | jq -r '.data.question.codeDefinition')
question_frontend_id=$(echo $response_json | jq -r '.data.question.questionFrontendId')
content=$(echo $response_json | jq -r '.data.question.content')
example_testcases=$(echo $response_json | jq -r '.data.question.exampleTestcases')

content=$(echo "$content" | sd '\n' '\n// ')
example_testcases=$(echo "$example_testcases" | sd '\n' '\n// ')

# finding out the rust default code
for row in $( echo "${code_definition}" | jq -c -r '.[] | @base64' ); do
  _jq() {
    echo $row | base64 --decode | jq -r ${1}
  }

  a=$(_jq '.value')
  if [ $a == "rust" ]; then
    code_definition=$(_jq '.defaultCode')
    break
  fi
done

if [ -z "$code_definition" ]; then
  echo "this problem is not support language: rust"
  exit 1
fi

problem_name=$(bb "$problem_name" -f - -t _)
file="./src/p${question_frontend_id}_${problem_name}.rs"
cp ./template.rs $file

sd "__PROBLEM_DESC__" "$content" $file
sd "__PROBLEM_DEFAULT_CODE__" "$code_definition" $file
sd "__PROBLEM_EXAMPLE_TESTCASES__" "$example_testcases" $file
sd "__PROBLEM_ID__" "$question_frontend_id" $file

rustfmt $file

echo "mod p${question_frontend_id}_${problem_name};" >> ./src/lib.rs
rustfmt ./src/lib.rs

template.rs

// __PROBLEM_DESC__
//
// __PROBLEM_EXAMPLE_TESTCASES__

#![allow(dead_code)]
pub struct Solution {}

__PROBLEM_DEFAULT_CODE__

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test___PROBLEM_ID___solution() {
    }
}