Scrape daily stock data from Budapest Stock Exchange (BÉT) and saves data to Graphite DB

scrape_bet.sh

stocks="3564,4IG 6494,ALTEO 4042,ANY 6329,APPENINN 8208,AUTOWALLIS 424,BIF 6515,CIGPANNONIA 9864,DUNAHOUSE 4338,GSPARK 506,KONZUM 7217,MASTERPLAST 518,MOL 511,MTELEKOM 540,OPUS 528,OTP 546,PANNERGY 604,RABA 608,RICHTER 3263,TAKAREKJZB 11141,WABERERS 639,ZWACK"
curl -H 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0' -v "https://bet.hu/oldalak/adatletoltes"  > /tmp/out.html 2>&1
sessionid=`cat /tmp/out.html | grep JSESSIONID | sed 's/< set-cookie: JSESSIONID=//g' | sed 's/;.*//g'`
csrf=`cat /tmp/out.html | grep csrf | sed  's/\s\+<meta name="_csrf" content="//g' | sed 's/".*//g'`

defaultStart=`date --date="2 day ago" +"%Y.%m.%d."`
today=`date +"%Y.%m.%d."`

# Max is 5 years
startDate=$1
endDate=$2

if [ -z $endDate ]; then
   startDate=$defaultStart
   endDate=$today
fi

echo "Querying between $startDate $endDate"

for stock in $stocks; do

  echo $stock

  id=`echo $stock | cut -d, -f1`
  name=`echo $stock | cut -d, -f2`

  curl "https://bet.hu/oldalak/adatletoltes/\$rspid0x117390x12/\$rihistoricalGenerator?_csrf=$csrf" -H 'User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:80.0) Gecko/20100101 Firefox/80.0' -H 'Accept: */*' -H 'Accept-Language: en-US,en;q=0.5' --compressed -H 'Referer: https://bet.hu/' -H 'Content-type: application/json' -H 'Origin: https://bet.hu' -H 'DNT: 1' -H 'Connection: keep-alive' -H "Cookie: JSESSIONID=$sessionid" --data "{\"startingValue\":\"$startDate\",\"endingValue\":\"$endDate\",\"resolution\":\"DAY_TO_DAY\",\"market\":\"PROMPT\",\"format\":\"CSV\",\"type\":\"CLOSING\",\"currentCategory\":\"W_RESZVENYA\",\"selectionList\":[{\"category\":\"Részvények Prémium\",\"selectedInstruments\":[{\"id\":\"$id\",\"code\":\"$name\"}]}]}" > "/tmp/input.csv"
  tail -n +2 /tmp/input.csv > /tmp/input_preprocessed.csv

  while IFS=, read -r name_col date_col price_col volume_col
  do
    echo "Feeding: $name_col $date_col"
    d=`echo "$date_col" | sed "s/\./-/g" | sed "s/-$//g"`
    d=`date -d "$d" "+%s"`

    if [ -n "$price_col" ]; then
      echo "stock.price.$name_col $price_col $d" | nc -q0 172.17.0.1 12003
      echo "stock.volume.$name_col $volume_col $d" | nc -q0 172.17.0.1 12003
    fi

  done < /tmp/input_preprocessed.csv
done