(Unofficial method) Get some finance information for Naver finance website with avoid excessive request frequency and allowing 1-minute delay

get_naver_finance_info_lite.py

import requests
import time
import random
import os
import ast
import pickle

class CustomUserAgentString:

    user_agent_string_list = [
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:106.0) Gecko/20100101 Firefox/106.0',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
        'Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:105.0) Gecko/20100101 Firefox/105.0',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:106.0) Gecko/20100101 Firefox/106.0',
        'Mozilla/5.0 (Windows NT 10.0; rv:106.0) Gecko/20100101 Firefox/106.0',
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.1 Safari/605.1.15',
        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36',
        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.52',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36 Edg/107.0.1418.35'
    ]

class get_kr_finance_information:

    @staticmethod
    def fetch_overall_information():

        # to prevent excessive requests, store data as a JSON file that is valid for 60 seconds
        kr_finance_raw_data_file_path = f"{os.getcwd()}/kr_finance_raw.data.pickle"

        if not os.path.isfile(kr_finance_raw_data_file_path) or not time.time() - os.path.getmtime(kr_finance_raw_data_file_path) < 60:
        
            request_url_list = { "overview"         : "https://api.finance.naver.com/service/mainSummary.naver",                                      # overview market
                                 "major_indexes"    : "https://polling.finance.naver.com/api/realtime?query=SERVICE_INDEX:KOSPI,KOSDAQ,KPI200" }      # major index
            headers = { "user-agent" : random.choice(CustomUserAgentString.user_agent_string_list) }

            kr_finance_raw_data = {}

            # annotation, url
            for key, value in request_url_list.items():
                response = requests.get(value, headers = headers)
                                                # Python uses "None" instead of "null"
                data = response.text.replace("null", "None")
                kr_finance_raw_data[key] = data

            with open(kr_finance_raw_data_file_path, 'wb') as file:
                pickle.dump(kr_finance_raw_data, file)
        

        with open(kr_finance_raw_data_file_path, 'rb') as file:
            kr_finance_data = pickle.load(file)

        return kr_finance_data


    @staticmethod
    def extract_data_by_criteria(criteria:str):

        kr_finance_data = get_kr_finance_information.fetch_overall_information()

        available_criterias = {
            # daily overall top ranking
            "top_trades"        : ["overview", "message", "result", "topItems", 0],
            "top_rise"          : ["overview", "message", "result", "topItems", 1],
            "top_fall"          : ["overview", "message", "result", "topItems", 2],
            "top_market_cap"    : ["overview", "message", "result", "topItems", 3],
            
            # daily specified ranking
            "group_top_list"    : ["overview", "message", "result", "groupTopList"],
            "theme_top_list"    : ["overview", "message", "result", "themeTopList"],

            # index ranking
            "daily_index_trend" : ["overview", "message", "result", "todayIndexDealTrendList"],     # buy/sell trend
            "daily_index_items" : ["overview", "message", "result", "todayIndexItemList"],          # rise/even/steady count in the index

            # index value
            "daily_index_stats" : ["major_indexes", "result", "areas"]                              # index stat with highly-abbreviated keys

        }

        if criteria not in available_criterias:
            print("Unavilable criteria")
            return

        # find multi-depth dictionary dynamically according to the given criteria
        result = kr_finance_data

        for search_option in available_criterias[criteria]:
            try:
                result = result[search_option]
            except TypeError:
                result = dict(ast.literal_eval(result))
                result = result[search_option]

        return result


    @staticmethod
    def get_index_overview(index:str):

        if index not in ["KOSPI", "KOSDAQ", "KPI200"]:
            print("Unavailable index")
            return
        
        daily_index_trend   = get_kr_finance_information.extract_data_by_criteria(criteria = "daily_index_trend")
        daily_index_items   = get_kr_finance_information.extract_data_by_criteria(criteria = "daily_index_items")
        daily_index_stats   = get_kr_finance_information.extract_data_by_criteria(criteria = "daily_index_stats")

        if index == "KOSPI":
            data_dict_index = 0
        elif index == "KOSDAQ":
            data_dict_index = 1
        else:       # KPI200
            data_dict_index = 2


        result = {
            "item_code"             : index,

            # index trends (unit : KRW)
            "personal_value"        : round(float(daily_index_trend[data_dict_index]["personalValue"]) * 100000000),
            "foreign_value"         : round(float(daily_index_trend[data_dict_index]["foreignValue"]) * 100000000),
            "institution_value"     : round(float(daily_index_trend[data_dict_index]["institutionalValue"]) * 100000000),

            # index items change overview (unit : EA)
            "upper_limit_count"     : daily_index_items[data_dict_index]["upperCnt"],
            "rise_count"            : daily_index_items[data_dict_index]["riseCnt"],
            "steady_count"          : daily_index_items[data_dict_index]["steadyCnt"],
            "fall_count"            : daily_index_items[data_dict_index]["fallCnt"],
            "lowerCnt"              : daily_index_items[data_dict_index]["lowerCnt"],

            # index statistics
            "market_open_close"     : daily_index_stats[0]["datas"][data_dict_index]["ms"],
            "current_value"         : int(daily_index_stats[0]["datas"][data_dict_index]["nv"]) / 100,
            "current_change_value"  : int(daily_index_stats[0]["datas"][data_dict_index]["cv"]) / 100,
            "current_change_rate"   : int(daily_index_stats[0]["datas"][data_dict_index]["cr"]),            # change rate (%)
            "daily_high_value"      : int(daily_index_stats[0]["datas"][data_dict_index]["hv"]) / 100,
            "daily_low_value"       : int(daily_index_stats[0]["datas"][data_dict_index]["lv"]) / 100,
            "acc_trading_volume"    : int(daily_index_stats[0]["datas"][data_dict_index]["aq"]) * 1000,
            "acc_trading_price"     : int(daily_index_stats[0]["datas"][data_dict_index]["aa"]) * 1000000,  # 1 = million won,
            "change_image_url"      : "https://ssl.pstatic.net/imgfinance/chart/main/KOSPI.png"
        }

        print(result)

# get_kr_finance_information.get_index_overview("KOSPI")
get_kr_finance_information.get_index_overview(index = "KOSPI")