Today Stock - Crawling

main.py

import time
import argparse

import stock_rank_cron

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('Mode', type=str, choices=['once','interval','cron'], default='once', help="Choose how you want to run the code")
    parser.add_argument('Market', type=str, choices=['KOSPI','KOSDAQ'], default='KOSPI', help="Which markets would you like to crawl?")
    parser.add_argument('Max', type=int, default='30', choices=[10, 30, 50, 100], help="How many are you going to crawl?")

    args = parser.parse_args()
    try:
        stockRankCron = stock_rank_cron.StockRankCron()
        stockRankCron.run(args.Mode, args.Market, args.Max)
        while True:
            time.sleep(1)
    except KeyboardInterrupt:
        stockRankCron.stop()

    
if __name__=="__main__":
    main()

stock_rank_cron.py

from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.jobstores.base import JobLookupError
import requests
import datetime

import stock_rank_dbmanager

class StockRankCron():
    def __init__(self):
        print ('크론 시작')
        self.scheduler = BackgroundScheduler(job_defaults={'max_instances': 10, 'coalesce': False})
        self.scheduler.start()
        self.dbManager = stock_rank_dbmanager.StockRankDBManager()
        self.market = 'KOSPI'
        self.max = 30

    def __del__(self): 
        self.stop()

    def exec(self):
        print ('DaumStockCron Start: ' + datetime.datetime.now().strftime("%m/%d/%Y, %H:%M:%S"))

        params = {
            'market': self.market,
            'perPage': self.max,
            'page': 1,
            'intervalType': 'TODAY',
            'changeType': 'RISE',
            'pagination': 'true',
            'order': 'desc'
          }

        headers = {
            'referer': 'https://finance.daum.net',
            'User-Agent': 'chrome'
        }

        URL = 'https://finance.daum.net/api/trend/price_performance'

        try: 
            self.dbManager.queryCreateStockRankTable(self.market)
            self.dbManager.queryDeleteAlltDaumStockRankTable()
            res = requests.get(URL, headers=headers, params=params)
            if res.status_code == 200:
                datas = res.json()['data']
                for data in datas:
                    self.dbManager.queryInsertStockRankTable(data)
            else:
                print ('DAUM API 연결 에러')
        except requests.exceptions.RequestException as err:
            print ('Error Requests: {}'.format(err))
    
    def run(self, mode, market, max):
        print ("실행!")
        self.market = market
        self.max = max
        if mode == 'once':
            self.scheduler.add_job(self.exec)
        elif mode == 'interval':
            self.scheduler.add_job(self.exec, 'interval', seconds=10)
        elif mode == 'cron':
            self.scheduler.add_job(self.exec, 'cron', hour='9,10,11,12,13,14,15', second='*/10')

    def stop(self):
        try: self.scheduler.shutdown() 
        except: pass
        try: self.dbManager.close() 
        except: pass

stock_rank_dbmanager.py

import sqlite3

class StockRankDBManager:
    def __init__(self):
        print ("DB Manager 시작")
        self.DBName = 'stock_rank.db'
        self.db = sqlite3.connect(self.DBName, check_same_thread=False)
        self.db.row_factory = sqlite3.Row
        self.daum_stock_rank_table = 'daum_stock_rank'
        self.daum_stock_rank_colums = {
            'rank': 'integer PRIMARY KEY',
            'name': 'text',
            'symbolCode': 'text',
            'tradePrice': 'integer',
            'change': 'text',
            'changePrice': 'integer',
            'changeRate': 'real',
            'accTradeVolume': 'integer',
            'accTradePrice': 'integer',
            'high52wPrice': 'integer',
            'low52wPrice': 'integer'
        }

    def __del__(self):
        self.stop()

    def stop(self):
        try: self.db.close()
        except: pass
    
    def queryCreateStockRankTable(self, market):
        self.daum_stock_rank_table = 'daum_stock_rank_' + market.lower()
        cursor = self.db.cursor()
        colum_info = ",".join(col_name + ' ' + col_type for col_name, col_type in self.daum_stock_rank_colums.items())
        query = "CREATE TABLE IF NOT EXISTS {} ({})".format(self.daum_stock_rank_table, colum_info)
        cursor.execute(query)
        self.db.commit()

    def queryInsertStockRankTable(self, values):
        cursor = self.db.cursor()
        colums = ",".join(self.daum_stock_rank_colums.keys())
        values = "','".join(str(values[col_name]) for col_name in self.daum_stock_rank_colums.keys())
        query = "INSERT INTO {} ({}) VALUES ('{}')".format(self.daum_stock_rank_table, colums, values)
        cursor.execute(query)
        self.db.commit()

    def queryDeleteAlltDaumStockRankTable(self):
        query = "DELETE FROM {}".format(self.daum_stock_rank_table)
        cursor = self.db.cursor()
        cursor.execute(query)
        self.db.commit()