crawl_hs_inven.py

import json
import requests
from bs4 import BeautifulSoup
from requests import Response

from cards.models import Card

CARD_LIST_URL = 'http://hs.inven.co.kr/dataninfo/card/list.ajax.php'


def fetch_card_list_from_inven(page: int = 1, filter_str: str = '') -> Response:
    """Inven에서 Card 목록 파싱
    :param page: 페이지 넘버
    :param filter_str: 필터링 문자열
    :return:
    """
    target_url = CARD_LIST_URL + filter_str
    headers = {
        'Content-Type': 'application/x-www-form-urlencoded',
        'Referer': 'http://hs.inven.co.kr/dataninfo/card/',
    }
    payload = {
        'list': 'list',
        'page': page,
    }
    response = requests.post(target_url, headers=headers, data=payload)
    return response


def get_card_list(page: int = 1, filter_str: str = '') -> list:
    """Card 목록 가공해서 list로 만듦
    :param page: 페이지 넘버
    :param filter_str: 필터링 문자열
    :return: 해당하는 Card 목록
    """
    result = []

    response = fetch_card_list_from_inven(page, filter_str)
    html_content = response.text.encode(response.encoding)
    html = BeautifulSoup(html_content, 'html.parser')

    tr_list = html.find_all('tr')

    for tr in tr_list[:-1]:
        td_list = tr.find_all('td')
        card_name_tags = td_list[1].find_all('a')

        kr_name = card_name_tags[0].text
        en_name = card_name_tags[1].text
        card_type = td_list[2].text
        rarity = td_list[3].text

        cost = td_list[-4].find('span')
        cost = int(cost.text) if cost else None

        attack = td_list[-3].find('span')
        attack = int(attack.text) if attack else None

        health = td_list[-2].find('span')
        health = int(health.text) if health else None

        desc = td_list[-1].text

        card = {
            'cid': None,
            'kr_name': kr_name,
            'en_name': en_name,
            'type': card_type,
            'rarity': rarity,
            'cost': cost,
            'attack': attack,
            'health': health,
            'desc': desc,
        }

        result.append(card)

    return result


for page in range(1, 57):
    card_list = get_card_list(page)
    card_model_list = []

    for card in card_list:
        if card['cost']:
            if card['rarity'] == '일반':
                rarity = 1
            elif card['rarity'] == '희귀':
                rarity = 2
            elif card['rarity'] == '영웅':
                rarity = 3
            elif card['rarity'] == '전설':
                rarity = 4
            else:
                # 무료
                rarity = 0

            card_model_list.append(
                Card(
                    kr_name=card['kr_name'],
                    en_name=card['en_name'],
                    card_type=card['type'],
                    rarity=rarity,
                    cost=card['cost'],
                    attack=card['attack'],
                    health=card['health'],
                    description=card['desc'],
                )
            )

    Card.objects.bulk_create(card_model_list)