b3_crawler.py

import re
import urllib.request
import urllib.parse
from datetime import datetime
import os
import boto3
import csv
from tempfile import TemporaryDirectory

from lxml.html import fragment_fromstring
import json
from utils import Pipeline, to_decimal, get_url_opener_config


def get_stock_list_from_fundamentus(stocks):
    url = 'http://www.fundamentus.com.br/resultado.php'
    opener = get_url_opener_config()

    data = {'pl_min': '',
            'pl_max': '',
            'pvp_min': '',
            'pvp_max': '',
            'psr_min': '',
            'psr_max': '',
            'divy_min': '',
            'divy_max': '',
            'pativos_min': '',
            'pativos_max': '',
            'pcapgiro_min': '',
            'pcapgiro_max': '',
            'pebit_min': '',
            'pebit_max': '',
            'fgrah_min': '',
            'fgrah_max': '',
            'firma_ebit_min': '',
            'firma_ebit_max': '',
            'margemebit_min': '',
            'margemebit_max': '',
            'margemliq_min': '',
            'margemliq_max': '',
            'liqcorr_min': '',
            'liqcorr_max': '',
            'roic_min': '',
            'roic_max': '',
            'roe_min': '',
            'roe_max': '',
            'liq_min': '',
            'liq_max': '',
            'patrim_min': '',
            'patrim_max': '',
            'divbruta_min': '',
            'divbruta_max': '',
            'tx_cresc_rec_min': '',
            'tx_cresc_rec_max': '',
            'setor': '',
            'negociada': 'ON',
            'ordem': '1',
            'x': '28',
            'y': '16'}

    with opener.open(url, urllib.parse.urlencode(data).encode('UTF-8')) as link:
        content = link.read().decode('ISO-8859-1')

    todays_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    pattern = re.compile('<table id="resultado".*</table>', re.DOTALL)
    content = re.findall(pattern, content)[0]
    page = fragment_fromstring(content)

    for rows in page.xpath('tbody')[0].findall("tr"):
        stocks.append({'ticker': rows.getchildren()[0][0].getchildren()[0].text,
                       'day_value': to_decimal(rows.getchildren()[1].text),
                       'p_l': to_decimal(rows.getchildren()[2].text),
                       'p_vp': to_decimal(rows.getchildren()[3].text),
                       'psr': to_decimal(rows.getchildren()[4].text),
                       'dy': to_decimal(rows.getchildren()[5].text),
                       'p_ativo': to_decimal(rows.getchildren()[6].text),
                       'p_cap_giro': to_decimal(rows.getchildren()[7].text),
                       'p_ebit': to_decimal(rows.getchildren()[8].text),
                       'p_acl': to_decimal(rows.getchildren()[9].text),
                       'ev_ebit': to_decimal(rows.getchildren()[10].text),
                       'ev_ebitda': to_decimal(rows.getchildren()[11].text),
                       'mrg_ebitda': to_decimal(rows.getchildren()[12].text),
                       'mrg_liq': to_decimal(rows.getchildren()[13].text),
                       'liq_corrent': to_decimal(rows.getchildren()[14].text),
                       'roic': to_decimal(rows.getchildren()[15].text),
                       'roe': to_decimal(rows.getchildren()[16].text),
                       'liq_2_meses': to_decimal(rows.getchildren()[17].text),
                       'patrimonio_liq': to_decimal(rows.getchildren()[18].text),
                       'div_brut_pat': to_decimal(rows.getchildren()[19].text),
                       'cresc_5_anos': to_decimal(rows.getchildren()[20].text),
                       'market_code': 'BR',
                       'inc_date': todays_date})

    return stocks


def save_stocks_as_tsv(stocks):
    bucket = os.environ.get('bucket')

    file_name = stocks[0]['inc_date'] + '.tsv'

    with TemporaryDirectory() as tmpdir:
        os.chdir(tmpdir)
        with open(file_name, 'w') as output_file:
            dw = csv.DictWriter(output_file, sorted(stocks[0].keys()), delimiter='\t')
            dw.writeheader()
            dw.writerows(stocks)
            output_file.close()

        if os.path.isfile(f'{file_name}'):
            print(f'{tmpdir}/{file_name} create successfully.')

        s3 = boto3.resource('s3')
        s3.Bucket(bucket).upload_file(file_name, file_name)

        # remove unexpected file
        os.remove(file_name)


def handler(event, context):
    stocks = list()
    steps = [
        get_stock_list_from_fundamentus,
        save_stocks_as_tsv
    ]

    response = {
        "statusCode": 200,
        "body": json.dumps({
            "message": "Cardinal-stocks finished."
        })
    }

    try:
        Pipeline.make_pipeline(steps, skip_step_when_fail=False).execute(stocks)
        print(f'Cardinal stocks worked, {len(stocks)} stocks processed')
    except Exception as e:
        print(e)
        response = {
            "statusCode": 400,
            "body": json.dumps({
                "message": str(e),
            })
        }

    return response