2023.05.04.md

파일 결합

텍스트 파일

import pandas as pd
from glob import glob

if __name__ == "__main__":
    files = glob("./data/**/**/*")
    df_list = []
    for file in files:
        file_st = file.split("\\")[-1].split("-")
        date = file_st[0]
        plate = file_st[1]
        temp_df = pd.read_fwf(file, encoding="cp949", skiprows=1, header=None)
        temp_df[date] = date
        temp_df[plate] = plate
        temp_df.columns = ["데이터", "날짜", "번호판"]
        df_list.append(temp_df)
    df = pd.concat(df_list, axis=0, ignore_index=True)
    df.to_csv("./data/raw.csv", index=False)

엑셀 파일

if __name__ == "__main__":
    # 파일 읽어오기
    df2019_a = pd.read_excel("./data/origin/o2019년_a.xlsx", skiprows=2)
    df2022_b = pd.read_excel("./data/origin/o2022년_b.xlsx", skiprows=2)
    df2022_c = pd.read_excel("./data/origin/o2022년_c.xlsx", skiprows=2)
    # isna가 6000개 이상인 컬럼 제거
    df2019_a = remove_isna(df2019_a)
    df2022_b = remove_isna(df2022_b)
    df2022_c = remove_isna(df2022_c)
    # 컬럼명 통일
    columns_intersect = reduce(
        np.intersect1d, (df2019_a.columns, df2022_b.columns, df2022_c.columns)
    )
    print(type(columns_intersect))
    # 컬럼명 기준으로 통일
    df2019_a = df2019_a[columns_intersect]
    print(df2019_a.columns)
    df2022_b = df2022_b[columns_intersect]
    print(df2022_b.columns)
    df2022_c = df2022_c[columns_intersect]
    print(df2022_c.columns)

    # 병합
    df = pd.concat([df2019_a, df2022_b, df2022_c], axis=0)

    df.to_csv("./data/raw.csv", index=False)

업비트(request)

url = "https://api.upbit.com/v1/candles/minutes/1?market=KRW-BTC&count=10"
headers = {"Accept": "application/json"}
response = requests.request("GET", url, headers=headers)
print(response.text, "\n")

업비트(라이브러리)

access = "본인값으로 변경"
secret = "본인값으로 변경"
df = pyupbit.get_ohlcv("KRW-BTC", count = time+10, period=0.01)

CSV/JSON to DB

# pip install pymysql sqlalchemy
from sqlalchemy import create_engine
import pymysql
import pandas as pd
db_connection_str = 'mysql+pymysql://root:qwer1234@localhost/upbit'
db_connection = create_engine(db_connection_str)
conn = db_connection.connect()
df_bitcoin.to_sql(name='upbit', con=db_connection, if_exists='append',index=False)