nltk_mae_test.ipynb

xxxx.py

import concurrent.futures
import os
import pandas as pd
import time
# from tqdm import tqdm_notebook as tqdm
from tqdm import tqdm
# import matplotlib.pyplot as plt

from textblob import TextBlob

def f(x):
    polarity = TextBlob(x).polarity
    
    return polarity

def main():
    dir_name = 'val/'
    jsons = os.listdir(dir_name)
    df_arr = []

    for val in tqdm(jsons):
        df_t = pd.read_json(dir_name + val)
        df_arr.append(df_t)

    df = pd.concat(df_arr, ignore_index=True)

    text_arr = list(df['text'])
    sent_arr = []

    with concurrent.futures.ProcessPoolExecutor() as executor:
        for sent in executor.map(f, tqdm(text_arr)):
            sent_arr.append(sent)

    print(sum(sent_arr) / len(sent_arr))

if __name__ == '__main__':
    s_time = time.time()
    main()
    print(f'end time {time.time() - s_time}')