join pandas DFs and write to CSV using multiprocessing

pandas_join.py

import feather
import os
import pandas as pd
import multiprocessing

folder_path = r'/home/ubuntu/raster-vector-to-tsv/output/925ff7f3-c08b-4de5-a385-469e0b26975a/final_tsv/'

csv_list = [os.path.join(folder_path, x) for x in os.listdir(folder_path) if os.path.splitext(x)[1] == '.tsv']


def mp_worker(csv_path):

    print 'starting merge'
    df1 = feather.read_dataframe(csv_path)
    df2 = feather.read_dataframe(csv_path)

    df = pd.merge(df1, df2, how='left', on=['0', '1'])
    print 'writing'

    out_csv = os.path.splitext(csv_path)[0] + '_out.csv'
    df.to_csv(out_csv, index=None, header=None)


def mp_handler():
    p = multiprocessing.Pool()
    p.map(mp_worker, csv_list)

if __name__ == '__main__':
    mp_handler()