profile by subprocess.Popen

profile_cmd.py

import queue
import random
import time
from datetime import datetime
from itertools import product
import concurrent.futures

import pandas as pd
from pipe import as_list


def worker(q, profiles):
    while not q.empty():
        arg = q.get()
        fname, algo, supp, conf = \
            [arg[k] for k in ['fname', 'algo', 'supp', 'conf']]
        p = subprocess.Popen(
            'exec ../fim/{} -tr -s{} -c{} {} /dev/null'
            .format(algo, supp, conf, fname),
            shell=True
        )
        start = time.time()

        try:
            p.wait(timeout=3)
        except subprocess.TimeoutExpired:
            p.kill()
            pass

        arg.update({
            'cost': time.time() - start
        })
        profiles.append(arg)


q = queue.Queue()
profiles = []

# generate task
length_range = range(10000, 60000, 5000)
width_range = range(40, 50, 10)
overlap_range = range(40, 50, 10)
supp_range = range(0, 105, 5)
conf_range = range(0, 105, 5)

for (length, width, overlap) in \
        product(length_range, width_range, overlap_range):
    # tracts = gene_tracts(length, width, overlap, True)
    # tracts = real_dataset[: length]
    tracts = merck[: length]
    # tracts = b2b[: length]
    fname = save_tracts(tracts, length, width, overlap)
    for (supp, conf) in product(supp_range, conf_range):
        map(q.put, [
            {
                'algo': 'apriori', 'fname': fname, 'supp': supp, 'conf': conf,
                'length': length, 'overlap': overlap, 'width': width
            }, {
                'algo': 'fpgrowth', 'fname': fname, 'supp': supp, 'conf': conf,
                'length': length, 'overlap': overlap, 'width': width
            }, {
                'algo': 'eclat', 'fname': fname, 'supp': supp, 'conf': conf,
                'length': length, 'overlap': overlap, 'width': width
            }]) | as_list


with concurrent.futures.ThreadPoolExecutor(max_workers=7) as executor:
    futures = [executor.submit(worker, q, profiles) for _ in range(7)]
    for future in concurrent.futures.as_completed(futures):
        future.result()

profile = pd.DataFrame(profiles)
profile.to_pickle('../out/merck_profile.pkl')
print('ok')