Spyder's IPython and process pools

pooled.py

# -*- coding: utf-8 -*-
"""
Created on Sat Feb 10 16:32:44 2018

@author: МакаровАС
"""

from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
import time

class Pooled():
    def __init__(self, worker, max_workers=1, pool_type="thread",
                 callback=None):
        self.callback = callback
        self.tasks_num = 0
        self.pool_type = pool_type.lower()
        self.max_workers = max_workers
        if self.pool_type == "thread":
            self.pool = ThreadPoolExecutor(max_workers=self.max_workers)
        elif self.pool_type == "process":
            self.pool = ProcessPoolExecutor(max_workers=self.max_workers)
        else:
            raise Exception("Unknown pool type " + pool_type)
        self.worker = worker

    def submit(self, *args, **kwargs):
        "Submit a new task"
        self.tasks_num += 1
        future = self.pool.submit(self.worker, *args, **kwargs)
        future.start_time = time.clock()
        future.add_done_callback(self.__done_callback)

    def __done_callback(self, future):
        "A task finished"
        if self.callback:
            self.callback(future, time.clock() - future.start_time)
        self.tasks_num -= 1

    def shutdown(self, wait=True):
        "Destroy pool threads/processes"
        self.pool.shutdown(wait)
    
def main():
    def cb(f, t):
        print(f.result())

    from pooled_worker import Test
    t = Test()
    p = Pooled(t.parse_card, 5, 'process', cb)
    p.submit({'status': 'done'})
    p.shutdown()

if __name__ == "__main__":
    __spec__ = None  # Fix multiprocessing in Spyder's IPython
    main()

pooled2.py

# -*- coding: utf-8 -*-
"""
Created on Sat Feb 10 16:32:44 2018

@author: МакаровАС
"""

from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor
import time

class Pooled():
    def __init__(self, worker, max_workers=1, pool_type="thread",
                 callback=None):
        self.callback = callback
        self.tasks_num = 0
        self.pool_type = pool_type.lower()
        self.max_workers = max_workers
        if self.pool_type == "thread":
            self.pool = ThreadPoolExecutor(max_workers=self.max_workers)
        elif self.pool_type == "process":
            self.pool = ProcessPoolExecutor(max_workers=self.max_workers)
        else:
            raise Exception("Unknown pool type " + pool_type)
        self.worker = worker

    def submit(self, *args, **kwargs):
        "Submit a new task"
        self.tasks_num += 1
        future = self.pool.submit(self.worker, *args, **kwargs)
        future.start_time = time.clock()
        future.add_done_callback(self.__done_callback)

    def __done_callback(self, future):
        "A task finished"
        if self.callback:
            self.callback(future, time.clock() - future.start_time)
        self.tasks_num -= 1

    def shutdown(self, wait=True):
        "Destroy pool threads/processes"
        self.pool.shutdown(wait)
    
def main():
    def cb(f, t):
        print(f.result())

    from pooled_worker import Test
    t = Test()
    p = Pooled(t.parse_card, 5, 'process', cb)
    p.submit({'status': 'done'})
    p.shutdown()

if __name__ == "__main__":
    __spec__ = None  # Fix multiprocessing in Spyder's IPython
    main()

pooled_worker.py

# -*- coding: utf-8 -*-
"""
Created on Sat Feb 10 17:25:26 2018

@author: МакаровАС
"""

class Test():
    def parse_card(self, card):
        return card["status"]

Modules pooled and pooled2 are completely the same. Module pooled_worker contains a function which is run parallely.

1. run pooled.py in a dedicated IPython
2. You fail, but you don't close the console
3. run pooled2.py in a new dedicated IPython
4. success