task.py

class ThreadPool:
  def __init(self, max_size = 2, task_complete_observer): #task_complete_observer - объект-слушатель прогресса выполнения
    self.maxSize = max_size
    self.queue = Queue()
    self.threads = []
    self.task_complete_observer = task_complete_observer
    
  def post_task(task):
    if self.has_available_thread():
      self.get_available_thread().execute(task)
    else:
      self.queue.add(task)
      
  def on_thread_completed_task(task): # точка в которой узнаем, что какой-то из потоков выполнил задачу и освободился
    self.task_complete_observer.notify_task_complete(task)
    if self.queue().size > 0:
      self.post_task(self.queue.pop())
      
def Task:
  def __init__(self, map, subStr, link):
    self.map = map
    self.link = link
    self.subStr = subStr
    self.nestedLinks = []
    
  def execute(self):
    map[self.link] = Status.Loading()
    pageContent, error = loadPage(self.link)
    if error is not None:
      map[self.link] = Status.Error(error)
      return None
    containsStr = pageContent.contains(subStr) # Boolean
    map[self.link] = Status.Found(containsStr)
    self.nestedLinks = findMoreLinks(pageContent)
  
class TaskCompletionObserver:
  def __init__(self, map, thread_pool=None):
    self.map = map
    self.thread_pool = map
  
  def notify_task_complete(self, task):
    for link in task.nestedLinks:
      if link not in map:
        map[link] = Status.Awaiting()
        self.thread_pool.post_task(Task(map, "Substring to find", link))
   
  
initial_link = 'http://firstlink.com'
map = {initial_link: Status.Awaiting()}
first_task = Task(map, "Substring to find", 'http://firstlink.com')

observer = TaskCompletionObserver(map)
thread_pool = ThreadPool(max_size=10, observer)
observer.thread_pool = thread_pool
thread_pool.post_task(first_task)