concurrent.py

concurrent.py

indexed = 0
all_links_final = set(iterate_list_of_urls)
indexed_list = {}
to_index = list(set(iterate_list_of_urls))

with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
  futures = [executor.submit(url_handling.crawl_urls, final_urls, namespaces_to_ignore, indexed, images_indexed, image_urls, links, external_links, discovered_urls, broken_urls, iterate_list_of_urls, site, crawl_budget, url) for url in to_index]
  no_more_found = False

  for future in concurrent.futures.as_completed(futures):
    print(indexed)

    if indexed == crawl_budget:
      break

    _, images_indexed, iterate_list_of_urls, all_links, final_urls, url_indexed, discovered = future.result()
    indexed += 1
    indexed_list[url_indexed] = True

    for item in discovered.keys():
      if not indexed_list.get(item):
        print("{} not indexed, added".format(item))
        futures.append(executor.submit(url_handling.crawl_urls, final_urls, namespaces_to_ignore, indexed, images_indexed, image_urls, links, external_links, discovered_urls, broken_urls, iterate_list_of_urls, site, crawl_budget, item))