bblfsh_hang.py

import threading
import time
import logging

from bblfsh import BblfshClient

BBLFSH_ENDPOINT = "172.17.0.1:9432"
LANG = "Python"
NUM_THREADS = 32
TIMEOUT = 120
LOG_LEVEL = "DEBUG"

bblfsh = [BblfshClient(BBLFSH_ENDPOINT) for _ in range(NUM_THREADS)]

with open("files.txt", "r") as f:
    python_files = f.readlines()
python_files = list(set(python_files))

def thread_loop(thread_idx, filenames):
    log = logging.getLogger('hang_test')
    log.setLevel(logging.INFO)
    for j in range(1):
        for i, filename in enumerate(filenames):
            filename = filename[:-1]
            log.warning("%d thread. %d/%d Parse %s", thread_idx, i, len(filenames), filename)
            try:
                res = bblfsh[thread_idx].parse(filename, language=LANG, timeout=TIMEOUT)
            except Exception as e:
                log.exception("%d thread. %d/%d Parse %s", thread_idx, i, len(filenames), filename)
    log.warning("\n\n\n%d thread. Done!\n\n\n", thread_idx)

start_time = time.time()
pool = [threading.Thread(target=thread_loop, args=(i, python_files[i::NUM_THREADS]),
                         name=str(i))
        for i in range(NUM_THREADS)]

for thread in pool:
    thread.start()

for thread in pool:
    thread.join()