bblfsh_bug.py

import threading
import time
import logging

from bblfsh import BblfshClient

BBLFSH_ENDPOINT = "172.17.0.1:9437"
#BBLFSH_ENDPOINT = "0.0.0.0:9432"
LANG = "Python"
NUM_THREADS = 32
STOP_TIME = 1200
TIMEOUT = 30

bblfsh = [BblfshClient(BBLFSH_ENDPOINT) for _ in range(NUM_THREADS)]
stop = False

with open("python_files.txt") as f:
    python_files = f.readlines()


def thread_loop(thread_idx, filenames):
    log = logging.getLogger('hang_test')
    log.setLevel(logging.INFO)
    for filename in filenames:
        log.warning("%d thread. Parse %s", thread_idx, filename[:-1])
        try:
            res = bblfsh[thread_idx].parse(filename[:-1], language=LANG, timeout=TIMEOUT)
        except Exception as e:
            log.exception("%d thread. Parse %s", thread_idx, filename)

start_time = time.time()
pool = [threading.Thread(target=thread_loop, args=(i, python_files[i::NUM_THREADS]),
                         name=str(i))
        for i in range(NUM_THREADS)]

for thread in pool:
    thread.start()

time.sleep(STOP_TIME)
stop = True

for thread in pool:
    thread.join()