AlexMikhalev

from transformers import BertForQuestionAnswering
import torch

bert_name = "bert-large-uncased-whole-word-masking-finetuned-squad"

model = BertForQuestionAnswering.from_pretrained(bert_name, torchscript=True)
model.eval()

inputs = [torch.ones(1, 2, dtype=torch.int64),
          torch.ones(1, 2, dtype=torch.int64),

AlexMikhalev

bg = GearsBuilder('KeysReader')
bg.foreach(process_item)
bg.count()
bg.register('sentence:*',  mode="async_local",onRegistered=OnRegisteredAutomata)

AlexMikhalev

import httpimport
    with httpimport.remote_repo(['stop_words'], "https://raw.githubusercontent.com/explosion/spaCy/master/spacy/lang/en/"):
        import stop_words
    from stop_words import STOP_WORDS

AlexMikhalev

global sym_spell
if not sym_spell:
	sym_spell=load_symspell()
suggestions = sym_spell.lookup_compound(sentence['value'], max_edit_distance=1,
                                        transfer_casing=True, ignore_non_words=True)

AlexMikhalev

"""
load symspell and relevant dictionaries
"""
sym_spell=None 

def load_symspell():
    import pkg_resources
    from symspellpy import SymSpell, Verbosity
    sym_spell = SymSpell(max_dictionary_edit_distance=1, prefix_length=7)
    dictionary_path = pkg_resources.resource_filename(

AlexMikhalev

gb = GB('KeysReader')
gb.filter(filter_language)
gb.flatmap(parse_paragraphs)
gb.map(spellcheck_sentences)
gb.foreach(save_sentences)
gb.count()
gb.register('paragraphs:*',keyTypes=['string','hash'], mode="async_local")

AlexMikhalev

docker exec -it rgcluster /cluster/create-cluster call RG.CONFIGSET ExecutionMaxIdleTime 300000
docker exec -it rgcluster /cluster/create-cluster call CONFIG SET proto-max-bulk-len 2048mb
docker exec -it rgcluster /cluster/create-cluster call CONFIG SET list-compress-depth 1
docker exec -it rgcluster /cluster/create-cluster call CONFIG SET cluster-node-timeout 30000

AlexMikhalev

import scispacy
import spacy
from scispacy.abbreviation import AbbreviationDetector
from scispacy.umls_linking import UmlsEntityLinker
nlp = spacy.load("en_core_sci_sm")
#^^^ at this import you area looking at 9 GB RAM consumption 
#even before processing starts


# Add the abbreviation pipe to the spacy pipeline.

AlexMikhalev

from notion.client import NotionClient
import datetime
import os
from slugify import slugify
import re
import requests
import time
import hashlib
import shutil
import sys

AlexMikhalev

=== REDIS BUG REPORT START: Cut & paste starting from here ===
13:S 11 Jan 2021 12:11:34.456 # === ASSERTION FAILED ===
13:S 11 Jan 2021 12:11:34.456 # ==> src/readers/keys_reader.c:228 'RedisModule_CallReplyType(reply) == REDISMODULE_REPLY_ARRAY' is not true
13:S 11 Jan 2021 12:11:34.456 # (forcing SIGSEGV to print the bug report.)
13:S 11 Jan 2021 12:11:34.456 # Redis 6.0.1 crashed by signal: 11
13:S 11 Jan 2021 12:11:34.456 # Crashed running the instruction at: 0x55b3cdb49d3b
13:S 11 Jan 2021 12:11:34.456 # Accessing address: 0xffffffffffffffff
13:S 11 Jan 2021 12:11:34.456 # Failed assertion: RedisModule_CallReplyType(reply) == REDISMODULE_REPLY_ARRAY (src/readers/keys_reader.c:228)

------ STACK TRACE ------