Marcin Zabłocki marrrcin

## processify.py
import os
import sys
import traceback
from functools import wraps
from multiprocessing import Process, Queue


def processify(func):
    '''Decorator to run a function as a process.
    Be sure that every argument and the return value

## imports.py
import pandas as pd
import numpy as np
from pathlib import Path
import os
from glob import glob
import re

## polish_sentence_nltk_tokenizer.py
import nltk

# interactive download
# nltk.download()
nltk.download('punkt')

extra_abbreviations = ['ps',  'inc', 'Corp', 'Ltd', 'Co', 'pkt', 'Dz.Ap', 'Jr', 'jr', 'sp', 'Sp', 'poj',  'pseud', 'krypt', 'sygn', 'Dz.U', 'ws', 'itd', 'np', 'sanskryt', 'nr', 'gł', 'Takht', 'tzw', 't.zw', 'ewan', 'tyt', 'oryg', 't.j', 'vs', 'l.mn', 'l.poj' ]

position_abbrev = ['Ks', 'Abp', 'abp','bp','dr', 'kard', 'mgr', 'prof', 'zwycz', 'hab', 'arch', 'arch.kraj', 'B.Sc', 'Ph.D', 'lek', 'med', 'n.med', 'bł', 'św', 'hr', 'dziek' ]

## broken-tokenizer.ipynb

      
              1 file
            
          
              0 forks
            
          
              0 comments
            
          
              0 stars
            
          
                marrrcin
                / broken-tokenizer.ipynb
            
            
              Created
              March 1, 2021 09:29
                — forked from LysandreJik/broken-tokenizer.ipynb
            
              
                Broken Tokenizer
              
          
      Sorry, something went wrong. Reload?
      Sorry, we cannot display this file.
      Sorry, this file is invalid so it cannot be displayed.
      
          Viewer requires iframe.
	import os
	import sys
	import traceback
	from functools import wraps
	from multiprocessing import Process, Queue


	def processify(func):
	'''Decorator to run a function as a process.
	Be sure that every argument and the return value
	import pandas as pd
	import numpy as np
	from pathlib import Path
	import os
	from glob import glob
	import re
	import nltk

	# interactive download
	# nltk.download()
	nltk.download('punkt')

	extra_abbreviations = ['ps', 'inc', 'Corp', 'Ltd', 'Co', 'pkt', 'Dz.Ap', 'Jr', 'jr', 'sp', 'Sp', 'poj', 'pseud', 'krypt', 'sygn', 'Dz.U', 'ws', 'itd', 'np', 'sanskryt', 'nr', 'gł', 'Takht', 'tzw', 't.zw', 'ewan', 'tyt', 'oryg', 't.j', 'vs', 'l.mn', 'l.poj' ]

	position_abbrev = ['Ks', 'Abp', 'abp','bp','dr', 'kard', 'mgr', 'prof', 'zwycz', 'hab', 'arch', 'arch.kraj', 'B.Sc', 'Ph.D', 'lek', 'med', 'n.med', 'bł', 'św', 'hr', 'dziek' ]