Question: can parallel pre- and postprocessing speed up Gensim Doc2Vec?
(only one run, so not a very scientific comparison)
Run on a single machine with 16GB RAM and Intel i7-8550U CPU @ 1.80GHz
This gist contains the first three lines from the input file, which for this example has 200k lines and 457MB
| from pyspark import SparkContext, SparkConf | |
| import ujson as json | |
| from gensim.models.doc2vec import TaggedDocument | |
| from gensim.parsing import preprocessing as pp | |
| from gensim import utils | |
| from gensim.models.doc2vec import Doc2Vec | |
| import logging | |
| import timeit | |
| import os | |
| filter_funcs = [ | |
| lambda s: s.lower(), | |
| pp.strip_tags, | |
| pp.strip_punctuation, | |
| pp.strip_multiple_whitespaces, | |
| pp.strip_numeric, | |
| pp.remove_stopwords, | |
| lambda s: pp.strip_short(s, 3), | |
| # pp.stem_text, | |
| pp.strip_non_alphanum | |
| ] | |
| def norm_doc(doc): | |
| s = utils.to_unicode(doc['paperAbstract']) | |
| for f in filter_funcs: | |
| s = f(s) | |
| doc['normed'] = s | |
| return doc | |
| def lookup_infer(doc): | |
| try: | |
| vector = bc_model.value.docvecs[doc['id']] | |
| except KeyError: | |
| td = to_tagged_doc(doc) | |
| vector = bc_model.value.infer_vector(td.words, epochs=5) | |
| doc['d2v'] = vector.tolist() | |
| return doc | |
| def filter_fields(doc): | |
| return { | |
| 'id': doc['id'], | |
| 'd2v': doc['d2v'], | |
| 'normed': doc['normed'] | |
| } | |
| def to_tagged_doc(doc): | |
| return TaggedDocument(doc['normed'].split(), [doc['id']]) | |
| class TDs: | |
| def __init__(self, data): | |
| self.data = data | |
| def __iter__(self): | |
| print('restart localIterator') | |
| docs = self.data.toLocalIterator() | |
| for _, sample in enumerate(docs): | |
| yield to_tagged_doc(sample) | |
| print('finished iterating localIterator') | |
| time0 = timeit.default_timer() | |
| conf = SparkConf() \ | |
| .setAppName('pipeline') \ | |
| .setMaster('local[*]') \ | |
| .set("spark.files.overwrite", "true") \ | |
| .set('spark.hive.mapred.supports.subdirectories', 'true') \ | |
| .set('spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive', 'true') \ | |
| .set('spark.driver.memory', '2g') \ | |
| .set('spark.executor.memory', '1g') \ | |
| .set('spark.executor.cores', '2') \ | |
| .set('spark.executor.instances', '2') | |
| ctx = SparkContext(conf=conf) | |
| data_source = '../data/s2-200000.json' | |
| model_file = '../data/pipeline/tmp/d2v.pickle' | |
| data_sink = '../data/pipeline/tmp/d2v' | |
| rdd = ctx.textFile(data_source, minPartitions=8).map(json.loads) | |
| rdd = rdd.map(norm_doc) | |
| if os.path.isfile(model_file): | |
| model = Doc2Vec.load(model_file) | |
| else: | |
| t_docs = TDs(rdd) | |
| model = Doc2Vec(documents=t_docs, min_count=2, max_vocab_size=10000, | |
| vector_size=48, epochs=5, workers=8) | |
| model.save(model_file) | |
| bc_model = ctx.broadcast(model) | |
| rdd = rdd.map(lookup_infer) | |
| rdd = rdd.map(filter_fields) | |
| rdd = rdd.map(json.dumps) | |
| rdd.saveAsTextFile(data_sink) | |
| print(f'Time: {timeit.default_timer() - time0:.1f}s') |
| import ujson as json | |
| from gensim.models.doc2vec import TaggedDocument | |
| from gensim.parsing import preprocessing as pp | |
| from gensim import utils | |
| from gensim.models.doc2vec import Doc2Vec | |
| import logging | |
| import os | |
| import timeit | |
| class NormData: | |
| FILTER_FUNCS = [ | |
| lambda s: s.lower(), | |
| pp.strip_tags, | |
| pp.strip_punctuation, | |
| pp.strip_multiple_whitespaces, | |
| pp.strip_numeric, | |
| pp.remove_stopwords, | |
| lambda s: pp.strip_short(s, 3), | |
| # pp.stem_text, | |
| pp.strip_non_alphanum | |
| ] | |
| def __init__(self, filename): | |
| self.filename = filename | |
| def __iter__(self): | |
| print('start norm') | |
| with open(self.filename, 'r') as f: | |
| for line in f: | |
| doc = json.loads(line) | |
| s = utils.to_unicode(doc['paperAbstract']) | |
| for func in self.FILTER_FUNCS: | |
| s = func(s) | |
| doc['normed'] = s | |
| yield doc | |
| class TDs: | |
| def __init__(self, data): | |
| self.data = data | |
| def __iter__(self): | |
| print('start td') | |
| for doc in self.data: | |
| yield TaggedDocument(doc['normed'].split(), [doc['id']]) | |
| class Vectoring: | |
| def __init__(self, d2v_model, data): | |
| self.model = d2v_model | |
| self.data = data | |
| def __iter__(self): | |
| print('start vectoring') | |
| for doc in self.data: | |
| try: | |
| vector = self.model.docvecs[doc['id']] | |
| except KeyError: | |
| td = TaggedDocument(doc['normed'].split(), [doc['id']]) | |
| vector = self.model.infer_vector(td.words, epochs=5) | |
| doc['d2v'] = vector.tolist() | |
| yield doc | |
| class Filter: | |
| def __init__(self, data): | |
| self.data = data | |
| def __iter__(self): | |
| print('start filter') | |
| for doc in self.data: | |
| yield { | |
| 'id': doc['id'], | |
| 'd2v': doc['d2v'], | |
| 'normed': doc['normed'] | |
| } | |
| def save(filename, data): | |
| print('saving') | |
| with open(filename, 'w') as f: | |
| for doc in data: | |
| f.write(json.dumps(doc) + '\n') | |
| data_source = '../data/s2-200000.json' | |
| model_file = '../data/pipeline/tmp/d2v.pickle' | |
| data_sink = '../data/pipeline/tmp/d2v' | |
| time0 = timeit.default_timer() | |
| normed = NormData(data_source) | |
| if os.path.isfile(model_file): | |
| model = Doc2Vec.load(model_file) | |
| else: | |
| t_docs = TDs(normed) | |
| model = Doc2Vec(documents=t_docs, min_count=2, max_vocab_size=10000, | |
| vector_size=48, epochs=5, workers=8) | |
| model.save(model_file) | |
| vec = Vectoring(model, normed) | |
| filtered = Filter(vec) | |
| save(data_sink, filtered) | |
| print(f'Time: {timeit.default_timer() - time0:.1f}s') |
| {"entities":["Jack Device Component"],"journalVolume":"109 49-50","journalPages":"26","pmid":"24568020v1","year":2013,"outCitations":[],"s2Url":"https://semanticscholar.org/paper/f2320c08c7d95bbf8bb72e4d6deaa6845ea4cf27","s2PdfUrl":"","id":"f2320c08c7d95bbf8bb72e4d6deaa6845ea4cf27","authors":[{"name":"Kate Jack","ids":["38280253"]}],"journalName":"Nursing times","paperAbstract":"","inCitations":[],"pdfUrls":[],"title":"60 seconds with Kate Jack.","doi":"","sources":["Medline"],"doiUrl":"","venue":"Nursing times"} | |
| {"entities":["Decision Making","Laboratory Certification Document","Organization administrative structures"],"journalVolume":"31 2","journalPages":"127-30","pmid":"3514907v1","year":1986,"outCitations":[],"s2Url":"https://semanticscholar.org/paper/5432a99cdd9f8b248c50274cd3d2a6016f3d081e","s2PdfUrl":"","id":"5432a99cdd9f8b248c50274cd3d2a6016f3d081e","authors":[{"name":"W N Spellacy","ids":["5862934"]},{"name":"J Burger","ids":["1757639"]}],"journalName":"The Journal of reproductive medicine","paperAbstract":"The search for new administrators in complex systems is an important activity. The special requirements of academic organizations, particularly those with health centers, present some unique considerations that can confound this important and difficult process. Typically, national searches attract a sizable candidate list composed of persons with diverse backgrounds and experiences, and a committee is empowered to sort through their qualifications. A critical step in the planning of each search is the development of a process that allows participatory decision making while not requiring too much time. Too often the search becomes an unmanageable activity that confuses the searchers and frustrates the administration. A seven-step process has proven successful for use by committees to attract and sort through written candidate applications, to agree upon a preliminary ranking of candidates and to reach a consensus on a final list of recommendations. The process could be applied in almost any organizational setting.","inCitations":[],"pdfUrls":[],"title":"Organizing a search for an academic administrator.","doi":"","sources":["Medline"],"doiUrl":"","venue":"The Journal of reproductive medicine"} | |
| {"entities":["Annexin A1","Annexins","Bacterial Infections","Chemotaxis","EPHA3 protein, human","FPR1 protein, human","FPR3 gene","Leukocytes","Ligands","Parathyroid Hormone Receptor","Peptide Receptor","Subfamily","fMet-Leu-Phe receptor"],"journalVolume":"172 12","journalPages":"7669-76","pmid":"15187149v1","year":2004,"outCitations":["c2b53b26c004fe57e85424df6ad101d283150648","d30e703f3d6d74bfc6845c9469d681e491bac5fe","6494f3ba9340ff0bf04bf6fcaee639a16c273d12","81e8afe731c3fceac456ef28027f96b40439e5e8","56101af10d0b74526d0365f850877a51c0e0866b","32ef6bd7082c817934c0163abb3e6d3e818c64d7","2f97ef8f51637774f47dcfdc5fc05ef728ed24b6","d21899d415d539ce3fad09032ad27afba84bcfd2","3929e76314143450e905f954c6226da5ced68954","5e69462299d07cac473aa1a864dd51cae398af9c","315a775b3219ae31367df439080806a1666b1624","e658ebc0cbb4443d1f79a1d6bc92111d9f536c98","e2164cb09414781bd46cb0d403a72a0ea5793682","43ef3abacf752f00d35ecdb4b08e32f0370d7abb","3123c56dadd34503128e73d556d0308748932c56","ec3405c7b34b3b9def5f87fe1ca0c9f0b37d4644","97ba43f452c3eaf3e994f39e426f86b3594369d1","4c23424e376c16e7fd501fc8742f51881e57d25b","497b0cfb888f36f32c4b4843aee01c5509b02123","f5ec2d092c6969299eeb2842060d6cab98613f73","c0f339442b3f84567f1f9ea05805480d5295c4a8","af02aed0e5bff33617f96d840e36ff065cf469ed","8559addff0abfeefbefeb7e7cff8667891adc0a3","941e38df2a1198b14dfc82aaa4cf6db12704004d","ad7b4b77fe7f98c6799d5a569db0380a6d352928","c5cd33ac650b3b7bbd6a4e8afc4ab89681ef16b0","136c57be9e4c36f8ba36e1966647724b27577ab9","6bc51ccd63cb18d5b950f0067d60d000c653696f","cbefdfe5df7696f92cd02ca8f27e768ab8597bed","bb2412da92ddae7063227e61c9e5cff080557e13","ed55dded3d9cb58e5b630840144548fe4c9b30f7","9aa2ae2573c701e5855977d7f354ee1b957e0fc3","050d7c0cc34d6dbd2485fca72e53498725f583fb","3e9a27b239b8b33259332b41b5541ee12e8fc8b1","614653777ab95341368b82744e40d079cce1599e","425ed8ee2d3c490ce1e2253d28fb6d5d447f2487","36063ad87d5de2293e63b26df1bb766ad10d19b0","32b823d9c492a2c1455b9cd2f292aa5d4002198f","c23f24bf3c29e2e48ea832809cd6da04293783d6","6d62795c42bc4397801968dd430303b56968d4b3","0fc68bb34163586d66d526c5e69792b038b01b8c","6a9dc9e2d9d3224d33dcc49d48c4cd60a4fd0a6a","bc0ab72643f01c9f3bbf957a9738c682da314d8a","32ef2f7cd4d56642e1991a39379738ae711fd231"],"s2Url":"https://semanticscholar.org/paper/155663331ea93379e99997bd43340eb54ab41a73","s2PdfUrl":"http://pdfs.semanticscholar.org/cb73/147dc0bf1b2eabc5513c5e7952ef4f36df16.pdf","id":"155663331ea93379e99997bd43340eb54ab41a73","authors":[{"name":"Stefanie Ernst","ids":["39900230"]},{"name":"Carsten Lange","ids":["38285107"]},{"name":"Andreas Wilbers","ids":["34505589"]},{"name":"Verena Goebeler","ids":["6302762"]},{"name":"Volker Gerke","ids":["6509600"]},{"name":"Ursula Rescher","ids":["4702003"]}],"journalName":"Journal of immunology","paperAbstract":"The human N-formyl peptide receptor (FPR) is a key modulator of chemotaxis directing granulocytes toward sites of bacterial infections. FPR is the founding member of a subfamily of G protein-coupled receptors thought to function in inflammatory processes. The other two members, FPR-like (FPRL)1 and FPRL2, have a greatly reduced affinity for bacterial peptides or do not bind them at all, with FPRL2 being considered an orphan receptor so far. In this study we show that a peptide derived from the N-terminal domain of the anti-inflammatory protein annexin 1 (lipocortin 1) can activate all three FPR family members at similar concentrations. The annexin 1 peptide initiates chemotactic responses in human monocytes that express all three FPR family members and also desensitizes the cells toward subsequent stimulation with bacterial peptide agonists. Experiments using HEK 293 cells stably expressing a single FPR family member reveal that all three receptors can be activated and desensitized by the N-terminal annexin 1 peptide. These observations identify the annexin 1 peptide as the first endogenous ligand of FPRL2 and indicate that annexin 1 participates in regulating leukocyte emigration into inflamed tissue by activating and desensitizing different receptors of the FPR family.","inCitations":["3738fad17126054f03cfe736b7156b6d6eef0481","927e690d772252cc95f7818a16a4cf43dab0edba","12a2c7031f12df3d00bfe5eaa9ae25851b825841","2f49fc254663c9b5b6e2fbc2b3e707eee5817c11","05b55180032103e7b31de248e227cb8d239d8acd","3f07ff1ee2813dd2c38b5bc28439769226bd1236","6c7ee2062b49226bfee08114f12cb1cd07509d9d","d1d3f4cc5671bbb339b43e63c065477c7c1f38c8","a986984b4e368907c36e0e078a935adcfd055076","bafabb8269cafefd06e039fed0833bab13f34afd","2d3bcb942422b792fa6242b4c79a7c5f2bc898e9","35767947edf698fbe772e0fdb545fe850b5b8d15","dbca6c002f78b9926459ce9b5bc87fc4738ad996","d32f93ced0ba660cf39c200b0186882ab1a149d1","b153a0589f5d2dbb8f21016cc202817bb1f937b4","5cbf95c2ff01c69e1be9da8a12324cd57ac73d82","17b36fa12edad261e163b414528b3edd01bddd28","24e442087f3533f64551f1a77b075e0be59b1f40","6057c5e6eba3b636711396ec2bb5e1d93c44632c","febd5b9457b8cfef884fa06b263d9117e23cf7be","031b6cb1f0f7a5c2d6a3aa08d83350b1579a2a08","2c282d7e1a95c41a0e6ea87f2ea44925fff0cc26","d18197d7be7433b5c253eac0d232ea00eac02c49","a647ec16597bedfbc94897042a45981b2694b32b","f9965f7617c5caea715fb749c4a964b141fe1ef2","10335c92eaf3130d9524170c2dfa1539590c3a30","b368b1c975450a779fa9ec9c75259914304f8928","2f01ccdec197a25d3a8840f7ac097d1f1c58c9ae","b677b11c5fe125e2d6b9a23f2f1c71e8f7253866","56a98d415a52893a9311de921f3332a4d5983f86","2759f0793ac28fcb6144a232d74a176ca0001a72","cfc4ccc8d6d7f541199324e80ff737ec32e90a04","210ae1d7c6386b4ffe07f6ae1065f567844da7d3","1910925eb67b1178271f889ed7ce3706ce51de01","e4d49a8d7d477165428e6c08b1066dc6d7251a4c","73ab48562b828beecd18d0b8b85eb8eac08b7f46","6bd3a7dfcd8b5582c6f3a37f6b21ee966d11d0a1","e10fd96b2c66623159ca50a126e6c398e9681f93","61d89c64d595992624b8d6e281174df893ec0311","06c2a40ad3b98bc414dff20c882867116082e7ac","38449740092b71d204a7f27aa68cf1f7a0a08b4b","c8f8030cadd63bfe6e24918aa12391fee171aa45","697729db06b9b4c70867cc6c8bce6416a7c5621d","0dfec70f31ae2294602602eba6066ac11e404a68","7b95f3b889c0428e3bfa16afa95d97d1f9d7c0f9","227944c357595eab0bb4bf8f73ddb0c8db35e2fa"],"pdfUrls":["http://www.jimmunol.org/content/jimmunol/172/12/7669.full.pdf","http://www.jimmunol.org/content/172/12/7669.full.pdf"],"title":"An annexin 1 N-terminal peptide activates leukocytes by triggering different members of the formyl peptide receptor family.","doi":"","sources":["Medline"],"doiUrl":"","venue":"Journal of immunology"} |