Created
April 22, 2021 11:47
-
-
Save Phil1108/e1821fec6eb746edc8e04ef5f76d23f1 to your computer and use it in GitHub Desktop.
GC4 Corpus Filtering Scripts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import gzip | |
import pathlib | |
import os | |
import pdb | |
from ast import literal_eval | |
from tqdm import tqdm | |
if __name__ == '__main__': | |
parent_dir = pathlib.Path("data_head_url") | |
for file in tqdm(parent_dir.iterdir()): | |
with gzip.open(file,'rt') as f: | |
a = f.readline() | |
a = a.split("{'url'") | |
a = [("{'url'" + item) for item in a] | |
b = [] | |
for item in tqdm(a): | |
try: | |
if literal_eval(item)['language_score'] > 0.98: | |
b.append(literal_eval(item)) | |
except: | |
None | |
with gzip.open(f"{file.name}_filtered.tar.gz", 'wt') as file_new: | |
for part in a[1:]: | |
file_new.write(part + '\n') |
I implemented some of the recommendations of @PhilipMay (only left out the tar one, simply changed the file extension) and added multiprocessing into this. Works quite well.
from multiprocessing import Pool
import json
import gzip
import pathlib
from ast import literal_eval
from tqdm import tqdm
def process_file(file):
with gzip.open(file, 'rt') as f:
a = f.readline()
a = a.split("{'url'")
a = [("{'url'" + item) for item in a]
b = []
for item in tqdm(a):
try:
evaluated_item = literal_eval(item)
if evaluated_item['language_score'] > 0.98:
b.append(evaluated_item)
except Exception as e:
pass
with gzip.open(parent_dir / f"{file.stem}_filtered.gz", 'wt') as file_new:
for part in b:
file_new.write(json.dumps(part) + '\n')
if __name__ == '__main__':
parent_dir = pathlib.Path("data_head_url")
files = list(parent_dir.iterdir())
with Pool(processes=4) as pool: # Adjust the number of workers as needed
list(tqdm(pool.imap(process_file, files), total=len(files)))
Thanks @sorgfresser
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Line 28 writes
a
but before that the script filtersa
to generateb
.The script should somehow write
b
.