Created
April 22, 2021 11:47
-
-
Save Phil1108/e1821fec6eb746edc8e04ef5f76d23f1 to your computer and use it in GitHub Desktop.
GC4 Corpus Filtering Scripts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import gzip | |
import pathlib | |
import os | |
import pdb | |
from ast import literal_eval | |
from tqdm import tqdm | |
if __name__ == '__main__': | |
parent_dir = pathlib.Path("data_head_url") | |
for file in tqdm(parent_dir.iterdir()): | |
with gzip.open(file,'rt') as f: | |
a = f.readline() | |
a = a.split("{'url'") | |
a = [("{'url'" + item) for item in a] | |
b = [] | |
for item in tqdm(a): | |
try: | |
if literal_eval(item)['language_score'] > 0.98: | |
b.append(literal_eval(item)) | |
except: | |
None | |
with gzip.open(f"{file.name}_filtered.tar.gz", 'wt') as file_new: | |
for part in a[1:]: | |
file_new.write(part + '\n') |
Line 28 writes a
but before that the script filters a
to generate b
.
The script should somehow write b
.
I implemented some of the recommendations of @PhilipMay (only left out the tar one, simply changed the file extension) and added multiprocessing into this. Works quite well.
from multiprocessing import Pool
import json
import gzip
import pathlib
from ast import literal_eval
from tqdm import tqdm
def process_file(file):
with gzip.open(file, 'rt') as f:
a = f.readline()
a = a.split("{'url'")
a = [("{'url'" + item) for item in a]
b = []
for item in tqdm(a):
try:
evaluated_item = literal_eval(item)
if evaluated_item['language_score'] > 0.98:
b.append(evaluated_item)
except Exception as e:
pass
with gzip.open(parent_dir / f"{file.stem}_filtered.gz", 'wt') as file_new:
for part in b:
file_new.write(json.dumps(part) + '\n')
if __name__ == '__main__':
parent_dir = pathlib.Path("data_head_url")
files = list(parent_dir.iterdir())
with Pool(processes=4) as pool: # Adjust the number of workers as needed
list(tqdm(pool.imap(process_file, files), total=len(files)))
Thanks @sorgfresser
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I saw that this extracts the gz part but not the tar part. IMO something like this is better to get the content: