GC4 Corpus Filtering Scripts

Dataset_filtering.py

import json
import gzip 
import pathlib 
import os 
import pdb 
from ast import literal_eval
from tqdm import tqdm

if __name__ == '__main__': 
    parent_dir = pathlib.Path("data_head_url")

    for file in tqdm(parent_dir.iterdir()):
        with gzip.open(file,'rt') as f:
            a = f.readline()
            
        a = a.split("{'url'")
        a = [("{'url'" + item) for item in a]
        
        b = []
        for item in tqdm(a): 
            try: 
                if literal_eval(item)['language_score'] > 0.98:
                    b.append(literal_eval(item))
            except: 
                None
        
        with gzip.open(f"{file.name}_filtered.tar.gz", 'wt') as file_new:
            for part in a[1:]: 
                file_new.write(part + '\n')

Thanks @sorgfresser