webz-dataset-text-extraction.py

# https://webz.io/free-datasets/
# Spanish: https://s3.amazonaws.com/webhose-archive/datasets/645_20170904091816.zip
# Extract text from the JSON files


import os
import json
from sentence_splitter import split_text_into_sentences
from tqdm import tqdm


filenames = []

root = "."
for item in os.listdir(root):
  if os.path.isfile(os.path.join(root, item)):
    if item.endswith(".json"):
      filenames.append(item)

filenames.sort()
print("Number of files:", len(filenames))

count = 0

for filename in tqdm(filenames):
  with open(filename, "r") as jsn, open("spanish-news-corpus.es", "a") as outputfilename:
    output = json.load(jsn)
    text = output["text"]
    lines = split_text_into_sentences(text=text, language="es")
    outputfilename.write("\n".join(line.strip() for line in lines) + "\n")
    count += len(lines)

print("Number of lines", count)