Skip to content

Instantly share code, notes, and snippets.

@xinzhel
Last active November 14, 2021 23:13
Show Gist options
  • Save xinzhel/1bdd7b3f94539f83ce0d7beed320020a to your computer and use it in GitHub Desktop.
Save xinzhel/1bdd7b3f94539f83ce0d7beed320020a to your computer and use it in GitHub Desktop.
convert Reuters dataset on kaggle to json file
import logging
import os
import sys
import json
from typing import Dict, Optional
from tqdm.auto import tqdm
import timeit
import numpy as np
test = []
train = []
with open("cats.txt") as f:
content = f.readlines()
for line in tqdm(content):
line = line.replace("\n","")
line = line.split(" ")
file = line[0]
labels = line[1:]
with open(file, encoding="utf8", errors='ignore') as f:
data = f.read().replace('\n', '')
if 'test' in file:
test.append({"text" : data, "labels" : labels})
elif 'train' in file:
train.append({"text" : data, "labels" : labels})
else:
print("invalid file ", file)
with open('test.json', 'w') as outFile:
for d in test:
outFile.write(json.dumps(d))
outFile.write('\n')
with open('train.json', 'w') as outFile:
for d in train:
outFile.write(json.dumps(d))
outFile.write('\n')
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment