Skip to content

Instantly share code, notes, and snippets.

@smac89
Last active January 24, 2023 21:32
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save smac89/bdcb9b08fcdf9d055150824d57ab3513 to your computer and use it in GitHub Desktop.
Save smac89/bdcb9b08fcdf9d055150824d57ab3513 to your computer and use it in GitHub Desktop.
Read a large json file, but do so lazily so as to not use too much memory. Thanks to Java 8 streams, this is quite possible
Located here: https://gist.github.com/smac89/15fc3bffa4f965d18587eec2db4972dd
import com.google.common.io.Resources;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonObject;
import java.io.IOException;
import java.net.URL;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Stream;
public class Main {
public static void main(String args[]) {
URL data = Resources.getResource("data.json");
GsonBuilder builder = new GsonBuilder();
builder.registerTypeAdapter(Student.class, (JsonDeserializer<Student>) (json, typeOfT, context) -> {
JsonObject object = json.getAsJsonObject();
Student.Gender gender = Student.Gender.valueOf(object.get("gender").getAsString());
String firstName = object.get("first_name").getAsString();
String lastName = object.get("last_name").getAsString();
String email = object.get("email").getAsString();
String Id = object.get("id").getAsString();
int numberOfClasses = object.get("class_count").getAsInt();
return new Student(Id, firstName + " " + lastName, gender, email, numberOfClasses);
});
try (Stream<Student> studentStream = ReadJsonFile.readJsonFromFile(builder.create(), data,
Student.class)) {
studentStream.forEach(System.out::println);
} catch (IOException ioe) {
Logger.getAnonymousLogger().log(Level.WARNING, ioe,
() -> "An exception occurred when reading the json");
}
}
}
import com.google.gson.Gson;
import com.google.gson.stream.JsonReader;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.Type;
import java.net.URL;
import java.util.Iterator;
import java.util.Spliterators;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
public class ReadJsonFile {
private static final class JsonIterator<T> implements Iterator<T> {
private final Gson gson;
private final Type objectType;
private final JsonReader reader;
private JsonIterator(JsonReader reader, Gson gson, Type objectType) {
this.gson = gson;
this.objectType = objectType;
this.reader = reader;
}
@Override
public boolean hasNext() {
try {
return reader.hasNext();
} catch (IOException ioe) {
return false;
}
}
@Override
public T next() {
return gson.fromJson(reader, objectType);
}
}
public static <J> Stream<J> readJsonFromFile(Gson gson, URL jsonFile, Type type) throws IOException {
JsonReader reader = new JsonReader(
new BufferedReader(new InputStreamReader(jsonFile.openStream())));
reader.beginArray();
if (!reader.hasNext()) {
return Stream.empty();
}
return StreamSupport.stream(Spliterators.spliteratorUnknownSize(
new JsonIterator<J>(reader, gson, type), 0), false).onClose(() -> {
try {
reader.close();
} catch (IOException ioe) {
}
});
}
}
public class Student {
public final String studentId;
public final String name;
public final Gender gender;
public final String email;
public final Integer numberOfClasses;
public Student(String studentId, String name, Gender gender, String email, Integer numberOfClasses) {
this.studentId = studentId;
this.name = name;
this.gender = gender;
this.email = email;
this.numberOfClasses = numberOfClasses;
}
@Override
public String toString() {
return "Name: " + name + ", ID: " + studentId + ", Gender: " + gender.name() +
", email: " + email + ", Class count: " + String.valueOf(numberOfClasses);
}
public enum Gender {
Male, Female
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment