Skip to content

Instantly share code, notes, and snippets.

@hniksic

hniksic/main.rs Secret

Last active August 5, 2021 20:01
Show Gist options
  • Save hniksic/8928ce950f51cabf2798b798c4688a80 to your computer and use it in GitHub Desktop.
Save hniksic/8928ce950f51cabf2798b798c4688a80 to your computer and use it in GitHub Desktop.
Workaround without channels
use serde::de::DeserializeOwned;
use serde_json::{self, Deserializer};
use std::io::{self, Read};
fn read_skipping_ws(mut reader: impl Read) -> io::Result<u8> {
loop {
let mut byte = 0u8;
reader.read_exact(std::slice::from_mut(&mut byte))?;
if !byte.is_ascii_whitespace() {
return Ok(byte);
}
}
}
fn invalid_data(msg: &str) -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, msg)
}
fn deserialize_single<T: DeserializeOwned, R: Read>(reader: R) -> io::Result<T> {
let next_obj = Deserializer::from_reader(reader).into_iter::<T>().next();
match next_obj {
Some(result) => result.map_err(Into::into),
None => Err(invalid_data("premature EOF")),
}
}
fn yield_next_obj<T: DeserializeOwned, R: Read>(
mut reader: R,
at_start: &mut bool,
) -> io::Result<Option<T>> {
if !*at_start {
*at_start = true;
if read_skipping_ws(&mut reader)? == b'[' {
// read the next char to see if the array is empty
let peek = read_skipping_ws(&mut reader)?;
if peek == b']' {
Ok(None)
} else {
deserialize_single(io::Cursor::new([peek]).chain(reader)).map(Some)
}
} else {
Err(invalid_data("`[` not found"))
}
} else {
match read_skipping_ws(&mut reader)? {
b',' => deserialize_single(reader).map(Some),
b']' => Ok(None),
_ => Err(invalid_data("`,` or `]` not found")),
}
}
}
pub fn iter_json_array<T: DeserializeOwned, R: Read>(
mut reader: R,
) -> impl Iterator<Item = Result<T, io::Error>> {
let mut at_start = false;
std::iter::from_fn(move || yield_next_obj(&mut reader, &mut at_start).transpose())
}
use serde::Deserialize;
#[derive(Deserialize, Debug)]
struct MyJson {
val1: String,
val2: Vec<i32>,
}
fn main() {
let data = setup_test_data();
for my_json in iter_json_array::<MyJson, _>(std::io::Cursor::new(data)) {
let _ = my_json.unwrap();
}
}
fn setup_test_data() -> Vec<u8> {
let mut data = vec![];
data.push(b'[');
for _ in 0..1_000_000 {
data.extend_from_slice(
br#"
{
"val1": "one",
"val2": [
0
]
},
{
"val1": "two",
"val2": [
1
]
},
{
"val1": "three",
"val2": [
2
]
},
"#,
);
}
data.extend_from_slice(
br#"{
"val1": "final",
"val2": [
3
]
}
]"#,
);
data
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment