-
-
Save hniksic/8928ce950f51cabf2798b798c4688a80 to your computer and use it in GitHub Desktop.
Workaround without channels
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
use serde::de::DeserializeOwned; | |
use serde_json::{self, Deserializer}; | |
use std::io::{self, Read}; | |
fn read_skipping_ws(mut reader: impl Read) -> io::Result<u8> { | |
loop { | |
let mut byte = 0u8; | |
reader.read_exact(std::slice::from_mut(&mut byte))?; | |
if !byte.is_ascii_whitespace() { | |
return Ok(byte); | |
} | |
} | |
} | |
fn invalid_data(msg: &str) -> io::Error { | |
io::Error::new(io::ErrorKind::InvalidData, msg) | |
} | |
fn deserialize_single<T: DeserializeOwned, R: Read>(reader: R) -> io::Result<T> { | |
let next_obj = Deserializer::from_reader(reader).into_iter::<T>().next(); | |
match next_obj { | |
Some(result) => result.map_err(Into::into), | |
None => Err(invalid_data("premature EOF")), | |
} | |
} | |
fn yield_next_obj<T: DeserializeOwned, R: Read>( | |
mut reader: R, | |
at_start: &mut bool, | |
) -> io::Result<Option<T>> { | |
if !*at_start { | |
*at_start = true; | |
if read_skipping_ws(&mut reader)? == b'[' { | |
// read the next char to see if the array is empty | |
let peek = read_skipping_ws(&mut reader)?; | |
if peek == b']' { | |
Ok(None) | |
} else { | |
deserialize_single(io::Cursor::new([peek]).chain(reader)).map(Some) | |
} | |
} else { | |
Err(invalid_data("`[` not found")) | |
} | |
} else { | |
match read_skipping_ws(&mut reader)? { | |
b',' => deserialize_single(reader).map(Some), | |
b']' => Ok(None), | |
_ => Err(invalid_data("`,` or `]` not found")), | |
} | |
} | |
} | |
pub fn iter_json_array<T: DeserializeOwned, R: Read>( | |
mut reader: R, | |
) -> impl Iterator<Item = Result<T, io::Error>> { | |
let mut at_start = false; | |
std::iter::from_fn(move || yield_next_obj(&mut reader, &mut at_start).transpose()) | |
} | |
use serde::Deserialize; | |
#[derive(Deserialize, Debug)] | |
struct MyJson { | |
val1: String, | |
val2: Vec<i32>, | |
} | |
fn main() { | |
let data = setup_test_data(); | |
for my_json in iter_json_array::<MyJson, _>(std::io::Cursor::new(data)) { | |
let _ = my_json.unwrap(); | |
} | |
} | |
fn setup_test_data() -> Vec<u8> { | |
let mut data = vec![]; | |
data.push(b'['); | |
for _ in 0..1_000_000 { | |
data.extend_from_slice( | |
br#" | |
{ | |
"val1": "one", | |
"val2": [ | |
0 | |
] | |
}, | |
{ | |
"val1": "two", | |
"val2": [ | |
1 | |
] | |
}, | |
{ | |
"val1": "three", | |
"val2": [ | |
2 | |
] | |
}, | |
"#, | |
); | |
} | |
data.extend_from_slice( | |
br#"{ | |
"val1": "final", | |
"val2": [ | |
3 | |
] | |
} | |
]"#, | |
); | |
data | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment