A silly PyO3 example to demonstrate memory allocation.

pyo3_bounded_mem.rs

#[pyfunction]
fn process_file(file: &PyAny, callback: &PyAny) {
    assert!(callback.is_callable());
    let file = file.to_string();
    if let Ok(lines) = read_lines(file) {
        for line in lines {
            Python::with_gil(|py| {
                let pool = unsafe { py.new_pool() };
                let py = pool.python();
                for msg in line.unwrap().replace("}{", "}\n{").split("\n") {
                    let mut msg = msg.to_owned();
                    let obj: Value = unsafe { simd_json::serde::from_str(&mut msg).expect("JSON parsing failed") };
                    let py_msg = PyDict::new(py);
                    for (key, value) in obj.as_object().unwrap() {
                        py_msg.set_item(key, value).expect("failed to set key");
                    }
                    callback.call(PyTuple::new(py, vec![py_msg]), None).expect("failed to invoke callback");
                }
            });
        }
    }
}

pyo3_unbounded_mem.rs

#[pyfunction]
fn process_file(file: &PyAny, callback: &PyAny) {
    assert!(callback.is_callable());
    let file = file.to_string();
    if let Ok(lines) = read_lines(file) {
        for line in lines {
            for msg in line.unwrap().replace("}{", "}\n{").split("\n") {
                let mut msg = msg.to_owned();
                let obj: Value = unsafe { simd_json::serde::from_str(&mut msg).expect("JSON parsing failed") };
                let py_msg = PyDict::new(py);
                for (key, value) in obj.as_object().unwrap() {
                    py_msg.set_item(key, value).expect("failed to set key");
                }
                callback.call(PyTuple::new(py, vec![py_msg]), None).expect("failed to invoke callback");
            }
        }
    }
}