Arrow-rs design

sketch.rs

enum ArrowType { 
   Int64,
   List(Box<ArrowType>),
   Struct(Vec<(String, Box<ArrowType>)>) 
}

struct Array<T: DataType> {
    dtype: ArrayType,
    store: T::Container,
    _marker: PhantomData<T>
}

struct PrimitiveStorage<T> {
    len: usize,
    values: RawVec<T>
}
    
struct ListStorage<T> {
    len: usize,
    offsets: RawVec<usize>,
    values: Array<T>
}

trait DataType : Copy {
    type Container: Storage;
    type RawValue;
    
    fn dtype() -> ArrowType;
}

trait PrimitiveType : DataType {}
trait ListType : DataType {}
trait StructType : DataType {}


trait Storage<T: DataType> {

    fn empty(dtype: T) -> Self;

}

trait ArrayLike<T: DataType> {

    fn new() -> Self;
    //fn push(&mut self, value: T::RawValue);
    
}

impl<T: DataType> Storage<T> for PrimitiveStorage<T> {
    fn empty() -> Self {
        Self {
            len: 0,
            values: RawVec::new()
        }
    }
}

impl<T: PrimitiveType> ArrayLike<T> for Array<T> {
    
    fn new() {
        Array {
            dtype: T::dtype(),
            store: T::DataStore::empty()
            _marker: PhantomData
        }
    }
    
}

impl Array<T> {
    fn from_dtype(dtype: ArrayType) -> Self {
        // what goes here?   
    }
}


impl DataType for i64 {
    type Container = PrimitiveData<i64>;
    type RawValue = i64;
    
    fn dtype() -> ArrowType {
        ArrowType::Int64   
    }
}

impl DataType for ArrowType {
    type Container = ???;
    type RawValue = Box<Any>;
}

// static primitive
Array::<i64>::new();  //it seems ok

// dynamic primitive
Array::from_dtype(ArrowType::Int64);  // ?

// static list
Array::<Vec<i64>>::new();

// dynamic list
let dtype = ArrowType::List(Box::new(ArrowType::Int64));
Array::from_dtype(dtype);

// static struct
#[derive(ArrowDataType)])
struct MyStruct {
    a: i64,
    blist: Vec<i64>
}

// dynamic struct
let dtype = ArrowType::Struct(vec![
     ("a", ArrowType::Int64),
     ("blist", ArrowType::List(Box::new(ArrowType::Int64)))
]);
Array::from_dtype(dtype);