View gist:e1ec39fde5971887fff7222830e7c437
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas as pd | |
from dataset import Dataset | |
data = {'Name':['Tom', 'nick', 'krish', 'jack'],'Age':[20, 21, 19, 18]} | |
df = pd.DataFrame(data) | |
dataset = Dataset(id='8fba0c5b-4792-4bc1-a8d6-3eea6cc5d086', | |
name='ppl_dataset', | |
dataframe=df) | |
encoded_dataset = dataset.json() | |
print(encoded_dataset) |
View gist:22d71e1f95f77d42c583fe5919c03a9b
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from pandas import DataFrame | |
import pyarrow as pa | |
import jsonpickle | |
def serialize_with_pyarrow(dataframe: DataFrame): | |
batch = pa.record_batch(dataframe) | |
write_options = pa.ipc.IpcWriteOptions(compression="zstd") | |
sink = pa.BufferOutputStream() | |
with pa.ipc.new_stream(sink, batch.schema, options=write_options) as writer: | |
writer.write_batch(batch) |
View gist:0bdb6aadf059f8346ab8bc85199879ff
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas | |
from pydantic import BaseModel | |
class CustomBaseModel(BaseModel): | |
class Config: | |
arbitrary_types_allowed = True | |
json_encoders = { | |
pandas.DataFrame: lambda v: serialize_with_pyarrow(v) | |
} |
View gist:933def0ad41a5db7bed66e29acebb096
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas | |
from pydantic import BaseModel | |
class Dataset(BaseModel): | |
id: str | |
name: constr(max_length=128) | |
dataframe: pandas.DataFrame | |
@validator('id') | |
def is_uuid4_string(cls, value): |
View gist:2f0b37d50fd43a22b9b1a833845801e4
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import pandas | |
from pydantic import BaseModel | |
class Dataset(BaseModel): | |
id: str | |
name: str | |
dataframe: pandas.DataFrame |