Write pickle to and read from S3 and local file system

write_read_pickle_example.py

from typing import Any
import fsspec
import pickle


def write_pickle(data: Any, path: str) -> None:
    with fsspec.open(path, "wb") as file:
        pickle.dump(data, file)

def read_pickle(path: str) -> Any:
    with fsspec.open(path, "rb") as file:
        return pickle.load(file)


data = {"a": 1}
write_pickle(data, "s3://my_bucket/data.tmp")
write_pickle(data, "file:///tmp/data.tmp")

assert read_pickle("s3://my_bucket/data.tmp") == data
assert read_pickle("file:///tmp/data.tmp") == data