joeltok

dictionary_of_lists = df.to_dict(orient='list')
total_size = 0
total_size += sys.getsizeof(dictionary_of_lists)
for key in dictionary_of_lists.keys():
    total_size += sys.getsizeof(key)
    total_size += sys.getsizeof(dictionary_of_lists[key])
    for item in dictionary_of_lists[key]:
        total_size += sys.getsizeof(item)

print(f'{total_size:,} B')

joeltok

import sys

total_size = 0
total_size += sys.getsizeof(list_of_dictionaries)
for dictionary in list_of_dictionaries:
    total_size += sys.getsizeof(dictionary)
    for key, value in zip(dictionary.keys(), dictionary.values()):
            total_size += sys.getsizeof(key)
            total_size += sys.getsizeof(value)

joeltok

import pandas as pd

df = pd.DataFrame(list_of_dictionaries)
mem = df.memory_usage(index=True, deep=True)
total_size = sum(mem)

print(f'{total_size:,} B')
print(f'{total_size/1000000:.2f} MB')

joeltok

import random
random.seed(123)

breeds = [
    'German Shepherd',
    'Golden Retriever',
    'Siberian Husky',
    'Japanese Spitz',
    'Sleeping Whippets',
    'Samoyed',

joeltok

transformable_obj.transform(lambda x: pickle.dumps(x, protocol=4))

joeltok

class MetaflowTask(object):

    ...
    
    def run_step(self,
             step_name,
             run_id,
             task_id,
             origin_run_id,
             input_paths,

joeltok

$> python flow.ipynb run

Network(
  (fc1): Linear(in_features=2, out_features=2, bias=True)
  (fc2): Linear(in_features=2, out_features=1, bias=True)
)
Parameter containing:
tensor([[-0.6120, -0.6511],
        [-0.3616, -0.3564]], requires_grad=True)
Parameter containing:

joeltok

# flow.ipynb

from metaflow import Flow
from inputs import inputs

flow = Flow('ExperimentalPyTorchFlow')
run = flow.latest_successful_run

print(run['a'].task.data.model)
for param in run['a'].task.data.model.parameters():

joeltok

$> python flow.py run

Network(
    (fc1): Linear(in_features=2, out_features=2, bias=True)
    (fc2): Linear(in_features=2, out_features=1, bias=True)
)
tensor([[-0.6120, -0.6511],
        [-0.3616, -0.3564]])
tensor([ 0.1328, -0.3496])
tensor([[0.4353, 0.4610]])

joeltok

# flow.py

from metaflow import FlowSpec, step, Parameter
from Network import Network
from inputs import inputs

class ExperimentalPyTorchFlow(FlowSpec):
    @step
    def start(self):
        self.model = Network()