Skip to content

Instantly share code, notes, and snippets.

@zilto
Last active February 6, 2024 01:51
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save zilto/d0426f15bb132d4df37cf5832578766b to your computer and use it in GitHub Desktop.
Save zilto/d0426f15bb132d4df37cf5832578766b to your computer and use it in GitHub Desktop.
class ExperimentTracker(
lifecycle.NodeExecutionHook,
lifecycle.GraphExecutionHook,
lifecycle.GraphConstructionHook,
):
def __init__(self, experiment_name: str, base_directory: str):
self.cache = Cache(base_directory)
self.init_directory = Path.cwd()
self.run_directory = resolve_path(base_directory)
self.materialized = list()
def post_graph_construct(self, config):
"""Collect the Driver config"""
self.config = config
def run_before_graph_execution(self, run_id, graph, inputs, overrides):
"""Gather metadata about the starting run"""
self.run_id = run_id
self.graph_hash = hash(graph)
self.inputs = get_input_nodes(graph.nodes)
self.overrides = overrides
def run_before_node_execution(self, node):
"""Move to run directory to produce materialized result"""
if is_materializer(node):
os.chdir(self.run_directory)
def run_after_node_execution(self, node):
"""Return to init directory after materialization. Log materialization"""
os.chdir(self.init_directory)
self.materializers.append(node)
def run_after_graph_execution(self):
"""Collect metadata stored in tracker and write to cache"""
metadata = get_run_metadata(self)
self.cache.write(self.run_id, metadata)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment