Skip to content

Instantly share code, notes, and snippets.

Show Gist options
  • Save saumalya75/742452368be92a2be644c7cf2aeffe47 to your computer and use it in GitHub Desktop.
Save saumalya75/742452368be92a2be644c7cf2aeffe47 to your computer and use it in GitHub Desktop.
Date-Time configuration handling
try:
dag_config = pipeline_config['dag']
if dag_config['schedule_interval_unit'] and dag_config['schedule_interval_val']:
dag_config['schedule_interval'] = _get_timedelta(
dag_config['schedule_interval_unit']
, dag_config['schedule_interval_val']
)
del(dag_config['schedule_interval_unit'])
del(dag_config['schedule_interval_val'])
else:
dag_config['schedule_interval'] = _get_timedelta('days', 1)
default_args = pipeline_config['dag']['default_args']
default_args = default_args
default_args['start_date'] = datetime(
default_args['start_date']['year'],
default_args['start_date']['month'],
default_args['start_date']['day']
)
if default_args['retry_delay_unit'] and default_args['retry_delay_val']:
default_args['retry_delay'] = _get_timedelta(
default_args['retry_delay_unit']
, default_args['retry_delay_val']
)
del(default_args['retry_delay_unit'])
del(default_args['retry_delay_val'])
else:
default_args['retry_delay'] = _get_timedelta('seconds', 30)
except Exception as e:
print("Something went wrong while refactoring the dag configuration: " + str(e))
print("Please ensure proper structure is maintained in configuration files.")
print("~" * 100)
traceback.print_exc(file=sys.stdout)
print("~" * 100)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment