Created
May 29, 2020 06:28
-
-
Save saumalya75/742452368be92a2be644c7cf2aeffe47 to your computer and use it in GitHub Desktop.
Date-Time configuration handling
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
try: | |
dag_config = pipeline_config['dag'] | |
if dag_config['schedule_interval_unit'] and dag_config['schedule_interval_val']: | |
dag_config['schedule_interval'] = _get_timedelta( | |
dag_config['schedule_interval_unit'] | |
, dag_config['schedule_interval_val'] | |
) | |
del(dag_config['schedule_interval_unit']) | |
del(dag_config['schedule_interval_val']) | |
else: | |
dag_config['schedule_interval'] = _get_timedelta('days', 1) | |
default_args = pipeline_config['dag']['default_args'] | |
default_args = default_args | |
default_args['start_date'] = datetime( | |
default_args['start_date']['year'], | |
default_args['start_date']['month'], | |
default_args['start_date']['day'] | |
) | |
if default_args['retry_delay_unit'] and default_args['retry_delay_val']: | |
default_args['retry_delay'] = _get_timedelta( | |
default_args['retry_delay_unit'] | |
, default_args['retry_delay_val'] | |
) | |
del(default_args['retry_delay_unit']) | |
del(default_args['retry_delay_val']) | |
else: | |
default_args['retry_delay'] = _get_timedelta('seconds', 30) | |
except Exception as e: | |
print("Something went wrong while refactoring the dag configuration: " + str(e)) | |
print("Please ensure proper structure is maintained in configuration files.") | |
print("~" * 100) | |
traceback.print_exc(file=sys.stdout) | |
print("~" * 100) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment