Skip to content

Instantly share code, notes, and snippets.

@dardanxhymshiti
Last active May 24, 2020 00:04
Show Gist options
  • Save dardanxhymshiti/e1a36e13684d8f799f0aeb634a31b1f8 to your computer and use it in GitHub Desktop.
Save dardanxhymshiti/e1a36e13684d8f799f0aeb634a31b1f8 to your computer and use it in GitHub Desktop.
from module_seed.jobs import \
job_one_seed,\
job_two_seed
from module_seed.utils.logger_utils import get_logger
from module_seed.utils.spark_utils import create_spark_session
jobs = {
'job_one_seed': job_one_seed.process,
'job_two_seed': job_two_seed.process
}
def run(parameters):
logger = get_logger()
for parameter, value in parameters.items():
logger.info('Param {param}: {value}'.format(param=parameter, value=value))
spark_config = parameters['spark_config']
spark = create_spark_session(spark_config=spark_config)
job_name = parameters['job_name']
process_function = jobs[job_name]
process_function(
spark=spark,
input_path=parameters['input_path'],
output_path=parameters['output_path'],
save_mode='append'
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment