Skip to content

Instantly share code, notes, and snippets.

@worldwise001
Created November 12, 2019 08:35
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save worldwise001/c437c616f3b7c30f0b899d673d478047 to your computer and use it in GitHub Desktop.
Save worldwise001/c437c616f3b7c30f0b899d673d478047 to your computer and use it in GitHub Desktop.
def __init__(self, config: Config, modules: List[str]):
kwargs = config.get('MYSQL_ENGINE_ARGS')
if not kwargs:
raise Exception('No MYSQL_ENGINE_ARGS set in setup_job_scheduler')
self.available_jobs: Dict[str, Any] = {}
# Import modules as necessary for the purpose of instantiating Jobs
for module_str in modules:
module = import_module(module_str)
if module is not None:
path = getattr(module, '__path__', None)
if path is not None:
for submodule_info in pkgutil.iter_modules(path):
submodule_str = f'{module.__name__}.{submodule_info.name}'
import_module(submodule_str)
# Instantiate all Job subclasses and add them to available_jobs
for job_class in Job.__subclasses__():
should_be_loaded = False
for module_str in modules:
if job_class.__module__.startswith(module_str):
should_be_loaded = True
break
if not should_be_loaded:
continue
# TODO: type this
job = job_class(config)
job.origin = JobOrigin.CONFIG
self.available_jobs[job.get_id()] = job
# NB: If jobs are missed for some reason, still try to run within 120 minutes
self.scheduler = BackgroundScheduler(job_defaults={'misfire_grace_time': 120 * 60})
self.scheduler.add_jobstore('sqlalchemy', engine=sqlalchemy.create_engine(**kwargs))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment