Gists for blog post about running notebooks in parallel. Ploomber, 2022.

run_papermill.py

# This requires:
# pip install papermill

import papermill as pm
from glob import glob

for nb in glob('*.ipynb'):
    pm.execute_notebook(
        input_path=nb,
        output_path=nb,
    )

run_papermill_ploomber_engine.py

# This requires:
# pip install papermill ploomber-engine

import papermill as pm
from glob import glob

for nb in glob('*.ipynb'):
    pm.execute_notebook(
        input_path=nb,
        output_path=nb,
        engine_name='embedded',
    )

run_ploomber_parallel.py

# This requires:
# pip install ploomber

from ploomber import DAG
from ploomber.products import File
from ploomber.tasks import NotebookRunner
from ploomber.executors import Parallel

from pathlib import Path
from glob import iglob

dag = DAG(executor=Parallel())


for path in iglob('*.ipynb'):
    NotebookRunner(Path(path), File(path), dag=dag, papermill_params=dict(engine_name='embedded'))


if __name__ ==  '__main__':
    dag.build(force=True)

run_ploomber_serial.py

# This requires:
# pip install ploomber

from ploomber import DAG
from ploomber.products import File
from ploomber.tasks import NotebookRunner
from ploomber.executors import Parallel

from pathlib import Path
from glob import iglob

dag = DAG()


for path in iglob('*.ipynb'):
    NotebookRunner(Path(path), File(path), dag=dag, papermill_params=dict(engine_name='embedded'))


if __name__ ==  '__main__':
    dag.build(force=True)