Skip to content

Instantly share code, notes, and snippets.

@annawoodard annawoodard/example.py
Last active Sep 26, 2018

Embed
What would you like to do?
import parsl
from parsl.providers import SlurmProvider
from parsl.config import Config
from parsl.executors.ipp import IPyParallelExecutor
from parsl.app.app import python_app
config = Config(
executors=[
IPyParallelExecutor(
provider=SlurmProvider(
'broadwl',
walltime="01:00:00",
init_blocks=1,
max_blocks=64,
nodes_per_block=1,
tasks_per_node=32,
),
)
]
)
parsl.set_stream_logger()
parsl.load(config)
@python_app
def inside_task(i, j):
import psutil
import os
return '{}/{} is running on {}: {}%'.format(i, j, os.environ['HOSTNAME'], psutil.cpu_percent())
@python_app
def outside_task(inputs=[]):
return '\n'.join(inputs)
outside_tasks = []
for i in range(0, 100):
inside_tasks = []
for j in range(0, 64):
inside_tasks.append(inside_task(i, j))
# outside_task will not be run until all of inside_tasks have completed
outside_tasks.append(outside_task(inputs=inside_tasks))
with open('results.txt', 'w') as f:
for t in outside_tasks:
print(t.result(), file=f)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.