Skip to content

Instantly share code, notes, and snippets.

@sean-smith
Created September 29, 2021 15:13
Show Gist options
  • Save sean-smith/d4116589f28ea5c9765c9f14d5228300 to your computer and use it in GitHub Desktop.
Save sean-smith/d4116589f28ea5c9765c9f14d5228300 to your computer and use it in GitHub Desktop.
Call AWS ParallelCluster API with Python
#!/usr/bin/env python3
import json
from base64 import b64decode, b64encode
from pprint import pprint
import boto3
import botocore
import yaml
import requests
def sigv4_request(method, host, path, params, headers, body):
"Adds authorization headers for sigv4 to headers parameter."
endpoint = host.replace("https://", "").replace("http://", "")
_api_id, _service, region, _domain = endpoint.split(".", maxsplit=3)
request_parameters = "&".join([f"{k}={v}" for k, v in params.items()])
url = f"{host}{path}?{request_parameters}"
session = botocore.session.Session()
body_data = json.dumps(body) if body else None
request = botocore.awsrequest.AWSRequest(method=method, url=url, data=body_data)
botocore.auth.SigV4Auth(session.get_credentials(), "execute-api", region).add_auth(request)
boto_request = request.prepare()
req_call = {
"POST": requests.post,
"GET": requests.get,
"PUT": requests.put,
"PATCH": requests.patch,
"DELETE": requests.delete,
}.get(method)
for k, val in headers.items():
boto_request.headers[k] = val
return req_call(boto_request.url, data=body_data, headers=boto_request.headers, timeout=30)
def create(region, baseurl, cluster_name, config_file):
# parse config file
path = "/v3/clusters"
config_data = b64decode(config_file).decode("ascii")
params = {"region": region}
body = {"clusterConfiguration": config_data, "clusterName": cluster_name}
headers = {"content-type": "application/json"}
resp = sigv4_request("POST", baseurl, path, params, headers, body)
pprint(resp.json())
def list_clusters(region, baseurl):
path = "/v3/clusters"
params = {"region": region}
resp = sigv4_request("GET", baseurl, path, params, {}, None)
pprint(resp.json())
if __name__ == "__main__":
config = """
Image:
Os: ubuntu2004
# CustomAmi: ami-0016baecd80fb8f18
HeadNode:
InstanceType: t2.large
Networking:
SubnetId: subnet-01ff8b514162d9a92
Ssh:
KeyName: enguard
Scheduling:
Scheduler: slurm
SlurmQueues:
- Name: queue0
ComputeResources:
- Name: queue0-i0
InstanceType: t2.micro
MinCount: 1
MaxCount: 11
Networking:
SubnetIds:
- subnet-08190f2e7fe1f140e
"""
baseurl = "https://mdxwg8m984.execute-api.us-east-2.amazonaws.com/prod"
cluster_name = "XXX"
list_clusters("us-east-2", baseurl)
create("us-east-2", baseurl, cluster_name, b64encode(config.encode("ascii")))
@sean-smith
Copy link
Author

Step 1

First Deploy the AWS ParallelCluster API

Region Launch
North Virginia (us-east-1) Launch
Oregon (us-west-2) Launch
Ireland (eu-west-1) Launch
Frankfurt (eu-central-1) Launch

Step 2

Look at the stack outputs and grab ParallelClusterApiInvokeUrl , paste that into the baseurl parameter in the example python script.

Also add a Cluster Name, Region and update SubnetID to your subnet.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment