Created
September 23, 2018 14:42
-
-
Save TakahisaShiratori/08a9a75938bd999b8d432de85aa8d113 to your computer and use it in GitHub Desktop.
Rescale Persistent Cluster with REST API
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "First Job", | |
"jobanalyses": [ | |
{ | |
"analysis": { | |
"code": "user_included", | |
"version": "0" | |
}, | |
"command": "echo first_job_start; sleep 5m; echo first_job_end", | |
"flags": {}, | |
"hardware": { | |
"coresPerSlot": 1, | |
"slots": 1, | |
"coreType": { | |
"code": "hpc-3" | |
}, | |
"walltime": 10 | |
}, | |
"inputFiles": [] | |
} | |
], | |
"isLowPriority": "true" | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import json | |
import requests | |
import sys | |
import time | |
from datetime import datetime | |
api_token = "your-api-token" | |
token_in_header = "Token " + api_token | |
platform = "platform.rescale.jp" | |
cluster_configuration_file = "testCluster.json" | |
job1_configuration_file = "firstJob.json" | |
job2_configuration_file = "secondJob.json" | |
# Read Cluster Configuration | |
cluster_configuration = json.load(open(cluster_configuration_file, "r")) | |
# Create Cluster | |
raw_reply = requests.post( | |
"https://" + platform + "/api/v3/clusters/", | |
headers={"Authorization": token_in_header}, | |
json=cluster_configuration | |
) | |
created_cluster = json.loads(raw_reply.text) | |
print("Cluster ID: " + created_cluster["id"]) | |
# Start Cluster | |
requests.post( | |
"https://" + platform + "/api/v3/clusters/" + created_cluster["id"] + "/start/", | |
headers={"Authorization": token_in_header} | |
) | |
# Sleep 5 sec | |
time.sleep(5) | |
# Get the Cluster's Status | |
raw_reply = requests.get( | |
"https://" + platform + "/api/v3/clusters/" + created_cluster["id"] + "/statuses/", | |
headers={"Authorization": token_in_header} | |
) | |
cluster_statuses = json.loads(raw_reply.text) | |
for result in cluster_statuses["results"]: | |
print("Status date: " + result["statusDate"] + ", Status: " + result["status"]) | |
# Read Job Configurations | |
job1_configuration = json.load(open(job1_configuration_file, "r")) | |
job2_configuration = json.load(open(job2_configuration_file, "r")) | |
# Create Jobs | |
raw_reply = requests.post( | |
"https://" + platform + "/api/v3/jobs/", | |
headers={"Authorization": token_in_header}, | |
json=job1_configuration | |
) | |
first_job = json.loads(raw_reply.text) | |
print("Job ID: " + first_job["id"]) | |
raw_reply = requests.post( | |
"https://" + platform + "/api/v3/jobs/", | |
headers={"Authorization": token_in_header}, | |
json=job2_configuration | |
) | |
second_job = json.loads(raw_reply.text) | |
print("Job ID: " + second_job["id"]) | |
# Submit Jobs | |
submit_configuration = {} | |
submit_configuration["id"] = first_job["id"] | |
submit_configuration["submit"] = True | |
raw_reply = requests.post( | |
"https://" + platform + "/api/clusters/" + created_cluster["id"] + "/jobs/", | |
headers={"Authorization": token_in_header}, | |
json=submit_configuration | |
) | |
submitted_job = json.loads(raw_reply.text) | |
print("Job " + submitted_job["id"] + " is submitted to the cluster " + created_cluster["id"]) | |
time.sleep(5) | |
submit_configuration["id"] = second_job["id"] | |
raw_reply = requests.post( | |
"https://" + platform + "/api/clusters/" + created_cluster["id"] + "/jobs/", | |
headers={"Authorization": token_in_header}, | |
json=submit_configuration | |
) | |
submitted_job = json.loads(raw_reply.text) | |
print("Job " + submitted_job["id"] + " is submitted to the cluster " + created_cluster["id"]) | |
# Wait until the Jobs are Completed | |
raw_reply = requests.get( | |
"https://" + platform + "/api/v3/jobs/" + first_job["id"] + "/statuses/", | |
headers={'Authorization': token_in_header} | |
) | |
first_job_status = json.loads(raw_reply.text)["results"][0]["status"] | |
raw_reply = requests.get( | |
"https://" + platform + "/api/v3/jobs/" + second_job["id"] + "/statuses/", | |
headers={'Authorization': token_in_header} | |
) | |
second_job_status = json.loads(raw_reply.text)["results"][0]["status"] | |
while first_job_status != "Completed" or second_job_status != "Completed": | |
time.sleep(5) | |
raw_reply = requests.get( | |
"https://" + platform + "/api/v3/jobs/" + first_job["id"] + "/statuses/", | |
headers={'Authorization': token_in_header} | |
) | |
first_job_status = json.loads(raw_reply.text)["results"][0]["status"] | |
raw_reply = requests.get( | |
"https://" + platform + "/api/v3/jobs/" + second_job["id"] + "/statuses/", | |
headers={'Authorization': token_in_header} | |
) | |
second_job_status = json.loads(raw_reply.text)["results"][0]["status"] | |
current_time = datetime.now().strftime("%Y/%m/%d %H:%M:%S") | |
sys.stdout.write("\r[" + current_time + "] First Job: " + first_job_status + ", Second Job: " + second_job_status) | |
sys.stdout.flush() | |
# Shutdown the Cluster | |
requests.post( | |
"https://" + platform + "/api/v3/clusters/" + created_cluster["id"] + "/shutdown/", | |
headers={"Authorization": token_in_header} | |
) | |
time.sleep(5) | |
# Get the Cluster's Status | |
raw_reply = requests.get( | |
"https://" + platform + "/api/v3/clusters/" + created_cluster["id"] + "/statuses/", | |
headers={"Authorization": token_in_header} | |
) | |
cluster_statuses = json.loads(raw_reply.text) | |
print("\nStatus date: " + cluster_statuses["results"][0]["statusDate"] + ", Status: " + cluster_statuses["results"][0]["status"]) | |
print("Done") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "Second Job", | |
"jobanalyses": [ | |
{ | |
"analysis": { | |
"code": "user_included", | |
"version": "0" | |
}, | |
"command": "echo second_job_start; sleep 5m; echo second_job_end", | |
"flags": {}, | |
"hardware": { | |
"coresPerSlot": 1, | |
"slots": 1, | |
"coreType": { | |
"code": "hpc-3" | |
}, | |
"walltime": 10 | |
}, | |
"inputFiles": [] | |
} | |
], | |
"isLowPriority": "true" | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"name": "My Persistent Cluster", | |
"hardware": { | |
"coresPerSlot": 1, | |
"slots": 1, | |
"coreType": { | |
"code": "hpc-3" | |
}, | |
"walltime": 10 | |
}, | |
"installedAnalyses": [ | |
{ | |
"analysis": { | |
"code": "user_included", | |
"version": "0" | |
} | |
} | |
], | |
"runLowPriority": true | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
How to use
Control Rescale Persistent Cluster with REST API – Takahisa Shiratori – Medium
https://medium.com/@takahisa.shiratori/control-rescale-persistent-cluster-with-rest-api-cba69cac74e7