Submitting a job to a remote Grid Engine cluster using Paramiko (Python SSH client)
import tempfile | |
import os | |
import time | |
import paramiko | |
# Hostname of a 'submission host' in the Grid Engine cluster we want to submit a job to | |
hostname = 'sharc.shef.ac.uk' | |
# Username for logging in to the cluster via SSH | |
username = "te1st" | |
# Path on the cluster where we will save our job submission script to | |
remote_qsub_script_path = os.path.join('/', 'home', username, "qsub_script_{}".format(int(time.time()))) | |
# Job submission script | |
qsub_script = """#!/bin/bash | |
#$ -m bea | |
#$ -M some.body@sheffield.ac.uk | |
#$ -P rse | |
echo "Hello from $HOSTNAME" | |
""" | |
# Save job submission script to a local temporary file | |
with tempfile.NamedTemporaryFile(mode='w', delete=False) as local_qsub_script_file: | |
local_qsub_script_file.write(qsub_script) | |
with paramiko.SSHClient() as client: | |
client.load_system_host_keys() | |
client.set_missing_host_key_policy(paramiko.WarningPolicy) | |
# Establish SSH connection | |
client.connect(hostname, username=username) | |
# Establish SFTP connection | |
with client.open_sftp() as sftp: | |
# Push job submission script to a particular path on the cluster | |
sftp.put(local_qsub_script_file.name, remote_qsub_script_path) | |
# Submit our Grid Engine job by running a remote 'qsub' command over SSH | |
stdin, stdout, stderr = client.exec_command("qsub {}".format(remote_qsub_script_path)) | |
# Show the standard output and error of our job | |
print("Standard output:") | |
print(stdout.read()) | |
print("Standard error:") | |
print(stderr.read()) | |
print("Exit status: {}".format(stdout.channel.recv_exit_status())) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment