Instantly share code, notes, and snippets.

Embed
What would you like to do?
Submitting a job to a remote Grid Engine cluster using Paramiko (Python SSH client)
import tempfile
import os
import time
import paramiko
# Hostname of a 'submission host' in the Grid Engine cluster we want to submit a job to
hostname = 'sharc.shef.ac.uk'
# Username for logging in to the cluster via SSH
username = "te1st"
# Path on the cluster where we will save our job submission script to
remote_qsub_script_path = os.path.join('/', 'home', username, "qsub_script_{}".format(int(time.time())))
# Job submission script
qsub_script = """#!/bin/bash
#$ -m bea
#$ -M some.body@sheffield.ac.uk
#$ -P rse
echo "Hello from $HOSTNAME"
"""
# Save job submission script to a local temporary file
with tempfile.NamedTemporaryFile(mode='w', delete=False) as local_qsub_script_file:
local_qsub_script_file.write(qsub_script)
with paramiko.SSHClient() as client:
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.WarningPolicy)
# Establish SSH connection
client.connect(hostname, username=username)
# Establish SFTP connection
with client.open_sftp() as sftp:
# Push job submission script to a particular path on the cluster
sftp.put(local_qsub_script_file.name, remote_qsub_script_path)
# Submit our Grid Engine job by running a remote 'qsub' command over SSH
stdin, stdout, stderr = client.exec_command("qsub {}".format(remote_qsub_script_path))
# Show the standard output and error of our job
print("Standard output:")
print(stdout.read())
print("Standard error:")
print(stderr.read())
print("Exit status: {}".format(stdout.channel.recv_exit_status()))
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment