Get the parameters for job queues and job definitions

get_aws_batch_params.py

def get_job_queues(client):
    """
    Get all job queues from aws batch service. Transform into dataframe and filter for valid ones.
    extract the repo name from the job queue name. Transform into dict and return it.
    :param client:
    :return:
    """
    jq = client.describe_job_queues()["jobQueues"]
    jq = pd.DataFrame.from_records(jq)
    jq = jq[jq.status == "VALID"]
    jq["job"] = jq.jobQueueName.str.split("-job").apply(lambda x: x[0])
    jq.index = jq.job.values
    jq_dict = jq.T.to_dict()
    return jq_dict


def get_job_definitions(client):
    """
    Get all job definitions from aws batch service. Transform into dataframe and filter for active ones. Only use latest
    revisions. Extract the repo name from the job definition name. Transform into dict and return it.
    :param client:
    :return:
    """
    jd = client.describe_job_definitions()
    jd = pd.DataFrame.from_records(jd['jobDefinitions'])
    jd = jd[jd.status == "ACTIVE"]
    jd = jd.sort_values("revision", ascending=False).drop_duplicates("jobDefinitionName").copy()
    jd["job"] = jd.jobDefinitionName.str.split("-job").apply(lambda x: x[0])
    jd.index = jd.job.values
    jd_dict = jd.T.to_dict()
    return jd_dict

def get_batch_params(client):
    """
    Get the job definitions and job queues from aws batch. Merge them with the Agent name as key and return the dict.
    This assumes that the job definitions and job queues are named after the agent name.
    To get the job definitions and job queues, use the get_job_definitions and get_job_queues functions.
    :param client:
    :return:
    """
    jd = get_job_definitions(client)
    jq = get_job_queues(client)
    j = {}
    for key in jq.keys():
        a = jd.get(key, {})
        b = jq.get(key, {})
        a.update(b)
        j[key] = a
    return j

import boto3
client = boto3.client("batch")
params = get_batch_params(client)