Job v2 CLI (WIP)

EXAMPLES.MD

Set the job queue

$ dcos config set job queue https://dcos.sf.mesosphere.io/eng/dev/queue

Add a job defintion

$ vi job-0.yaml
name: job-0
job:
  hello:
    count: 1
    tasks:
      foo:
        cmd: /bin/true
        cpu: 1
        mem: 128
w!

$ dcos job definition add job-0.yaml

$ dcos job definition list
NAME     REVISION  NEXT-RUN  LAST-RUN  LAST-RUN-STATUS
job-0           1

Add another job defintion

$ vi job-1.yaml
name: job-1
job:
  parameters:
    cores:
      min: 1
      max: 64
      default: 4
  hello:
    count: 1
    tasks:
      foo:
        cmd: /bin/true
        cpu: {{ cores }}
        mem: 128
w!

$ dcos job definition add job-1.yaml

$ dcos job definition list
NAME     REVISION  NEXT-RUN  LAST-RUN  LAST-RUN-STATUS
job-0           1
job-1           1

Two jobs definitions cannot have the same name

$ cp job-1.yaml job-dup.yaml

$ dcos job definition add job-dup.yaml
Could not add 'job-1' because it already exists. Try 'dcos job definition update' to update an existing job definition.

$ dcos job definition list
NAME     REVISION  NEXT-RUN  LAST-RUN  LAST-RUN-STATUS
job-0           1
job-1           1

Update a job definition

$ vi job-1.yaml
name: job-1
job:
  parameters:
    cores:
      min: 1
      max: 64
      default: 4
  hello:
    count: 2
    tasks:
      foo:
        cmd: /bin/true
        cpu: {{ cores }}
        mem: 128
w!

$ dcos job definition update job-1.yaml

$ dcos job definition list
NAME     REVISION  NEXT-RUN  LAST-RUN  LAST-RUN-STATUS
job-0           1
job-1           2

Show latest revision of a job defintion

$ dcos job definition show job-1
name: job-1
job:
  parameters:
    cores:
      min: 1
      max: 64
      default: 4
  hello:
    count: 2
    tasks:
      foo:
        cmd: /bin/true
        cpu: {{ cores }}
        mem: 128

Show specific revision of a job definition

$ dcos job definition show job-1 --revision 1
name: job-1
job:
  parameters:
    cores:
      min: 1
      max: 64
      default: 4
  hello:
    count: 1
    tasks:
      foo:
        cmd: /bin/true
        cpu: {{ cores }}
        mem: 128

Delete a job definition

$ dcos job definition list
NAME     REVISION  NEXT-RUN  LAST-RUN  LAST-RUN-STATUS
job-0           1
job-1           2

$ dcos job definition delete job-0

$ dcos job definition list
NAME     REVISION  NEXT-RUN  LAST-RUN  LAST-RUN-STATUS
job-1           2

Add a job schedule to a job definition

$ vi job-1.yaml
name: job-1
job:
  parameters:
    cores:
      min: 1
      max: 64
      default: 4
  hello:
    count: 1
    tasks:
      foo:
        cmd: /bin/true
        cpu: {{ cores }}
        mem: 128
schedules: 
  daily: 
    cron: "0 0 * * *"
w!

$ dcos job definition update job-1.yaml

$ dcos job definition list
NAME     REVISION  NEXT-RUN  LAST-RUN  LAST-RUN-STATUS
job-1           3  1d

# Notice the job schedule is enabled by default
$ dcos job definition show job-1
name: job-1
job:
  parameters:
    cores:
      min: 1
      max: 64
      default: 4
  hello:
    count: 1
    tasks:
      foo:
        cmd: /bin/true
        cpu: {{ cores }}
        mem: 128
schedules: 
  daily: 
    cron: "0 0 * * *"
    enabled: true

Disable a job schedule

$ vi job-1.yaml
name: job-1
job:
  parameters:
    cores:
      min: 1
      max: 64
      default: 4
  hello:
    count: 1
    tasks:
      foo:
        cmd: /bin/true
        cpu: {{ cores }}
        mem: 128
schedules: 
  daily: 
    cron: "0 0 * * *"
    enabled: false
w!

$ dcos job definition update job-1.yaml

$ dcos job definition list
NAME     REVISION  NEXT-RUN  LAST-RUN  LAST-RUN-STATUS
job-1           4         

Enable a job schedule

$ vi job-1.yaml
name: job-1
job:
  parameters:
    cores:
      min: 1
      max: 64
      default: 4
  hello:
    count: 1
    tasks:
      foo:
        cmd: /bin/true
        cpu: {{ cores }}
        mem: 128
schedules: 
  daily: 
    cron: "0 0 * * *"
    enabled: true
w!

$ dcos job definition update job-1.yaml

$ dcos job definition list
NAME     REVISION  NEXT-RUN  LAST-RUN  LAST-RUN-STATUS
job-1           5  1d

Add the second job schedule

$ vi job-1.yaml
name: job-1
job:
  parameters:
    cores:
      min: 1
      max: 64
      default: 4
  hello:
    count: 1
    tasks:
      foo:
        cmd: /bin/true
        cpu: {{ cores }}
        mem: 128
schedules: 
  daily: 
    cron: "0 0 * * *"
    enabled: true
  friday-at-midnight:
    iso8601: 2018-04-13T23:59:00-08:00
w!

$ dcos job definition update job-1.yaml

$ dcos job definition list
NAME     REVISION  NEXT-RUN  LAST-RUN  LAST-RUN-STATUS
job-1           6  1d

Delete the second job schedule

$ vi job-1.yaml
name: job-1
job:
  parameters:
    cores:
      min: 1
      max: 64
      default: 4
  hello:
    count: 1
    tasks:
      foo:
        cmd: /bin/true
        cpu: {{ cores }}
        mem: 128
schedules: 
  daily: 
    cron: "0 0 * * *"
    enabled: true
w!

$ dcos job definition update job-1.yaml

$ dcos job definition list
NAME     REVISION  NEXT-RUN  LAST-RUN  LAST-RUN-STATUS
job-1           7  1d

Run a job definition manually

$ dcos job run job-1
RUN: CAkCDQIBCw8AAAADB

$ dcos job run list
RUN                 NAME      STATUS     STATUS-DETAIL  POSITION
CAkCDQIBCw8AAAADBA  job-1     Deploying  1m ago                

Run a job defintion with parameters

$ vi options.yaml
cores: 4
w!

$ dcos job run job-1 --parameters options.yaml
RUN: CAwgLBgUKDgsOAgBBA
        
$ dcos job run list
RUN                 NAME      STATUS     STATUS-DETAIL  POSITION
CAkCDQIBCw8AAAADBA  job-1     Succeeded  10m ago               
CAwgLBgUKDgsOAgBBA  job-1     Running    2m ago                

Run an ad-hoc job

$ vi rm-logs.yaml
name: rm-logs
job:
  hello:
    count: 1
    tasks:
      foo:
        cmd: find . -type f -name .log -delete
        cpu: 1
        mem: 128
w!

$ dcos job ad-hoc run rm-logs.yaml
RUN: gQKDwUBDgDOgBDwUBD

# Notice the ad-hoc job is not show in the job definition list
$ dcos job definition list
NAME     REVISION  NEXT-RUN  LAST-RUN  LAST-RUN-STATUS
job-1           7  1d        5m ago    Running

# Note the ad-hoc job run is shown is the list
$ dcos job run list
RUN                 NAME      STATUS     STATUS-DETAIL  POSITION
CAkCDQIBCw8AAAADBA  job-1     Succeeded  10m ago               
CAwgLBgUKDgsOAgBBA  job-1     Running    5m ago
gQKDwUBDgDOgBDwUBD  rm-logs   Queued     1m ago                1

Show details for a run

$ dcos job run show CAkCDQIBCw8AAAADBA
NAME     REVISION  STATUS     ACCEPTED   STARTED   ENDED    ATTEMPTS
job-1           7  Succeeded  10m ago    10m ago   8m ago       2/10

Show parameters for a job run

$ dcos job run show CAkCDQIBCw8AAAADBA --parameters
PARAMETER  VALUE
cores      2

List run attempts

$ dcos job run attempt list CAkCDQIBCw8AAAADB
ATTEMPT  STATUS     STARTED  ENDED 
1        Failed     10m ago  9m ago
2        Succeeded  9m ago   8m ago

Show details for an run attempt

$ dcos job run attempt show CAkCDQIBCw8AAAADB 2
name: job-1
└─ pod: hello
   ├─ pod-id: hello-0
   │  └─ pod-attempt: 0
   │     └─ task: foo
   │        └─ task-attempt: 0
   │           ├─ task-id: CAoDBQsOBAcHCQIHBA
   │           ├─ host: agent-71.mesosphere.com   
   │           ├─ status: Succeeded
   │           ├─ started: 2018-04-10T11:57:47
   │           ├─ ended: 2018-04-10T11:59:00
   │           ├─ cpu: 2
   │           ├─ gpu: 0
   │           └─ mem: 4
   └─ pod-id: hello-1
      └─ pod-attempt: 0
         └─ task: foo
            └─ task-attempt: 0
               ├─ task-name: foo
               ├─ task-id: BwMGAQAFBw8KBw8PBA
               ├─ host: agent-21.mesosphere.com   
               ├─ status: Succeeded
               ├─ started: 2018-04-10T11:57:46
               ├─ ended: 2018-04-10T11:59:00
               ├─ cpu: 2
               ├─ gpu: 0
               └─ mem: 4

Show task log

$ dcos task log CAoDBQsOBAcHCQIHBA
2018-03-04 07:27:53 Activity 0x22310 4267 0 syncdefaultsd: (libsystem_info.dylib) Retrieve User by ID
2018-03-04 07:27:53 Activity 0x22311 4267 0 syncdefaultsd: (CoreFoundation) Loading Preferences From User CFPrefsD
2018-03-04 07:27:53 Activity 0x22312 4267 0 syncdefaultsd: (libsystem_info.dylib) Membership API: translate identifier
2018-03-04 07:27:53 Activity 0x22313 4267 0 syncdefaultsd: (CoreFoundation) Loading Preferences From System CFPrefsD

Cancel a job run

$ dcos job run cancel CAkCDQIBCw8AAAADBA

$ dcos job run list
RUN                 NAME      STATUS     STATUS-DETAIL  POSITION
CAkCDQIBCw8AAAADBA  job-1     Succeeded  10m ago               - 
CAwgLBgUKDgsOAgBBA  job-1     Running    5m ago                -
gQKDwUBDgDOgBDwUBD  rm-logs   Cancelled  5s ago                -

Submit three adhoc job runs quickly

$ for i in `seq 1 3`;
> do
>     dcos job adhoc run rm-logs.yaml
> done
RUN: AQoEDAYMDwwICwgABA
RUN: BgYICwYEBAgLAQEOBA
RUN: AgMNAgINDw4OBAUPBA

$ dcos job run list
RUN                  NAME      STATUS     STATUS-DETAIL  POSITION
CAkCDQIBCw8AAAADBA   job-1     Succeeded  10m ago               - 
CAwgLBgUKDgsOAgBBA   job-1     Running    5m ago                -
gQKDwUBDgDOgBDwUBD   rm-logs   Cancelled  5s ago                -
AQoEDAYMDwwICwgABA   rm-logs   Deploying  1s ago                -
BgYICwYEBAgLAQEOBA   rm-logs   Queued     1s ago                1
AgMNAgINDw4OBAUPBA   rm-logs   Queued     1s ago                2

List job runs with filtering by status

$ dcos job run list --status succeeded
RUN                  NAME      STATUS     STATUS-DETAIL  POSITION
CAkCDQIBCw8AAAADBA   job-1     Succeeded  15m ago               - 
CAwgLBgUKDgsOAgBBA   job-1     Succeeded  5m ago                -

List job runs with filtering by name

$ dcos job run list --name job-1
RUN                  NAME      STATUS     STATUS-DETAIL  POSITION
CAkCDQIBCw8AAAADBA   job-1     Succeeded  15m ago               - 
CAwgLBgUKDgsOAgBBA   job-1     Succeeded  5m ago                -

List job runs with filtering by start time and end time

$ dcos job run list --start-time 2018-04-10T12:00:00 --end-time 2018-04-10T12:00:10
RUN                  NAME      STATUS     STATUS-DETAIL  POSITION
gQKDwUBDgDOgBDwUBD   rm-logs   Cancelled  5m ago                -
AQoEDAYMDwwICwgABA   rm-logs   Deploying  1m ago                -
BgYICwYEBAgLAQEOBA   rm-logs   Queued     1m ago                1
AgMNAgINDw4OBAUPBA   rm-logs   Queued     1m ago                2

List runs with multiple filters

$ dcos job run list --status queued --start-time 2018-04-10T12:00:00 --end-time 2018-04-10T12:00:10
RUN                  NAME      STATUS     STATUS-DETAIL  POSITION
BgYICwYEBAgLAQEOBA   rm-logs   Queued     1m ago                1
AgMNAgINDw4OBAUPBA   rm-logs   Queued     1m ago                2

Delete the only job definition

$ dcos job definition delete job-1

$ dcos job definition list
No job definitions found.

JOB-V2.MD

$ dcos config set job queue https://dcos-1.us-west-2.elb.amazonaws.com/eng/dev/queue-service

$ dcos job --help
Description: Define, schedule, and run jobs.

Usage:
    dcos job definition add <definition-file>
    dcos job definition list
    dcos job definition show <definition> [--revision]
    dcos job definition update <definition>
    dcos job definition delete <definition> [--cancel-runs]
    dcos job run <definition> [--parameters <parameters-file>]
    dcos job run list [--name --status --start-time --end-time --time-format --max-results]
    dcos job run show <run>
    dcos job run cancel <run>
    dcos job run attempt list <run>
    dcos job run attempt show <run> <attempt>
    dcos job run attempt cancel <run> <attempt>
    dcos job adhoc run <definition> [--parameters <parameters-file>]

Commmands:
    dcos job --help
    dcos job --info
    dcos job --version
    dcos job definition add <definition-file> [--parameters <parameters-file>]
        Add a job definition to the job definition store.
    dcos job definition list
        List job definitions in the job definition store.
    dcos job definition show <definition> [--revision]
        Show details of a stored job definition.
    dcos job definition update <definition-file>
        Update a job definition in the job defintion store.
    dcos job definition delete <definition> [--cancel-runs]
        Delete a job definition from the job definition store.
    dcos job run <definition> [--parameters <option-file>]
        Run a job definition.     
    dcos job run list [--name --status --start-time --end-time --time-format --max-results]
        List job runs.
    dcos job run show <run>
        Show details of a job run.
    dcos job run cancel <run>
        Cancel a job run.
    dcos job run attempt list <run>
        List attempts for a job run.
    dcos job run attempt show <run> <attempt>
        Show details of a job run attempt.
    dcos job run attempt cancel <run> <attempt>
        Cancel a job run attempt. The next attempt if start if the attempt limit has not been reached. 
    dcos job adhoc run <definition-file> [--parameters <parameters-file>]
        Submit an adhoc job run. The job definition do not show up is the list of job definitions.
      
Options:
    -h, --help
        Print usage.
    --version
        Print version information.
    --info
        Print a short description of this subcommand.
    --json
        Print JSON-formatted list instead of a table.
    --yaml
        Print YAML-formatted list instead of a table.
    --cancel-runs
        Indicates all runs should be cancelled.
    --parameters
        A YAML formatted parameters for a job run (for example, --parameters job-credentials.yaml).
    --revision
        Show a specific revision of a job (for example, --revision 2).
    --attempt
        The attempt ID of a job run attempt (for example, --attempt 2).
    --name
        The name of the job run to filter by.
    --status
        One of more comma seperated statuses (for example, --status SUCCEEDED,FAILED) where valid values are:
            * WAITING: The job run is waiting for job dependencies to complete successfully.
            * QUEUED: The job run is eledgible to run and assigned a position.
            * DEPLOYING: The job run is staging to the cluster.
            * RUNNING: The job run is executing in the cluster.
            * SUCCEEDED: The job run was completed successfully (exit code was 0).
            * FAILED: The job run was completed but all attempts failed (exit code was non-zero).
            * CANCELLED: The job run was cancelled by user.
    --start-time
        The time that determines the first result to return. Acceptable time formats include:
            * Date (for example, --start-time 2018-12-31).
            * Date and time in local time (for example, --start-time 2018-12-31T23:59:59).
            * Date and time in UTC time (for example, --start-time 2018-12-31T23:59:59Z).
            * Date and time in UTC time with offset (for example, --start-time 2018-12-31T23:59:59Z-08:00).
            * Unix time in seconds (for example, --start-time 1517471999). 
    --end-time
        The time that determines the last result to return. Acceptable time formats include:
            * Date (for example, --end-time 2018-12-31).
            * Date and time in local time (for example, --end-time 2018-12-31T23:59:59).
            * Date and time in UTC time (for example, --end-time 2018-12-31T23:59:59Z).
            * Date and time in UTC time with offset (for example, --end-time 2018-12-31T23:59:59Z-08:00).
            * Unix time in seconds (for example, --end-time 1517471999).
    --time-format
        Indicates the format to display date and time. Default is simple. Acceptable values are:
            * simple - Summary of the remaining time or elapsed time (for example, 10m ago).
                s: Seconds
                m: Minutes
                h: Hours
                d: Days
                W: Weeks
                M: Months
                D: Years 
            * timestamps - Unix timestamp in seconds since epoch (for example, 1517471999).
            * datatime - ISO 8601 compatible data and time (for example, 2018-12-31T23:59:59-8000). 
    --max-result
        The maximum number of results to return (for example, --max-results 100). Default is 25.
        
Positional Arguments:
    <definition>
        The job ID (for example, backup-job).
    <run>
        The run ID of a job run (for example, 5999fc27-6415-4f42-91e8-29df545bbf8b).
    <definition-file>
        A YAML formatted definition of the job (for example, rm-logs.yaml).

JOBS-V1.MD

$ dcos job --help
Description:
    Deploy and manage jobs in DC/OS.

    Usage:
        dcos job --config-schema
        dcos job --help
        dcos job --info
        dcos job --version
        dcos job add <job-file>
        dcos job remove <job-id> [--stop-current-job-runs]
        dcos job show <job-id>
        dcos job update <job-file>
        dcos job kill <job-id> [<run-id>][--all]
        dcos job run <job-id> [--json]
        dcos job list [--json|--quiet]
        dcos job schedule add <job-id> <schedule-file>
        dcos job schedule show <job-id> [--json]
        dcos job schedule remove <job-id> <schedule-id>
        dcos job schedule update <job-id> <schedule-file>
        dcos job show runs <job-id> [--run-id <run-id>][--json|--quiet]
        dcos job history <job-id> [--json|--quiet] [--failures --last]

Commands:
    job add
        Add a job.
    job remove
        Remove a job.
    job show
        Show the job.
    job update
        Update the job.
    job kill
        Kill the job.
    job run
        Run a job now.
    job list
        Show all job definitions.
    job schedule add
        Adds a schedule to a job.
    job schedule show
        Shows the schedules of a job.
    job schedule remove
        Removes a schedule from a job.
    job schedule update
        Updates a schedule on a job.
    job show runs
        Shows the successful and failure runs of a job.
    job history
        Provides a job run history.

Options:
    -h, --help
        Print usage.
    --version
        Print version information.
    --config-schema
        Show the configuration schema for the Metronome subcommand.
    --info
        Print a short description of this subcommand.
    --stop-current-job-runs
        Indicates on a job removal that all current running jobs should be killed.
    --all
        Instead of specifying a run_id to kill, indicates all runs should be killed.
    --json
        Print JSON-formatted list instead of a table.
    -q, --quiet
        Indicates a quiet mode which results in just an array of run ids.
    -f, --failures
        Indicates to show the failure table and statistics for history.
    -l, --last
        Shows the last ID for history job.

Positional Arguments:
    <job-id>
        The job ID.
    <job-file>
        A JSON formatted file of a job.
    <run-id>
        The run ID of a job run.
    <schedule-file>
        A JSON formatted file of a job schedule.
    <schedule-id>
        The schedule ID.