Git-like upload and download shortcuts for neuro

.neuro__user.toml

[storage]
cp-exclude = [".git"]
cp-exclude-from-files = [".neuroignore", ".gitignore"]


# helpers

[alias.proj-name]
exec = "sh -c 'basename `neuro proj-local`'"

[alias.proj-local]
help = "Prints local project root"
exec = "sh -c 'for f in .git Makefile; do p=`pwd`; while [ -n \"$p\" ] && [ ! -f $p/$f ] && [ ! -d $p/$f ]; do p=${p%/*}; done; [ -n \"$p\" ] && echo $p && break || true; done'"

[alias.proj-storage]
help = "Prints project root on storage"
exec = "sh -c 'echo storage:`neuro proj-name`'"

[alias.proj-postfix]
help = "Prints common job name postfix"
exec = "sh -c 'neuro proj-name'"

[alias.proj-tags]
exec = "sh -c 'n=`neuro proj-name`; echo \"--tag kind:project --tag project:$n\"'"

[alias.proj-image]
help = "Run `export IMG=image:$(neuro proj-name):latest` to overwrite this value"
exec = "sh -c 'echo ${IMG:-neuromation/base:latest}'"

[alias.base-image]
exec = "sh -c 'echo neuromation/base:latest'"

[alias.pwd-relative]
exec = "sh -c 'root=`neuro proj-local`; p=`pwd`; [[ $p = $root ]] && echo . || echo ${p#$root/}'"


# storage

[alias.pwd]
help = "Print corresponding directory on storage"
exec = "sh -vc 'p=`neuro pwd-relative`; echo `neuro proj-storage`/$p'"

[alias.ll]
help = "List corresponding directory on storage"
args = "[FILE]"
exec = "sh -xvc 'neuro --verbose ls `neuro pwd`/{file}'"

[alias.del]
help = "Delete corresponding directory or file on storage"
args = "[FILE]"
exec = "sh -xvc 'neuro --verbose rm -r `neuro pwd`/{file}'"

[alias.sync-up]
args = "[FILE]"
help = "Upload local FILE to the corresponding directory on storage"
# Usage: "neuro upload; neuro upload config/train.yaml"
exec = "sh -xvc 'dst=`neuro pwd`/{file}; neuro --verbose mkdir -p `dirname $dst` && neuro --verbose cp -ru ./{file} -T $dst'"

[alias.sync-down]
args = "[FILE]"
help = "Download file FILE from the corresponding directory on storage"
# Usage: "neuro download; neuro download config/train.yaml"
exec = "sh -xvc 'neuro --verbose cp -ru `neuro pwd`/{file} .'"


# jobs

[alias.pp]
help = "List active jobs for current project"
exec = "sh -xvc 'neuro ps `neuro proj-tags`'"

[alias.s]
args = "JOB"
help = "Get job's status"
exec = "sh -xvc 'neuro status {job}-`neuro proj-postfix`'"

[alias.l]
args = "JOB"
help = "Get job's logs (output)"
exec = "sh -xvc 'neuro logs {job}-`neuro proj-postfix`'"

[alias.k]
args = "JOB"
help = "Kill the job"
exec = "sh -xvc 'neuro kill {job}-`neuro proj-postfix`'"

[alias.e]
args = "JOB"
help = "Exec (connect) to the job"
exec = "sh -xvc 'neuro exec {job}-`neuro proj-postfix` bash'"

[alias.b]
args = "JOB"
help = "Open job's URL in browser"
exec = "sh -xvc 'neuro job browse {job}-`neuro proj-postfix`'"


[alias.r]
help = "Internal job runner"
args = "JOB IMAGE BASH_CMD"
options = [
    "-s, --preset=PRESET",
    "-v, --volume=VOLUME",
    "--no-wait-start",
    "--detach",
    "--browse",
    "--http=N",
    "--no-http-auth",
    "-e, --env=KEY_VAL",
    "--pass-config",
]
# Examples:
#  $ neuro r --no-http-auth --env KEY=VAL  job-name  image  "ls && pwd && whoami"
exec = "sh -xvc \"neuro run -n {job}-`neuro proj-postfix` `neuro proj-tags`  {preset} {no_wait_start} {detach} {browse} {http} {no_http_auth} {pass_config} -e PYTHONPATH=/project  -v `neuro proj-storage`:/project:rw  {image} bash -c {bash_cmd}\""


[alias.r-srv]
help = "run 'server' job (just sleep infinity on gpu-small)"
# Examples: 
#  $ neuro r-srv
exec = "sh -xvc 'neuro r  --preset=gpu-small  srv  `neuro base-image`  bash'"


[alias.r-j]
help = "run 'jupyter' job"
options = [
    "--no-http-auth",
    "-e, --env=KEY_VAL",
    "--pass-config",
]
# Examples: 
#  $ neuro r-j
#  $ neuro r-j --no-http-auth --env KEY=VAL --pass-config
exec = "sh -xvc 'neuro r  --preset=gpu-small --http=8888 --browse --detach {no_http_auth} {env} {pass_config}  j  `neuro base-image`  \"jupyter notebook --no-browser --ip=0.0.0.0 --allow-root --NotebookApp.token= --notebook-dir=/project\"'"


[alias.r-tb]
args = "RESULTS_DIR"
help = "run 'tensorboard' job"
options = [
    "--no-http-auth",
]
# Examples: 
#  $ neuro r-tb
#  $ neuro r-tb --no-http-auth
exec = "sh -xvc 'neuro mkdir -p `neuro pwd`/{results_dir} && neuro r  --preset=cpu-small --http=6006 --browse --detach {no_http_auth}  tb  tensorflow/tensorflow:latest \"tensorboard --host=0.0.0.0 --logdir=/project/{results_dir}\"'"


# $ cd `mktemp -d`
# $ git clone https://github.com/pytorch/examples.git
# $ cd examples/mnist
# $ neuro sync-up
# $ neuro r-srv
# $ neuro e srv
# root@job-93b5db29-78cd-45a7-925f-41a9c2fd5669:/# cd /project/mnist
# root@job-93b5db29-78cd-45a7-925f-41a9c2fd5669:/# pip install -r requirements.txt
# root@job-93b5db29-78cd-45a7-925f-41a9c2fd5669:/# python main.py
# $ neuro sync-down results/  # if script was writing to results/
# #OR:
# $ CMD="cd /project/$(neuro pwd-relative) && pip install -r requirements.txt && python main.py"
# $ neuro r -s gpu-small train `neuro base-image` $CMD

storage://neuro-public/mvasilkovsky/detector/config/detector is truly non-existent, as my project name is mask-detector and it should be storage://neuro-public/mvasilkovsky/mask-detector/config/detector

Thanks, prepended all commands with mkdir -p. Also, added neuro pwd and changed names upload->sync-up and download->sync-down as neuro upload seems unclear IMO. Also, it's not really upload, it's closer to rsync.

neuro ll also new

Does not work for files with mkdir -p. May be we should add a dircheck/filecheck through if and -d or any other.

Also rm or delete would be useful

Fixed! thanks

Added neuro del

added aliases for jobs

Aliases performance test

:( it seems that when aliases call other aliases recursively, each time neuro spends effort on finding and processing .neuro.toml, it doesn't cache

fixed sync-down.

Updated:

1. allow this script to be global (put it to ~/.neuro/user.toml)
2. indicator of project-root is now either .git/ or Makefile
3. rename aliases project-* to p-*

added job runners:

• neuro r-j for jupyter
• neuro r-srv for server (make develop in attached mode)
• neuro r-tb for tensorboard
• `neuro r -s gpu-small train $(neuro base-image) "cd /project/$(neuro pwd-relative) && python train.py" for custom train