Skip to content

Instantly share code, notes, and snippets.

@ma7dev

ma7dev/ci.yaml

Created July 7, 2021 20:58
Show Gist options
  • Save ma7dev/c4200da526c8ce5c6d0534179f48335f to your computer and use it in GitHub Desktop.
Save ma7dev/c4200da526c8ce5c6d0534179f48335f to your computer and use it in GitHub Desktop.
Download ZIP
Example for my jobs setup
Raw
ci.yaml
name: Automated Experiments
on:
push:
branches: [ main ]
jobs:
build_on_cn-gpu5:
runs-on: [ cn-gpu5 ]
timeout-minutes: 4320
defaults:
run:
shell: bash -l {0}
env:
JOB: "5"
EXPERIMENTS: "detection"
CONDA: "mot"
CUDA_HOME: "/usr/local/apps/cuda/cuda-10.1"
CMAKE_HOME: "/usr/local/apps/cmake/3.14.5"
steps:
- uses: actions/checkout@v2
- name: set environment variables
run: |
echo "---------------set environment variables---------------"
export DATE=$(date "+%Y-%m-%d-%H-%M-%S")
export SCRATCH=/nfs/hpc/share/$USER
export WORKSPACE=$SCRATCH/actions-runner-${{ env.JOB }}/_work/mot/mot
export CONFIG="$WORKSPACE/experiments/cfgs/cn-gpu${{ env.JOB }}.yml"
export LOGGER=$SCRATCH/mot/outputs/logs
export PYTHON=$SCRATCH/m33/envs/$CONDA/bin/python
echo "---------------add to GITHUB ENV---------------"
echo "DATE=$DATE" >> $GITHUB_ENV
echo "SCRATCH=$SCRATCH" >> $GITHUB_ENV
echo "WORKSPACE=$WORKSPACE" >> $GITHUB_ENV
echo "CONFIG=$CONFIG" >> $GITHUB_ENV
echo "LOGGER=$LOGGER" >> $GITHUB_ENV
echo "PYTHON=$PYTHON" >> $GITHUB_ENV
- name: set permission
run: |
echo x-x-x-x-x-x-x-x-x-x-x ${{ env.DATE }} x-x-x-x-x-x-x-x-x-x-x
echo "---------------source bash---------------"
. ~/.bashrc
echo "---------------permissions---------------"
chmod -R 777 ${{ env.WORKSPACE }}
chmod -R 777 ${{ env.LOGGER }}
echo "---------------create log files---------------"
IFS=','
read -a arr <<< "${{ env.EXPERIMENTS }}"
for i in "${arr[@]}"
do
echo "${{ env.LOGGER }}/${{ env.DATE }}_${i}.log"
touch "${{ env.LOGGER }}/${{ env.DATE }}_${i}.log";
chmod 777 "${{ env.LOGGER }}/${{ env.DATE }}_${i}.log";
done
- name: server information
run: |
echo x-x-x-x-x-x-x-x-x-x-x ${{ env.DATE }} x-x-x-x-x-x-x-x-x-x-x
echo "---------------source bash---------------"
. ~/.bashrc
echo "---------------CUDA---------------"
export PATH="${{ env.CUDA_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CUDA_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------CMAKE---------------"
export PATH="${{ env.CMAKE_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CMAKE_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------source conda env---------------"
conda activate ${{ env.CONDA }}
echo "---------------info---------------"
python -V
conda list
nvidia-smi
ifconfig
- name: start a job
run: |
echo x-x-x-x-x-x-x-x-x-x-x ${{ env.DATE }} x-x-x-x-x-x-x-x-x-x-x
echo "---------------source bash---------------"
. ~/.bashrc
echo "---------------CUDA---------------"
export PATH="${{ env.CUDA_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CUDA_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------CMAKE---------------"
export PATH="${{ env.CMAKE_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CMAKE_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------source conda env---------------"
conda activate ${{ env.CONDA }}
echo "---------------run experiments---------------"
cd ${{ env.WORKSPACE }}/experiments/scripts/
${{ env.PYTHON }} experiments.py --config=${{ env.CONFIG }} --selected="${{ env.EXPERIMENTS }}" --logger="${{ env.LOGGER }}/${{ env.DATE }}"
build_on_cn-gpu5-2:
runs-on: [ cn-gpu5-2 ]
timeout-minutes: 4320
defaults:
run:
shell: bash -l {0}
env:
JOB: "5-2"
EXPERIMENTS: "joint"
CONDA: "mot"
CUDA_HOME: "/usr/local/apps/cuda/cuda-10.1"
CMAKE_HOME: "/usr/local/apps/cmake/3.14.5"
steps:
- uses: actions/checkout@v2
- name: set environment variables
run: |
echo "---------------set environment variables---------------"
export DATE=$(date "+%Y-%m-%d-%H-%M-%S")
export SCRATCH=/nfs/hpc/share/$USER
export WORKSPACE=$SCRATCH/actions-runner-${{ env.JOB }}/_work/mot/mot
export CONFIG="$WORKSPACE/experiments/cfgs/cn-gpu${{ env.JOB }}.yml"
export LOGGER=$SCRATCH/mot/outputs/logs
export PYTHON=$SCRATCH/m33/envs/$CONDA/bin/python
echo "---------------add to GITHUB ENV---------------"
echo "DATE=$DATE" >> $GITHUB_ENV
echo "SCRATCH=$SCRATCH" >> $GITHUB_ENV
echo "WORKSPACE=$WORKSPACE" >> $GITHUB_ENV
echo "CONFIG=$CONFIG" >> $GITHUB_ENV
echo "LOGGER=$LOGGER" >> $GITHUB_ENV
echo "PYTHON=$PYTHON" >> $GITHUB_ENV
- name: set permission
run: |
echo x-x-x-x-x-x-x-x-x-x-x ${{ env.DATE }} x-x-x-x-x-x-x-x-x-x-x
echo "---------------source bash---------------"
. ~/.bashrc
echo "---------------permissions---------------"
chmod -R 777 ${{ env.WORKSPACE }}
chmod -R 777 ${{ env.LOGGER }}
echo "---------------create log files---------------"
IFS=','
read -a arr <<< "${{ env.EXPERIMENTS }}"
for i in "${arr[@]}"
do
echo "${{ env.LOGGER }}/${{ env.DATE }}_${i}.log"
touch "${{ env.LOGGER }}/${{ env.DATE }}_${i}.log";
chmod 777 "${{ env.LOGGER }}/${{ env.DATE }}_${i}.log";
done
- name: server information
run: |
echo x-x-x-x-x-x-x-x-x-x-x ${{ env.DATE }} x-x-x-x-x-x-x-x-x-x-x
echo "---------------source bash---------------"
. ~/.bashrc
echo "---------------CUDA---------------"
export PATH="${{ env.CUDA_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CUDA_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------CMAKE---------------"
export PATH="${{ env.CMAKE_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CMAKE_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------source conda env---------------"
conda activate ${{ env.CONDA }}
echo "---------------info---------------"
python -V
conda list
nvidia-smi
ifconfig
- name: start a job
run: |
echo x-x-x-x-x-x-x-x-x-x-x ${{ env.DATE }} x-x-x-x-x-x-x-x-x-x-x
echo "---------------source bash---------------"
. ~/.bashrc
echo "---------------CUDA---------------"
export PATH="${{ env.CUDA_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CUDA_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------CMAKE---------------"
export PATH="${{ env.CMAKE_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CMAKE_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------source conda env---------------"
conda activate ${{ env.CONDA }}
echo "---------------run experiments---------------"
cd ${{ env.WORKSPACE }}/experiments/scripts/
${{ env.PYTHON }} experiments.py --config=${{ env.CONFIG }} --selected="${{ env.EXPERIMENTS }}" --logger="${{ env.LOGGER }}/${{ env.DATE }}"
build_on_cn-gpu6:
runs-on: [ cn-gpu6 ]
timeout-minutes: 4320
defaults:
run:
shell: bash -l {0}
env:
JOB: "6"
EXPERIMENTS: "tracking_bce"
CONDA: "mot"
CUDA_HOME: "/usr/local/apps/cuda/cuda-10.1"
CMAKE_HOME: "/usr/local/apps/cmake/3.14.5"
steps:
- uses: actions/checkout@v2
- name: set environment variables
run: |
echo "---------------set environment variables---------------"
export DATE=$(date "+%Y-%m-%d-%H-%M-%S")
export SCRATCH=/nfs/hpc/share/$USER
export WORKSPACE=$SCRATCH/actions-runner-${{ env.JOB }}/_work/mot/mot
export CONFIG="$WORKSPACE/experiments/cfgs/cn-gpu${{ env.JOB }}.yml"
export LOGGER=$SCRATCH/mot/outputs/logs
export PYTHON=$SCRATCH/m33/envs/$CONDA/bin/python
echo "---------------add to GITHUB ENV---------------"
echo "DATE=$DATE" >> $GITHUB_ENV
echo "SCRATCH=$SCRATCH" >> $GITHUB_ENV
echo "WORKSPACE=$WORKSPACE" >> $GITHUB_ENV
echo "CONFIG=$CONFIG" >> $GITHUB_ENV
echo "LOGGER=$LOGGER" >> $GITHUB_ENV
echo "PYTHON=$PYTHON" >> $GITHUB_ENV
- name: set permission
run: |
echo x-x-x-x-x-x-x-x-x-x-x ${{ env.DATE }} x-x-x-x-x-x-x-x-x-x-x
echo "---------------source bash---------------"
. ~/.bashrc
echo "---------------permissions---------------"
chmod -R 777 ${{ env.WORKSPACE }}
chmod -R 777 ${{ env.LOGGER }}
echo "---------------create log files---------------"
IFS=','
read -a arr <<< "${{ env.EXPERIMENTS }}"
for i in "${arr[@]}"
do
echo "${{ env.LOGGER }}/${{ env.DATE }}_${i}.log"
touch "${{ env.LOGGER }}/${{ env.DATE }}_${i}.log";
chmod 777 "${{ env.LOGGER }}/${{ env.DATE }}_${i}.log";
done
- name: server information
run: |
echo x-x-x-x-x-x-x-x-x-x-x ${{ env.DATE }} x-x-x-x-x-x-x-x-x-x-x
echo "---------------source bash---------------"
. ~/.bashrc
echo "---------------CUDA---------------"
export PATH="${{ env.CUDA_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CUDA_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------CMAKE---------------"
export PATH="${{ env.CMAKE_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CMAKE_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------source conda env---------------"
conda activate ${{ env.CONDA }}
echo "---------------info---------------"
python -V
conda list
nvidia-smi
ifconfig
- name: start a job
run: |
echo x-x-x-x-x-x-x-x-x-x-x ${{ env.DATE }} x-x-x-x-x-x-x-x-x-x-x
echo "---------------source bash---------------"
. ~/.bashrc
echo "---------------CUDA---------------"
export PATH="${{ env.CUDA_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CUDA_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------CMAKE---------------"
export PATH="${{ env.CMAKE_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CMAKE_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------source conda env---------------"
conda activate ${{ env.CONDA }}
echo "---------------run experiments---------------"
cd ${{ env.WORKSPACE }}/experiments/scripts/
${{ env.PYTHON }} experiments.py --config=${{ env.CONFIG }} --selected="${{ env.EXPERIMENTS }}" --logger="${{ env.LOGGER }}/${{ env.DATE }}"
build_on_cn-gpu6-2:
runs-on: [ cn-gpu6-2 ]
timeout-minutes: 4320
defaults:
run:
shell: bash -l {0}
env:
JOB: "6-2"
EXPERIMENTS: "tracking_fcl"
CONDA: "mot"
CUDA_HOME: "/usr/local/apps/cuda/cuda-10.1"
CMAKE_HOME: "/usr/local/apps/cmake/3.14.5"
steps:
- uses: actions/checkout@v2
- name: set environment variables
run: |
echo "---------------set environment variables---------------"
export DATE=$(date "+%Y-%m-%d-%H-%M-%S")
export SCRATCH=/nfs/hpc/share/$USER
export WORKSPACE=$SCRATCH/actions-runner-${{ env.JOB }}/_work/mot/mot
export CONFIG="$WORKSPACE/experiments/cfgs/cn-gpu${{ env.JOB }}.yml"
export LOGGER=$SCRATCH/mot/outputs/logs
export PYTHON=$SCRATCH/m33/envs/$CONDA/bin/python
echo "---------------add to GITHUB ENV---------------"
echo "DATE=$DATE" >> $GITHUB_ENV
echo "SCRATCH=$SCRATCH" >> $GITHUB_ENV
echo "WORKSPACE=$WORKSPACE" >> $GITHUB_ENV
echo "CONFIG=$CONFIG" >> $GITHUB_ENV
echo "LOGGER=$LOGGER" >> $GITHUB_ENV
echo "PYTHON=$PYTHON" >> $GITHUB_ENV
- name: set permission
run: |
echo x-x-x-x-x-x-x-x-x-x-x ${{ env.DATE }} x-x-x-x-x-x-x-x-x-x-x
echo "---------------source bash---------------"
. ~/.bashrc
echo "---------------permissions---------------"
chmod -R 777 ${{ env.WORKSPACE }}
chmod -R 777 ${{ env.LOGGER }}
echo "---------------create log files---------------"
IFS=','
read -a arr <<< "${{ env.EXPERIMENTS }}"
for i in "${arr[@]}"
do
echo "${{ env.LOGGER }}/${{ env.DATE }}_${i}.log"
touch "${{ env.LOGGER }}/${{ env.DATE }}_${i}.log";
chmod 777 "${{ env.LOGGER }}/${{ env.DATE }}_${i}.log";
done
- name: server information
run: |
echo x-x-x-x-x-x-x-x-x-x-x ${{ env.DATE }} x-x-x-x-x-x-x-x-x-x-x
echo "---------------source bash---------------"
. ~/.bashrc
echo "---------------CUDA---------------"
export PATH="${{ env.CUDA_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CUDA_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------CMAKE---------------"
export PATH="${{ env.CMAKE_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CMAKE_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------source conda env---------------"
conda activate ${{ env.CONDA }}
echo "---------------info---------------"
python -V
conda list
nvidia-smi
ifconfig
- name: start a job
run: |
echo x-x-x-x-x-x-x-x-x-x-x ${{ env.DATE }} x-x-x-x-x-x-x-x-x-x-x
echo "---------------source bash---------------"
. ~/.bashrc
echo "---------------CUDA---------------"
export PATH="${{ env.CUDA_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CUDA_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------CMAKE---------------"
export PATH="${{ env.CMAKE_HOME }}/bin:$PATH"
export LD_LIBRARY_PATH="${{ env.CMAKE_HOME }}/lib64:$LD_LIBRARY_PATH"
echo "---------------source conda env---------------"
conda activate ${{ env.CONDA }}
echo "---------------run experiments---------------"
cd ${{ env.WORKSPACE }}/experiments/scripts/
${{ env.PYTHON }} experiments.py --config=${{ env.CONFIG }} --selected="${{ env.EXPERIMENTS }}" --logger="${{ env.LOGGER }}/${{ env.DATE }}"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment