Skip to content

Instantly share code, notes, and snippets.

@tikenn
Created March 20, 2019 22:45
Show Gist options
  • Save tikenn/7b4c21d0f48914f92eb8a433fae419c3 to your computer and use it in GitHub Desktop.
Save tikenn/7b4c21d0f48914f92eb8a433fae419c3 to your computer and use it in GitHub Desktop.
Creates the directory structure and environments for python (conda) and nodejs for a machine learning project
#!/bin/bash
#
# --------------------------------------------------------------------------------------------
# Machine Learning Project Directory and Environment Setup
# --------------------------------------------------------------------------------------------
# Automatically generates a basic scaffold for machine learning projects. Sets up
# the directory structure along with both a nodejs and python environment as well.
# Environment can be modified as needed with additional packages installed for both
# python and nodejs as well as additional directories being created.
#
# --------------------------------------------------------------------------------------------
# Author Info
# --------------------------------------------------------------------------------------------
# Name :: Tim Kennell Jr. ~ tikenn
#
# --------------------------------------------------------------------------------------------
# Config
# --------------------------------------------------------------------------------------------
# LICENSE_FILE :: the location of the license file to attach to the project
#
# --------------------------------------------------------------------------------------------
# Setting up crontab
# --------------------------------------------------------------------------------------------
# - Create a file in /etc/cron.d/
# - Suggested to run the file once a week
# - Example line (runs at midnight): "0 0 * * 1 /path/to/le-renew-haproxy"
#
# ~ tikenn
LICENSE_FILE='licenses/MIT.txt'
# Detects "y", "n", "yes", and "no" user response in a case-insensitive manner
# param String $1
## Ex: yes_response "y" --> returns 0
## Ex: yes_response "yes" --> returns 0
## Ex: yes_response "n" --> returns 0
## Ex: yes_response "no" --> returns 0
## Ex: yes_response "" --> returns 1
## Ex: yes_response "gobble de guk" --> returns 1
yes_no_response() {
if [[ $(echo "$1" | tr -s '[:upper:]' '[:lower:]') = y ]] \
|| [[ $(echo "$1" | tr -s '[:upper:]' '[:lower:]') = yes ]] \
|| [[ $(echo "$1" | tr -s '[:upper:]' '[:lower:]') = n ]] \
|| [[ $(echo "$1" | tr -s '[:upper:]' '[:lower:]') = no ]] ; then
return 0
else
return 1
fi
}
# Detect "y" or "yes" response as affirmative answer in case-insensitive manner
# param String $1
## Ex: yes_response "yes" --> returns 0
## Ex: yes_response "no" --> returns 1
## Ex: yes_response "gobble de guk" --> returns 1
yes_response() {
if [[ $(echo "$1" | tr -s '[:upper:]' '[:lower:]') = y ]] \
|| [[ $(echo "$1" | tr -s '[:upper:]' '[:lower:]') = yes ]] ; then
return 0
else
return 1
fi
}
# Detect "n" or "no" response as affirmative answer in case-insensitive manner
# param String $1
## Ex: no_response "no" --> returns 0
## Ex: no_response "yes" --> returns 1
## Ex: no_response "gobble de guk" --> returns 1
no_response() {
if [[ $(echo "$1" | tr -s '[:upper:]' '[:lower:]') = n ]] \
|| [[ $(echo "$1" | tr -s '[:upper:]' '[:lower:]') = no ]] ; then
return 0
else
return 1
fi
}
# Asks for the name of the project
get_project_name() {
local project_name
while [[ -z ${project_name} ]] ; do
read -p "Project name: " project_name
[[ -z "${project_name}" ]] && echo "A project name must be supplied"
done
# return value
echo "$project_name"
}
# Asks a user for the location to create the project in
# @return {String} The folder location to create the project
ml_project_creation_folder() {
local folder
local current_folder_creation
while ! yes_no_response "$current_folder_creation" ; do
read -p "Create project in the current folder [Y/n]: " current_folder_creation
if [[ -z "$current_folder_creation" ]] || yes_response ${current_folder_creation} ; then
current_folder_creation='y'
folder='.'
elif no_response ${current_folder_creation} ; then
while [[ -z ${folder} ]] ; do
read -p "Project folder location (abs path): " folder
[[ -z ${folder} ]] && echo "Must provide a folder!"
done
else
echo "This program speaks english, please try again..."
fi
done
# return value
echo "$folder"
}
# --------------------------------
# Get Project Name
# --------------------------------
PROJECT_NAME=''
while [[ -z ${PROJECT_NAME} ]] ; do
read -p "Project name: " PROJECT_NAME
[[ -z "${PROJECT_NAME}" ]] && echo "A project name must be supplied"
done
echo "$PROJECT_NAME"
# --------------------------------
# Get Project Folder Location
# --------------------------------
PROJECT_PARENT_FOLDER=''
current_folder_creation=''
while ! yes_no_response "$current_folder_creation" ; do
read -p "Create project in the current folder [Y/n]: " current_folder_creation
if [[ -z "$current_folder_creation" ]] || yes_response ${current_folder_creation} ; then
current_folder_creation='y'
PROJECT_PARENT_FOLDER='.'
elif no_response ${current_folder_creation} ; then
while [[ -z ${PROJECT_PARENT_FOLDER} ]] ; do
read -p "Project folder location (abs path): " PROJECT_PARENT_FOLDER
[[ -z ${PROJECT_PARENT_FOLDER} ]] && echo "Must provide a folder!"
done
else
echo "This program speaks english, please try again..."
fi
done
echo "$PROJECT_PARENT_FOLDER"
# --------------------------------
# Create project folders
# --------------------------------
# Project directory
if [[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME" ]] ; then
echo "Error: project ($PROJECT_PARENT_FOLDER/$PROJECT_NAME) already exists; terminating..."
exit 1
fi
[[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME" ]] || mkdir "$PROJECT_PARENT_FOLDER/$PROJECT_NAME"
# High level directories for managing configurations and data
[[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/config" ]] || mkdir "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/config"
[[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/databases" ]] || mkdir "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/databases"
# Lib directories for universal functions and objects
[[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/lib/data" ]] || mkdir -p "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/lib/data"
[[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/lib/features" ]] || mkdir -p "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/lib/features"
[[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/lib/models" ]] || mkdir -p "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/lib/models"
[[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/lib/visualizations" ]] || mkdir -p "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/lib/visualizations"
[[ -f "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/lib/__init__.py" ]] || touch "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/lib/__init__.py"
# Training directories for iterating through machine learning process
[[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/train/iteration 1/feature_exploration" ]] || mkdir -p "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/train/iteration 1/feature_exploration"
[[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/train/iteration 1/log" ]] || mkdir -p "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/train/iteration 1/log"
[[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/train/iteration 1/results" ]] || mkdir -p "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/train/iteration 1/results"
[[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/train/iteration 1/visualizations" ]] || mkdir -p "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/train/iteration 1/visualizations"
[[ -d "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/train/iteration 1/model" ]] || mkdir -p "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/train/iteration 1/model"
# --------------------------------
# License and Readme
# --------------------------------
cat "$LICENSE_FILE" | sed -r \
-e 's/\[year\]/'$(date +"%Y")'/i' \
-e 's/\[fullname\]/Timothy Kennell Jr\./i' \
> "$PROJECT_PARENT_FOLDER/$PROJECT_NAME/LICENSE"
# --------------------------------
# Setup NodeJS environment
# --------------------------------
CWD=$(pwd)
cd "$PROJECT_PARENT_FOLDER/$PROJECT_NAME"
npm init
npm install mysql
# --------------------------------
# Setup Python environment
# --------------------------------
conda_environment_name=$(echo "$PROJECT_NAME" | tr '[:upper:]' '[:lower:]' | sed -re 's/ /_/g')
conda create -n "$conda_environment_name"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment