Makefile with yaml configuration for AWS Batch Job project. Defines variables in Makefile based on a target's dependencies.

README.md

Motivation

This Gist contains two important files: a Makefile and a project.yml configuration template. The motivation behind creating this Gist was to provide a ready-to-use setup for managing a project with specific configurations and build processes. By using the Makefile and project.yml, you can streamline your development workflow and easily manage different environments, such as production, testing, and development, within your project. The Makefile is a powerful tool for automating tasks and defining build processes. It leverages the project.yml configuration template to retrieve environment-specific settings and provide flexibility in configuring your project for different use cases. With the Makefile, you can easily build, test, and deploy your project, while the project.yml file allows you to define and manage environment-specific configurations.

Contents

• Makefile: This file contains a set of rules and targets that automate various tasks related to building, testing, and deploying your project. It uses the project.yml configuration template to retrieve environment-specific settings and provide a standardized workflow.
• project.yml: This YAML configuration template is designed to define project-specific and environment-specific settings. It includes sections for the project name, default AWS settings, and different environments such as production, testing, and development. You can customize these settings to match your project requirements and manage different environments effectively.

Note:

Some values in the project.yml file, such as AWS credentials, region, and bucket names, need to be filled in with appropriate values specific to your project and environment. Make sure to review and update these values accordingly before using the Makefile.

Disclaimer

This README was generated mostly by a LLM.

Makefile

.phony: all

all: push

# configuration setup
PROJECT_FILE_NAME=project.yml
CONFIG := python3 -c "import yaml; import json; from pathlib import Path; print(json.dumps(yaml.safe_load(Path(\"$(PROJECT_FILE_NAME)\").read_text())))"
OS := $(shell python3 -c "import platform; print(platform.system())")

# defaults
IMG_NAME := `$(CONFIG) | jq '.project.name' --raw-output`
REGION := `$(CONFIG) | jq '.default.aws.region' --raw-output`
OUTPUT_TYPE := `$(CONFIG) | jq '.default.aws.json' --raw-output`
REV=$(shell git rev-parse HEAD | cut -c1-7)

# debugging
print-config:
	@echo $(IMG_NAME)
	$(CONFIG) | jq '.'

# setup environments
testing-env:
	@echo Using Testing Configuration 🔊
	@$(eval env_name="testing")
	$(eval AWS_PROFILE=`$(CONFIG) | jq '.environment.$(env_name).aws.profile_name'  --raw-output`)
	$(eval AWS_ACCOUNT_ID=`$(CONFIG) | jq '.environment.$(env_name).aws.sso_account_id'  --raw-output`)
	$(eval START_URL=`$(CONFIG) | jq '.environment.$(env_name).aws.sso_start_url'  --raw-output`)
	$(eval SESSION_NAME=`$(CONFIG) | jq '.environment.$(env_name).aws.sso_session_name'  --raw-output`)
	$(eval ROLE_NAME=`$(CONFIG) | jq '.environment.$(env_name).aws.sso_role_name'  --raw-output`)
	$(eval TAG=`$(CONFIG) | jq '.environment.$(env_name).aws.container_tag'  --raw-output`)
	$(eval BUCKET_NAME=`$(CONFIG) | jq '.environment.$(env_name).aws.bucket_name'  --raw-output`)
	$(eval REGISTRY_URL=$(AWS_ACCOUNT_ID).dkr.ecr.$(REGION).amazonaws.com)
	$(eval REPO=$(REGISTRY_URL)/$(IMG_NAME))

production-env:
	@echo Using Production Configuration 🔊
	@$(eval env_name="production")
	$(eval AWS_PROFILE=`$(CONFIG) | jq '.environment.$(env_name).aws.profile_name'  --raw-output`)
	$(eval AWS_ACCOUNT_ID=`$(CONFIG) | jq '.environment.$(env_name).aws.sso_account_id'  --raw-output`)
	$(eval START_URL=`$(CONFIG) | jq '.environment.$(env_name).aws.sso_start_url'  --raw-output`)
	$(eval SESSION_NAME=`$(CONFIG) | jq '.environment.$(env_name).aws.sso_session_name'  --raw-output`)
	$(eval ROLE_NAME=`$(CONFIG) | jq '.environment.$(env_name).aws.sso_role_name'  --raw-output`)
	$(eval TAG=`$(CONFIG) | jq '.environment.$(env_name).aws.container_tag'  --raw-output`)
	$(eval BUCKET_NAME=`$(CONFIG) | jq '.environment.$(env_name).aws.bucket_name'  --raw-output`)
	$(eval REGISTRY_URL=$(AWS_ACCOUNT_ID).dkr.ecr.$(REGION).amazonaws.com)
	$(eval REPO=$(REGISTRY_URL)/$(IMG_NAME))

os:
	@echo Using $(OS) operating system
	@echo

development: os
	$(eval RAW_SRC=`$(CONFIG) | jq '.environment.development.$(OS).raw_src' --raw-output`)
	$(eval PROCESSED_SRC=`$(CONFIG) | jq ".environment.development.$(OS).processed_src" --raw-output`)
	@echo Developing with Directories...
	@echo RAW_SRC=$(RAW_SRC) | xargs -I{} echo " - {}"
	@echo PROCESSED_SRC=$(PROCESSED_SRC) | xargs -I{} echo " - {}"
	@echo
	@mkdir -p $(PROCESSED_SRC)

container:
	$(eval CPUS=`$(CONFIG) | jq '.default.aws.batch.cpus' --raw-output`)
	$(eval MEMORY=`$(CONFIG) | jq '.default.aws.batch.memory' --raw-output`)
	$(eval RAW_TARGET=`$(CONFIG) | jq '.default.aws.batch.raw_target' --raw-output`)
	$(eval PROCESSED_TARGET=`$(CONFIG) | jq '.default.aws.batch.processed_target' --raw-output`)
	$(eval HOSTNAME=`$(CONFIG) | jq '.default.aws.batch.hostname' --raw-output`)

	@echo Container Settings
	@echo CPUS=$(CPUS) | xargs -I{} echo " - {}"
	@echo MEMORY=$(MEMORY) | xargs -I{} echo " - {}"
	@echo RAW_TARGET=$(RAW_TARGET) | xargs -I{} echo " - {}"
	@echo PROCESSED_TARGET=$(PROCESSED_TARGET) | xargs -I{} echo " - {}"
	@echo HOSTNAME=$(HOSTNAME) | xargs -I{} echo " - {}"
	@echo

print-test-env: testing-env
	@echo SESSION_NAME=$(SESSION_NAME) | xargs -I{} echo " - {}"
	@echo AWS_PROFILE=$(AWS_PROFILE) | xargs -I{} echo " - {}"
	@echo REGISTRY_URL=$(REGISTRY_URL) | xargs -I{} echo " - {}"
	@echo REPO=$(REPO) | xargs -I{} echo " - {}"
	@echo REV=$(REV) | xargs -I{} echo " - {}"

print-prod-env: production-env
	@echo AWS_PROFILE=$(AWS_PROFILE) | xargs -I{} echo " - {}"
	@echo REGISTRY_URL=$(REGISTRY_URL) | xargs -I{} echo " - {}"
	@echo REPO=$(REPO) | xargs -I{} echo " - {}"
	@echo REV=$(REV) | xargs -I{} echo " - {}"

# sso management
configure:
	aws configure sso

setup-accounts:
	bash scripts/setup-aws-config.sh testing
	bash scripts/setup-aws-config.sh production

login-prod: production-env
	aws sso login --profile $(AWS_PROFILE)

login-test: testing-env
	aws sso login --profile $(AWS_PROFILE)

# container management
ecr-login:
	@# either call one of the login targets or have an environment co-depended on
	@aws ecr get-login-password --profile $(AWS_PROFILE) \
		| docker login --password-stdin --username AWS $(REGISTRY_URL) \
		|| echo "\033[0;31mFailed to log into ecr! Be sure you're logged into an sso profile."

build-prod: production-env
	@env DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build \
		--build-arg CURRENT_SHA=$(REV) \
		--build-arg BUCKET_NAME=$(BUCKET_NAME) \
		-t $(IMG_NAME):$(TAG) .
	@echo Built $(TAG) ⚒️

build-test: testing-env
	@env DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build \
		--build-arg CURRENT_SHA=$(REV) \
		--build-arg BUCKET_NAME=$(BUCKET_NAME) \
		-t $(IMG_NAME):$(TAG) .
	@echo Built $(TAG) ⚒️

build: build-test build-prod
	@echo Done Building ⚒️

# Push docker image
confirm:
	@read -p "Do you really want to push to $(env_name)? (y/n) " answer && [ "$$answer" = "y" ] || (echo "Aborted." && exit 1)

push: login-prod production-env ecr-login build confirm
	@docker tag $(IMG_NAME):$(TAG) $(REPO):$(TAG)
	@docker tag $(IMG_NAME):$(TAG) $(REPO):$(REV)
	@docker push $(REPO):$(TAG)
	@docker push $(REPO):$(REV)

push-test: testing-env ecr-login build confirm
	@docker tag $(IMG_NAME):$(TAG) $(REPO):$(TAG)
	@docker tag $(IMG_NAME):$(TAG) $(REPO):$(REV)
	@docker push $(REPO):$(TAG)
	@docker push $(REPO):$(REV)

# Development / Debugging Container
enter-dev: os development container
	@# without gpu's activated
	@docker run \
		--cpus=$(CPUS) \
		--memory=$(MEMORY) \
		--mount src=$(RAW_SRC),target=$(RAW_TARGET),type=bind,readonly \
		--mount src=$(PROCESSED_SRC),target=$(PROCESSED_TARGET),type=bind \
		--hostname $(HOSTNAME) \
		--entrypoint /bin/bash \
		-it $(IMG_NAME) || echo "\033[0;31mFailed to enter. Check that your mounts exist."

enter: os development container
	@# with gpu's activated
	@docker run \
		--gpus all \
		--cpus=$(CPUS) \
		--memory=$(MEMORY) \
		--mount src=$(RAW_SRC),target=$(RAW_TARGET),type=bind,readonly \
		--mount src=$(PROCESSED_SRC),target=$(PROCESSED_TARGET),type=bind \
		--hostname $(HOSTNAME) \
		--entrypoint /bin/bash \
		-it $(IMG_NAME) || echo "\033[0;31mFailed to enter. Check that your mounts exist."

run-dev: development container
	@# Syntax:
	@# 	make input_dir=<YOUR_INPUT_DIR> output_dir=<YOUR_OUTPUT_DIR>  run-dev

	@echo Arguments:
	@echo input_dir = "$(input_dir)" | xargs -I{} echo " - {}"
	@echo output_dir = "$(output_dir)" | xargs -I{} echo " - {}"
	@echo

	@# without gpu's activated
	@docker run \
		--cpus=$(CPUS) \
		--memory=$(MEMORY) \
		--mount src=$(RAW_SRC),target=$(RAW_TARGET),type=bind,readonly \
		--mount src=$(PROCESSED_SRC),target=$(PROCESSED_TARGET),type=bind \
		--hostname $(HOSTNAME) \
		-it $(IMG_NAME) \
			$(input_dir) \
			$(output_dir) \
		|| echo "\033[0;31mFailed to enter. Check that your mounts exist."

run: development container
	@# Syntax:
	@# 	make input_dir=<YOUR_INPUT_DIR> output_dir=<YOUR_OUTPUT_DIR>  run-dev

	@echo Arguments:
	@echo input_dir = "$(input_dir)" | xargs -I{} echo " - {}"
	@echo output_dir = "$(output_dir)" | xargs -I{} echo " - {}"
	@echo
	
	@# with gpu's activated
	@docker run \
		--gpus all \
		--cpus=$(CPUS) \
		--memory=$(MEMORY) \
		--mount src=$(RAW_SRC),target=$(RAW_TARGET),type=bind,readonly \
		--mount src=$(PROCESSED_SRC),target=$(PROCESSED_TARGET),type=bind \
		--hostname $(HOSTNAME) \
		-it $(IMG_NAME) \
			$(input_dir) \
			$(output_dir) \
		|| echo "\033[0;31mFailed to enter. Check that your mounts exist."

test:
	@# TODO: implement me!
	@echo Not Implented Yet

project.yml

project:
  name: &project_name noneya

default:
    aws: &default_aws
      region:
      output: json
      sso_start_url: 
      batch: # TODO: could expand to other compute environments
        # docker run arguments
        cpus: 8
        memory: 30g
        # docker run --mount args
        raw_target: /mnt/raw_data
        processed_target: /mnt/processed_data
        hostname: batch_job
  
    application:
      # TODO: application-specific defaults
  
environment:
  production:
    aws:
      <<: *default_aws
      bucket_name: 
      profile_name: prod
      sso_account_id: 
      sso_role_name: 
      sso_session_name: prod
      container_tag: prod

  testing:
    aws:
      <<: *default_aws
      bucket_name: 
      profile_name: testing
      sso_account_id: 
      sso_role_name: 
      sso_session_name: testing
      container_tag: test

  development:
    Darwin: # macOS
      # in Finder you can use "command + k" to open up the mounting configuration
      raw_src: /Volumes/Raw
      # data is automatically deleted on restart
      processed_src: /tmp/noneya/
    Linux:
      # TODO: define
    Windows:
      # TODO: define

setup-aws-config.sh

#! /bin/bash
PROJECT_FILE_NAME=project.yml
CONFIG=`python3 -c "import yaml; import json; from pathlib import Path; print(json.dumps(yaml.safe_load(Path(\"$PROJECT_FILE_NAME\").read_text())))" `
echo $CONFIG | jq 

if [ $# -eq 0 ]; then
  echo "No argument supplied. Exiting."
  exit 1
fi

env_name=$1
echo Setting up for $env_name environment
AWS_PROFILE=`echo $CONFIG | jq ".environment.$env_name.aws.profile_name"  --raw-output`
REGION=`echo $CONFIG | jq ".environment.$env_name.aws.region"  --raw-output`
OUTPUT_TYPE=`echo $CONFIG | jq ".environment.$env_name.aws.output"  --raw-output`
IMG_NAME=`echo $CONFIG | jq '.project.name' --raw-output`
AWS_ACCOUNT_ID=`echo $CONFIG | jq ".environment.$env_name.aws.sso_account_id" --raw-output`
START_URL=`echo $CONFIG | jq ".environment.$env_name.aws.sso_start_url" --raw-output`
SESSION_NAME=`echo $CONFIG | jq ".environment.$env_name.aws.sso_session_name" --raw-output`
ROLE_NAME=`echo $CONFIG | jq ".environment.$env_name.aws.sso_role_name" --raw-output`
REGISTRY_URL=$AWS_ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com
REPO=$REGISTRY_URL/$IMG_NAME

# append to ~/.aws/config
echo -e "\n[profile $AWS_PROFILE]\n	sso_session = $SESSION_NAME\n	sso_account_id = $AWS_ACCOUNT_ID\n	sso_role_name = $ROLE_NAME\n	region = $REGION\n	output = $OUTPUT_TYPE\n	sso_start_url = $START_URL" >>  ~/.aws/config

Wow! Very useful