python-pip-tools.mk

export SHELL = /bin/bash
MAKEFILE_PATH := $(abspath $(firstword $(MAKEFILE_LIST)))
MAKEFILE_DIR := $(dir $(MAKEFILE_PATH))
PYTHON_VERSION := 3
PYTHON := $(PYTHON_VERSION:%=python%)
PYTHON_ENV_TAG := $(shell .venv/bin/python -c 'import wheel.bdist_wheel as d; print(d.python_tag(), d.get_abi_tag(), d.get_platform(None), sep="-")' 2> /dev/null || printf '\e[33mWARNING\e[0m: missing python virtual environment; run "make venv" first and then retry\n' 1>&2)
REQUIREMENTS_IN := $(wildcard *requirements.in)
#REQUIREMENTS_OUT_DIR := $(MAKEFILE_DIR)
REQUIREMENTS_OUT_DIR := $(abspath $(MAKEFILE_DIR)/requirements-out)/
REQUIREMENTS_OUT := $(REQUIREMENTS_IN:%.in=$(abspath $(REQUIREMENTS_OUT_DIR)/%-$(PYTHON_ENV_TAG).txt))
GIT_CLEAN_ARGS :=
GIT_CLEAN_PATHS := $(PWD)
PYTEST_DEFAULT_ARGS := --durations=0 --verbose --junit-xml=test-report.xml --doctest-modules --log-level=INFO --log-file=pytest.log --log-file-level=DEBUG --log-file-format='%(levelname)s %(asctime)s %(module)s(%(lineno)d): %(message)s' --log-file-date-format='%Y-%m-%d %H:%M:%S' --cov-report=term --cov-report=html --cov-report=xml
PYTEST_ARGS := --cov
PYTEST_PATHS := tests
DOCKER_OPTS := --rm
DOCKER_IMAGE := python:$(PYTHON_VERSION)
DOCKER_CMD := bash -c 'make clean clean_venv venv && make update_all'

define print-rule-info =
@printf '\n%s\n' "--------------------------------------------------------------------------------"
@printf '%-5s %-20s \e[32m%-15s\e[0m\n' "" "" "RULE INFO"
@printf '%-20s %-2s \e[33m%-15s\e[0m\n' "" "target:" "$@"
@printf '%-20s %-2s \e[31m%-15s\e[0m\n' "" "all prerequisites:" "$^"
@printf '%-20s %-2s \e[31m%-15s\e[0m\n' "" "changed prerequisites:" "$?"
@printf '%s\n' "--------------------------------------------------------------------------------"
endef

all: install

help:
	@echo "usage: make [TARGET]"
	@echo "TARGETS"
	@echo "	sync|install:"
	@echo "		install/synchronize all production dependencies (this is the default if no target(s) are given)"
	@echo ""
	@echo "	sync_all|sync_dev|install_all|install_dev:"
	@echo "		install/synchronize all production and dev dependencies (and any specified in other *-requirements.in files)"
	@echo ""
	@echo "	recompile|update:"
	@echo "		(re)compile production requirements file with up-to-date dependencies"
	@echo ""
	@echo "	recompile_all|recompile_dev|update_all|update_dev:"
	@echo "		(re)compile all (production, dev, etc.) requirements files with up-to-date dependencies"
	@echo ""
	@echo "	upgrade:"
	@echo "		update/recompile then synchronize production requirements"
	@echo ""
	@echo "	upgrade_all|upgrade_dev:"
	@echo "		update/recompile then synchronize all requirements (prod, dev, etc.)"
	@echo ""
	@echo "	test:"
	@echo "		install/synchronize all production and dev dependencies (and any specified in other *-requirements.in files) and run tests"
	@echo ""

clean:
	$(print-rule-info)
	@echo 'Cleaning up'
	git clean -fx \
		-e '.venv' \
		-e '**/.python-version' \
		$(shell git ls-files -o --exclude-standard -z | xargs -0 printf -- "-e '%s'\n") \
		$(GIT_CLEAN_ARGS) $(GIT_CLEAN_PATHS)

clean_venv:
	$(print-rule-info)
	rm -rf .venv/

venv:
	$(print-rule-info)
	@echo 'Checking/setting up virtual environment...'
	if ! .venv/bin/python -m wheel version > /dev/null 2>&1 || ! .venv/bin/pip-compile --version > /dev/null 2>&1; then \
		rm -rf .venv/; \
		$(PYTHON) -m venv .venv; \
		.venv/bin/pip install --progress-bar off --upgrade pip setuptools wheel pip-tools; \
	fi

$(REQUIREMENTS_OUT): $(REQUIREMENTS_OUT_DIR)%-$(PYTHON_ENV_TAG).txt: %.in
	$(print-rule-info)
	@echo "Compiling $?"
	if [ ! -d "$(dir $@)" ]; then \
		mkdir -pv "$(dir $@)"; \
	fi
	.venv/bin/pip-compile -v --pip-args '--progress-bar off' --upgrade --allow-unsafe --generate-hashes --emit-index-url --output-file $@ $<
	sed -E -e 's/^(-e.+|.+#egg=.+)/#\1/' $@ > $@.bak && mv -fv $@.bak $@
	for file_url in $$(grep -Eio 'file:[^[:space:]?#]+' $@ | sort | uniq); do \
		file_path="$$(printf '%s' "$$file_url" | sed -E 's|^file:(//)?||')"; \
		relative_file_path="$$(realpath --relative-to="$(@D)" "$$file_path")"; \
		sed -E "s|$$file_url|$$relative_file_path|g" $@ > $@.bak && mv -fv $@.bak $@; \
	done
	rm -fv .venv/.last_sync_all $(@:%=.venv/.last_sync_%)

clean_compile:
	$(print-rule-info)
	@echo "Deleting compiled requirements file: requirements-$(PYTHON_ENV_TAG).txt"
	rm -fv $(abspath $(REQUIREMENTS_OUT_DIR)/)requirements-$(PYTHON_ENV_TAG).txt

clean_compile_all clean_compile_dev:
	$(print-rule-info)
	@echo "Deleting all compiled requirements files: $(REQUIREMENTS_OUT)"
	rm -fv $(REQUIREMENTS_OUT)

compile: $(abspath $(REQUIREMENTS_OUT_DIR)/requirements-$(PYTHON_ENV_TAG).txt)
	$(print-rule-info)

compile_all compile_dev: $(REQUIREMENTS_OUT)
	$(print-rule-info)

recompile update: clean_compile compile
	$(print-rule-info)

recompile_all recompile_dev update_all update_dev: clean_compile_all compile_all
	$(print-rule-info)

.venv/.last_sync_%: $(REQUIREMENTS_OUT_DIR)%
	$(print-rule-info)
	@echo "Syncing with dependencies in: $^"
	rm -fv $(wildcard .venv/.last_sync_*)
	.venv/bin/pip-sync --pip-args '--progress-bar off' $^
	for dependency in "$$(grep --extended-regexp --no-filename --no-messages '.+#egg=.+' "$(@:.venv/.last_sync_%-$(PYTHON_ENV_TAG).txt=$(MAKEFILE_DIR)%.in)" | grep --extended-regexp --invert-match '^-e[[:space:]]' | sort | uniq)"; do \
	    if [ -n "$$dependency" ]; then \
	        printf 'installing: %s\n' "$$dependency"; \
	        .venv/bin/pip install --progress-bar off --no-deps $$dependency; \
	    fi \
	done
	for dependency in "$$(grep --extended-regexp --no-filename --no-messages '^-e[[:space:]].+' "$(@:.venv/.last_sync_%-$(PYTHON_ENV_TAG).txt=$(MAKEFILE_DIR)%.in)" | sort | uniq)"; do \
	    if [ -n "$$dependency" ]; then \
	        printf 'installing: %s\n' "$$dependency"; \
	        .venv/bin/pip install --progress-bar off --no-deps $$dependency; \
	    fi \
	done
	touch $@

.venv/.last_sync_all: $(REQUIREMENTS_OUT)
	$(print-rule-info)
	@echo "Syncing with dependencies in: $^"
	rm -fv $(wildcard .venv/.last_sync_*)
	.venv/bin/pip-sync --pip-args '--progress-bar off' $^
	for dependency in "$$(grep --extended-regexp --no-filename --no-messages '^[^#].+#egg=.+' $(REQUIREMENTS_IN) | grep --extended-regexp --invert-match '^-e[[:space:]]' | sort | uniq)"; do \
	    if [ -n "$$dependency" ]; then \
	        printf 'installing: %s\n' "$$dependency"; \
	        .venv/bin/pip install --progress-bar off --no-deps $$dependency; \
	    fi \
	done
	for dependency in "$$(grep --extended-regexp --no-filename --no-messages '^-e[[:space:]].+' $(REQUIREMENTS_IN) | sort | uniq)"; do \
	    if [ -n "$$dependency" ]; then \
	        printf 'installing: %s\n' "$$dependency"; \
	        .venv/bin/pip install --progress-bar off --no-deps $$dependency; \
	    fi \
	done
	touch .venv/.last_sync_all

sync: .venv/.last_sync_requirements-$(PYTHON_ENV_TAG).txt
	$(print-rule-info)

install: venv
	$(print-rule-info)
	$(MAKE) sync

sync_all sync_dev: .venv/.last_sync_all
	$(print-rule-info)

install_all install_dev: venv
	$(print-rule-info)
	$(MAKE) sync_all

upgrade: clean_compile sync
	$(print-rule-info)

upgrade_all upgrade_dev: clean_compile_all sync_all
	$(print-rule-info)

test: .venv/.last_sync_all
	$(print-rule-info)
	.venv/bin/pytest $(PYTEST_DEFAULT_ARGS) $(PYTEST_ARGS) $(PYTEST_PATHS)

docker:
	$(print-rule-info)
	docker run -it $(DOCKER_OPTS) \
		-v "$(PWD):/mnt/src" \
		--workdir /mnt/src \
		$(DOCKER_IMAGE) $(DOCKER_CMD)

.PHONY: all help clean clean_venv venv clean_compile clean_compile_all clean_compile_dev compile compile_all compile_dev recompile update recompile_all recompile_dev update_all update_dev sync install sync_all sync_dev install_all install_dev upgrade upgrade_all upgrade_dev test docker

#GIT_CLEAN_ARGS += -n
PYTEST_DEFAULT_ARGS += --cov-append
#PYTEST_ARGS := --cov
#PYTEST_PATHS := tests
DOCKER_OPTS += -v "$(HOME)/.ssh/known_hosts:/root/.ssh/known_hosts:ro"
#DOCKER_CMD := bash -c 'make clean clean_venv venv && make update_all'