skylight-rust Makefile

deps-Makefile

# Absolute root of the project
ROOT := $(shell cd $(dir $(firstword $(MAKEFILE_LIST))) && pwd)

include $(ROOT)/../mk/*.mk

# Darwin host string, we only care about 64bit darwin
DARWIN := x86_64-darwin

# Extra flags we want when compiling libs
CFLAGS := -fpic -ffunction-sections -fdata-sections

ifdef NEEDS_SSL
	LIBS := $(HOST)/libssl.a $(HOST)/libcrypto.a
endif

ifdef IS_MUSL
	export CC=musl-gcc
endif

LIBS += $(HOST)/libcurl.a $(HOST)/libz.a

ifdef IS_64B
	ZLIB_CFG = -m64
else
	ZLIB_CFG = -m32
endif

TARGET = $(ROOT)/$(HOST)

OSSL_VER = openssl-1.0.1p
OSSL_URL = https://www.openssl.org/source/$(OSSL_VER).tar.gz
OSSL_DIR = $(HOST)/$(OSSL_VER)
OSSL_CFG = ./config

# Must come after openssl config opts
CURL_VER = curl-7.43.0
CURL_URL = https://curl.haxx.se/download/$(CURL_VER).tar.gz
CURL_DIR = $(HOST)/$(CURL_VER)
CURL_OPT = --with-ssl=$(TARGET)

ZLIB_VER = zlib-1.2.11
# Not ideal that this isn't https, but zlib doesn't appear to have https as default and it's not correctly configured
ZLIB_URL = http://zlib.net/$(ZLIB_VER).tar.gz
ZLIB_DIR = $(HOST)/$(ZLIB_VER)
ZLIB_SHA = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1

# OpenSSL on darwin has a custom configure script for some reason
ifdef IS_DARWIN
	OSSL_CFG = ./Configure darwin64-x86_64-cc
	CURL_OPT = --with-darwinssl --without-ssl
endif

all: skylight-rust-deps-$(HOST).tar.gz

prereqs:
	@echo "Checking prereqs"
	@echo `pwd`

skylight-rust-deps-$(HOST).tar.gz: $(LIBS)
	cd $(HOST) && tar czvf ../skylight-rust-deps-$(HOST).tar.gz *.a

#
#
# ==== OpenSSL ====
#
#

# TODO: Perform checksum
$(OSSL_VER).tar.gz: | $(HOST)
	curl -L "$(OSSL_URL)" > $(OSSL_VER).tar.gz

$(OSSL_DIR): $(OSSL_VER).tar.gz
	tar -xzv -C $(HOST) -f $<

$(OSSL_DIR)/libssl.a $(OSSL_DIR)/libcrypto.a: | $(OSSL_DIR)
	cd $(OSSL_DIR) && \
		$(OSSL_CFG) threads shared no-krb5 no-err $(CFLAGS) --prefix=$(TARGET) && \
		make depend && make && make install_sw

$(HOST)/libssl.a: $(OSSL_DIR)/libssl.a
	cp $< $@

$(HOST)/libcrypto.a : $(OSSL_DIR)/libcrypto.a
	cp $< $@

openssl: $(HOST)/libssl.a $(HOST)/libcrypto.a

#
#
# ==== cURL ====
#
#

# TODO: Perform checksum
$(CURL_VER).tar.gz: | $(HOST)
	curl -L "$(CURL_URL)" > $(CURL_VER).tar.gz

$(CURL_DIR): $(CURL_VER).tar.gz
	tar -xzv -C $(HOST) -f $<

$(CURL_DIR)/lib/.libs/libcurl.a: $(CURL_DIR)
	cd $(CURL_DIR) && \
		./configure CFLAGS="$(CFLAGS)" --enable-static --enable-shared=no  \
			--disable-ldap      \
			--disable-ldaps     \
			--disable-ftp       \
			--disable-file      \
			--disable-rtsp      \
			--disable-dict      \
			--disable-telnet    \
			--disable-tftp      \
			--disable-pop3      \
			--disable-imap      \
			--disable-smtp      \
			--disable-gopher    \
			--disable-cookies   \
			--disable-smb       \
			--without-libidn    \
			--without-nghttp2   \
			--without-librtmp   \
			--without-libssh2   \
			--without-ca-bundle \
			--without-ca-path   \
			$(CURL_OPT) &&      \
		cd lib && make

$(HOST)/libcurl.a: $(CURL_DIR)/lib/.libs/libcurl.a
	cp $< $@

curl: $(HOST)/libcurl.a

#
#
# ===== zlib
#
#

# TODO: Perform checksum
$(ZLIB_VER).tar.gz: | $(HOST)
	curl -L "$(ZLIB_URL)" > $(ZLIB_VER).tar.gz
	sha256sum $(ZLIB_VER).tar.gz > $(ZLIB_VER).sha256
	grep $(ZLIB_SHA) $(ZLIB_VER).sha256

$(ZLIB_DIR): $(ZLIB_VER).tar.gz
	tar -xzv -C $(HOST) -f $<

$(ZLIB_DIR)/libz.a: |$(ZLIB_DIR)
	cd $(ZLIB_DIR) && \
		./configure --static && \
		make CFLAGS="$(CFLAGS) -O3 $(ZLIB_CFG)"

$(HOST)/libz.a: $(ZLIB_DIR)/libz.a
	cp $< $@

zlib: $(HOST)/libz.a

$(HOST):
	mkdir -p $(HOST)

#
#
# ===== dist
#
#

dist:
ifeq ($(HOST),$(DARWIN))
	# Build x86_64
	@vagrant up build-linux-x86_64
	@vagrant ssh build-linux-x86_64 -c 'cd /vagrant/deps && make'

	# Build x86
	@vagrant up build-linux-x86
	@vagrant ssh build-linux-x86 -c 'cd /vagrant/deps && make'
else
	@echo "Must be on OS X to build dist"
	@exit 1
endif

clean:
	rm -f $(HOST)/*.a
	rm -rf $(OSSL_DIR) $(CURL_DIR) $(ZLIB_DIR)

.PHONY: all clean prereqs openssl curl zlib dist

.SUFFIXES:

Makefile

# Fail if piped commands fail
SHELL=bash -o pipefail

# Absolute root of the project
ROOT := $(shell cd $(dir $(firstword $(MAKEFILE_LIST))) && pwd)

include $(ROOT)/mk/*.mk

RUST_VERSION := $(shell cat $(ROOT)/.rust-version)
RUSTUP = rustup run $(RUST_VERSION)
CARGO = $(RUSTUP) cargo

DEP_DIR ?= $(ROOT)/deps/$(HOST)

ifdef IS_DARWIN
	# Libraries that must be linked against:
	# * pthread
	# * curl
	# * System
	# * c
	# * m
	SO_OPTS = -lcurl -lssl -lcrypto -lz -lSystem -lpthread -lc -lm

	LD_OPTS = -Wl,-dead_strip -Wl,-macosx_version_min,10.7

	# This may make the linker flag superfluous, but I'm not certain.
	CARGO_EXTRA_ENV = MACOSX_DEPLOYMENT_TARGET=10.7
else
	ifndef IS_FREEBSD
		SO_LDL = -ldl
	endif

	SO_OPTS = -lrt $(SO_LDL) -lpthread -lgcc_s -lc -lm
	LD_OPTS = -Wl,--gc-sections -Wl,--version-script ./skylight.map
	DEPS = $(addprefix $(DEP_DIR)/, libcrypto.a libcurl.a libssl.a libz.a)
	# FIXME: Ideally the version wouldn't be hardcoded here, maybe symlink to 'openssl' in the deps Makefile
	CARGO_EXTRA_ENV = DEP_OPENSSL_ROOT=$(DEP_DIR)/openssl-1.0.1p OPENSSL_LIB_DIR=${DEP_DIR}/lib OPENSSL_INCLUDE_DIR=${DEP_DIR}/include
endif

ifdef IS_MUSL
	COMPILER = musl-gcc
else
	COMPILER = gcc
endif

ifdef DEBUG
	DIST ?= target/dist-debug
	CARGO_BUILD_DIR = debug
else
	DIST ?= target/dist
	CARGO_EXTRA_ARGS = --release
	CARGO_BUILD_DIR = release
endif

TARGET  ?= $(DIST)/$(HOST)
OBJ_DIR := $(TARGET)/objs
HDRGEN_TARGET ?= util/hdrgen/target/$(RS_HOST)/debug

# Rust source
RS_SRC = $(shell find src -name *.rs) Cargo.toml
HDRGEN = $(HDRGEN_TARGET)/hdrgen
HDRGEN_SRC = $(shell find util/hdrgen/src/ -name *.rs) util/hdrgen/Cargo.toml

# Grab the current version from the git sha
VERSION_SHA = $(shell cat .git/`cat .git/HEAD | awk '{print $$2}'` | cut -c1-7)
# Hard coded because this variable should only be evaled on darwin
VERSION_NUM = $(shell target/dist/x86_64-darwin/skylightd -v | head -1 | awk '{print $$2}' | sed 's/[-+].*//g')
VERSION = $(VERSION_NUM)-$(VERSION_SHA)

all: dist
	echo $(DIST)

toolchain: $(TARGET)/.toolchain-lock

dist: $(DIST)/skylight_$(HOST).tar.gz

clean:
	rm -rf $(TARGET)
	rm -rf target/$(RS_HOST)/release/libskylight-*.a

clean-all: clean
	rm -rf target/$(RS_HOST)
	rm -rf $(HDRGEN_TARGET)
	$(MAKE) -C deps clean


#
#
# ===== hdrgen =====
#
#

$(HDRGEN): $(HDRGEN_SRC)
	cd util/hdrgen && $(CARGO) build --target $(RS_HOST)

#
#
# ===== skylight =====
#
#

.rust-version:

Cargo.lock:

$(DEPS): | $(OBJ_DIR)
	cd $(ROOT)/deps && $(MAKE)

$(TARGET)/.toolchain-lock: .rust-version
	rustup toolchain install $(RUST_VERSION)
ifeq ($(MUSL),1)
	rustup target add x86_64-unknown-linux-musl --toolchain $(RUST_VERSION)
endif
	mkdir -p $(TARGET)
	touch $(TARGET)/.toolchain-lock

$(TARGET)/libskylight.a: $(TARGET)/.toolchain-lock $(RS_SRC) | $(TARGET) Cargo.lock
ifdef CARGO_CURL
	grep -A 1 -F target.$(RS_HOST).curl .cargo/config |tail -n 1 |grep rustc-link-search
endif
	SKYLIGHT_BUILD="$(VERSION_SHA)" $(CARGO_EXTRA_ENV) $(CARGO) build --verbose $(CARGO_EXTRA_ARGS) --target $(RS_HOST)
	cp target/$(RS_HOST)/$(CARGO_BUILD_DIR)/libskylight.a $(TARGET)/libskylight.a

$(TARGET)/objs/skylight.timestamp: $(TARGET)/libskylight.a $(DEPS) | $(OBJ_DIR)
	cd $(OBJ_DIR) && echo "../libskylight.a $(DEPS)" | xargs -n 1 ar x
	touch $@

$(TARGET)/libskylight$(SO_EXT): $(TARGET)/objs/skylight.timestamp
	$(COMPILER) --shared -Wall -o $@ $(TARGET)/objs/*.o $(SO_OPTS) $(LD_OPTS) 2>&1 | tee so-build-output
ifdef IS_DARWIN
	@if $$(grep -q "built for newer OSX version" so-build-output); then rm $@ && echo "Built for incorrect OS X version"; exit 1; fi
endif
	rm so-build-output

$(TARGET)/skylightd: src/main.c $(TARGET)/libskylight$(SO_EXT)
	$(COMPILER) -O3 -o $@ -L $(TARGET) src/main.c -lskylight

HDRS = skylight.h skylight_dlopen.h

$(addprefix $(TARGET)/,$(HDRS)): src/c_api/mod.rs $(HDRGEN_SRC) | $(HDRGEN)
	$(HDRGEN) -o $(TARGET) src/c_api/mod.rs

DIST_FILES = skylightd libskylight$(SO_EXT) $(HDRS) skylight_dlopen.c

$(DIST)/skylight_$(HOST).tar.gz: $(addprefix $(TARGET)/,$(DIST_FILES))
	cd $(TARGET) && tar -cvzf skylight_$(HOST).tar.gz $(DIST_FILES)
	mv $(TARGET)/skylight_$(HOST).tar.gz $(DIST)

$(OBJ_DIR):
	mkdir -p $@
	rm -rf $@/*.o

$(TARGET):
	mkdir -p $@

#
#
# ===== Deploy the build to S3 =====
#
#

ensure_clean_git:
ifndef SKIP_GIT_CHECK
	@git diff --exit-code > /dev/null || (echo "git directory dirty" && exit 1)
endif

dist-all:
ifeq ($(HOST),$(DARWIN))
	# Build on darwin
	$(MAKE) dist

	# Build x86_64
	@vagrant up build-linux-x86_64
	@vagrant ssh build-linux-x86_64 -c 'cd /vagrant && make dist && MUSL=1 make dist'

	# Build i386
	@vagrant up build-linux-x86
	@vagrant ssh build-linux-x86 -c 'cd /vagrant && make dist'

	# Build FreeBSD
	@vagrant up build-freebsd-x86_64
	@vagrant rsync # This happens on `up`, but it may already be up
	# Not clear why we have to add cargo to the PATH. Apparently the .profile isn't being loaded this way?
	@vagrant ssh build-freebsd-x86_64 -c 'cd /vagrant && PATH=$$PATH:~/.cargo/bin SQL_LEXER_MAKE=gmake gmake dist'
	@vagrant rsync-back # Get the built stuff back
else
	@echo "Must run on OS X"
	@exit 1
endif

deploy: dist-all | ensure_clean_git
	ruby util/s3/s3-push --version $(VERSION) target/dist

test:
ifeq ($(CI),true)
	# CI isn't set up for a full dist, but do as much as we can
	$(MAKE) toolchain
	$(CARGO) build --target $(RS_HOST) --release $(CARGO_TEST_FLAGS)
else
	@$(MAKE) dist
endif
ifneq ($(MUSL),1)
	$(CARGO) test --verbose --target $(RS_HOST)
	cd sql_lexer && $(CARGO) test --verbose --target $(RS_HOST) $(CARGO_TEST_FLAGS)
else
	# Won't work because we don't install the global musl interpreter
	@echo "Can't run musl tests, skipping."
endif

test-all:
ifeq ($(HOST),$(DARWIN))
	# Test on darwin
	$(MAKE) test

	# Test x86_64
	@vagrant up build-linux-x86_64
	@vagrant ssh build-linux-x86_64 -c 'cd /vagrant && make test && MUSL=1 CARGO_TEST_FLAGS=$(CARGO_TEST_FLAGS) make test '

	# Test i386
	@vagrant up build-linux-x86
	@vagrant ssh build-linux-x86 -c 'cd /vagrant && make test CARGO_TEST_FLAGS=$(CARGO_TEST_FLAGS) $(CARGO_TEST_FLAGS)'

	# Test FreeBSD
	@vagrant up build-freebsd-x86_64
	@vagrant rsync # This happens on `up`, but it may already be up
	# Not clear why we have to add cargo to the PATH. Apparently the .profile isn't being loaded this way?
	@vagrant ssh build-freebsd-x86_64 -c 'cd /vagrant && PATH=$$PATH:~/.cargo/bin SQL_LEXER_MAKE=gmake CARGO_TEST_FLAGS=$(CARGO_TEST_FLAGS) gmake test'
else
	@echo "Must run on OS X"
	@exit 1
endif

try-musl:
	docker build --rm -t tilde/skylight-alpine alpine-docker
	docker run --rm -v $(ROOT):/opt/skylight tilde/skylight-alpine /test

.PHONY: all clean clean-all dist dist-all deploy test test-all try-musl toolchain

.SUFFIXES: