Created
February 3, 2017 16:13
-
-
Save duggan/f1f3561b980f5bb06311c550efa9d634 to your computer and use it in GitHub Desktop.
A shell script for configuring Mesos/Storm via environment variables in Docker
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env bash | |
# | |
# Write a Storm configuration from environment variables. | |
# | |
# Environment variables namespaced with `MESOS_STORM_` | |
set -o errexit | |
set -o nounset | |
# Location where config should be written | |
: ${MESOS_STORM_CONFIG_LOCATION:=/tmp/storm.yaml} | |
# Mesos master | |
: ${MESOS_STORM_MESOS_MASTER_URL:=zk://localhost:2181/mesos} | |
# Mesos Zookeeper | |
: ${MESOS_STORM_STORM_ZOOKEEPER_SERVERS:=localhost} | |
# Trick for creating a list | |
__MESOS_STORM_ZOOKEEPER_SERVERS="" | |
for zk in ${MESOS_STORM_STORM_ZOOKEEPER_SERVERS} | |
do | |
__MESOS_STORM_ZOOKEEPER_SERVERS="${__MESOS_STORM_ZOOKEEPER_SERVERS} - \"${zk}\""$'\n' | |
done | |
# Other mesos settings are mostly key=value | |
: ${MESOS_STORM_STORM_ZOOKEEPER_ROOT:=/storm} | |
# These are specified by the scheduler | |
MESOS_STORM_UI_PORT=${PORT0:-8080} | |
MESOS_STORM_NIMBUS_THRIFT_PORT=${PORT1:-6627} | |
MESOS_STORM_LOGVIEWER_PORT=${PORT2:-8000} | |
MESOS_STORM_MESOS_LOCAL_FILE_SERVER_PORT=${PORT3:-""} | |
: ${MESOS_STORM_STORM_LOCAL_DIR:=storm-local} | |
: ${MESOS_STORM_STORM_LOGS_DIR:=/var/log/storm} | |
: ${MESOS_STORM_MESOS_SUPERVISOR_SUICIDE_INACTIVE_TIMEOUT_SECS:=120} | |
: ${MESOS_STORM_MESOS_MASTER_FAILOVER_TIMEOUT_SECS:=2473600} | |
: ${MESOS_STORM_MESOS_ALLOWED_HOSTS:=} | |
: ${MESOS_STORM_MESOS_DISALLOWED_HOSTS:=} | |
: ${MESOS_STORM_MESOS_OFFER_LRU_CACHE_SIZE:=1000} | |
: ${MESOS_STORM_MESOS_OFFER_FILTER_SECONDS:=120} | |
: ${MESOS_STORM_MESOS_OFFER_EXPIRY_MULTIPLIER:=2.5} | |
# Worker resources | |
: ${MESOS_STORM_TOPOLOGY_MESOS_WORKER_CPU:=1.0} | |
: ${MESOS_STORM_TOPOLOGY_MESOS_WORKER_MEM_MB:=1200} | |
: ${MESOS_STORM_WORKER_CHILDOPTS:=-Xmx1000m} | |
# Supervisor resources | |
: ${MESOS_STORM_TOPOLOGY_MESOS_EXECUTOR_CPU:=0.1} | |
: ${MESOS_STORM_TOPOLOGY_MESOS_EXECUTOR_MEM_MB:=500} | |
: ${MESOS_STORM_SUPERVISOR_CHILDOPTS:=-Xmx256m} | |
# Logviewer resources | |
: ${MESOS_STORM_LOGVIEWER_CHILDOPTS:=-Xmx128m} | |
: ${MESOS_STORM_LOGVIEWER_CLEANUP_AGE_MINS:=10080} | |
: ${MESOS_STORM_LOGVIEWER_APPENDER_NAME:=A1} | |
: ${MESOS_STORM_SUPERVISOR_AUTOSTART_LOGVIEWER:=true} | |
# Storm build | |
: ${MESOS_STORM_MESOS_EXECUTOR_URI:=} | |
# Storm container | |
: ${MESOS_STORM_MESOS_CONTAINER_DOCKER_IMAGE:=mesosphere/storm} | |
: ${MESOS_STORM_STORM_MESSAGING_TRANSPORT:=backtype.storm.messaging.netty.Context} | |
: ${MESOS_STORM_MESOS_FRAMEWORK_ROLE:=*} | |
: ${MESOS_STORM_MESOS_FRAMEWORK_CHECKPOINT:=true} | |
: ${MESOS_STORM_MESOS_FRAMEWORK_SECRET_FILE:=} | |
: ${MESOS_STORM_MESOS_FRAMEWORK_PRINCIPAL:=} | |
: ${MESOS_STORM_MESOS_PREFER_RESERVED_RESOURCES:=true} | |
: ${MESOS_STORM_MESOS_FRAMEWORK_NAME:=Storm} | |
# Write configuration | |
printf "Writing configuration file to ${MESOS_STORM_CONFIG_LOCATION}\n" | |
cat > ${MESOS_STORM_CONFIG_LOCATION} <<- EndOfConfig | |
# Mandatory configuration | |
# ----------------------------------------------------------- | |
mesos.master.url: "${MESOS_STORM_MESOS_MASTER_URL}" | |
storm.zookeeper.servers: | |
${__MESOS_STORM_ZOOKEEPER_SERVERS} | |
# ----------------------------------------------------------- | |
storm.zookeeper.root: "${MESOS_STORM_STORM_ZOOKEEPER_ROOT}" | |
nimbus.thrift.port: ${MESOS_STORM_NIMBUS_THRIFT_PORT} | |
ui.port: ${MESOS_STORM_UI_PORT} | |
storm.local.dir: "${MESOS_STORM_STORM_LOCAL_DIR}" | |
storm.logs.dir: "${MESOS_STORM_STORM_LOGS_DIR}" | |
# Seconds to wait before supervisor to suicides if supervisor has no task to run. | |
mesos.supervisor.suicide.inactive.timeout.secs: ${MESOS_STORM_MESOS_SUPERVISOR_SUICIDE_INACTIVE_TIMEOUT_SECS} | |
# Framework failover timeout in second. | |
mesos.master.failover.timeout.secs: ${MESOS_STORM_MESOS_MASTER_FAILOVER_TIMEOUT_SECS} | |
# Allowed hosts to run topology, which takes hostname list as a whitelist. | |
mesos.allowed.hosts: ${MESOS_STORM_MESOS_ALLOWED_HOSTS} | |
# Disallowed hosts to run topology, which takes hostname list as a blacklist | |
mesos.disallowed.hosts: ${MESOS_STORM_MESOS_DISALLOWED_HOSTS} | |
# LRU cache size | |
mesos.offer.lru.cache.size: ${MESOS_STORM_MESOS_OFFER_LRU_CACHE_SIZE} | |
# Number of seconds to filter unused Mesos offers. | |
# These offers may be revived by the framework when needed | |
mesos.offer.filter.seconds: ${MESOS_STORM_MESOS_OFFER_FILTER_SECONDS} | |
# Offer expiry multiplier for nimbus.monitor.freq.secs | |
mesos.offer.expiry.multiplier: ${MESOS_STORM_MESOS_OFFER_EXPIRY_MULTIPLIER} | |
# Port for the local file server to bind to | |
mesos.local.file.server.port: ${MESOS_STORM_MESOS_LOCAL_FILE_SERVER_PORT} | |
# Framework role to use | |
mesos.framework.role: "${MESOS_STORM_MESOS_FRAMEWORK_ROLE}" | |
# Enabled framework checkpoint or not. | |
mesos.framework.checkpoint: ${MESOS_STORM_MESOS_FRAMEWORK_CHECKPOINT} | |
# Framework name. | |
mesos.framework.name: "${MESOS_STORM_MESOS_FRAMEWORK_NAME}" | |
# Framework principal to use to register with Mesos | |
mesos.framework.principal: ${MESOS_STORM_MESOS_FRAMEWORK_PRINCIPAL} | |
# Location of file that contains the principal's secret. Secret cannot end with a NL | |
mesos.framework.secret.file: ${MESOS_STORM_MESOS_FRAMEWORK_SECRET_FILE} | |
# Prefer reserved resources over unreserved (i.e., "*" role) | |
mesos.prefer.reserved.resources: ${MESOS_STORM_MESOS_PREFER_RESERVED_RESOURCES} | |
# Worker resources | |
topology.mesos.worker.cpu: ${MESOS_STORM_TOPOLOGY_MESOS_WORKER_CPU} | |
# Example: worker heap with 20% overhead | |
topology.mesos.worker.mem.mb: ${MESOS_STORM_TOPOLOGY_MESOS_WORKER_MEM_MB} | |
worker.childopts: "${MESOS_STORM_WORKER_CHILDOPTS}" | |
# Supervisor resources | |
topology.mesos.executor.cpu: ${MESOS_STORM_TOPOLOGY_MESOS_EXECUTOR_CPU} | |
# Example: supervisor memory, with 20% overhead | |
topology.mesos.executor.mem.mb: ${MESOS_STORM_TOPOLOGY_MESOS_EXECUTOR_MEM_MB} | |
supervisor.childopts: "${MESOS_STORM_SUPERVISOR_CHILDOPTS}" | |
# The default behavior is to launch the logviewer unless autostart is false. | |
# If you enable the logviewer, you'll need to add memory overhead to the | |
# executor for the logviewer. | |
logviewer.port: ${MESOS_STORM_LOGVIEWER_PORT} | |
logviewer.childopts: "${MESOS_STORM_LOGVIEWER_CHILDOPTS}" | |
logviewer.cleanup.age.mins: ${MESOS_STORM_LOGVIEWER_CLEANUP_AGE_MINS} | |
logviewer.appender.name: "${MESOS_STORM_LOGVIEWER_APPENDER_NAME}" | |
supervisor.autostart.logviewer: ${MESOS_STORM_SUPERVISOR_AUTOSTART_LOGVIEWER} | |
# Use the public Mesosphere Storm build | |
# Please note that it won't work with other distributions. | |
# You may want to make this empty if you use 'mesos.container.docker.image' instead. | |
mesos.executor.uri: "${MESOS_STORM_MESOS_EXECUTOR_URI}" | |
# Alternatively, use a Docker image instead of URI. If an image is specified, | |
# Docker will be used instead of Mesos containers. | |
mesos.container.docker.image: "${MESOS_STORM_MESOS_CONTAINER_DOCKER_IMAGE}" | |
# Use Netty to avoid ZMQ dependencies | |
storm.messaging.transport: "${MESOS_STORM_STORM_MESSAGING_TRANSPORT}" | |
EndOfConfig |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Lines 20-24 are a useful bit of voodoo for writing a list. Can't recall where I originally picked it up now.