A shell script for configuring Mesos/Storm via environment variables in Docker

configure.sh

#!/usr/bin/env bash

#
# Write a Storm configuration from environment variables.
#
# Environment variables namespaced with `MESOS_STORM_`

set -o errexit
set -o nounset

# Location where config should be written
: ${MESOS_STORM_CONFIG_LOCATION:=/tmp/storm.yaml}

# Mesos master
: ${MESOS_STORM_MESOS_MASTER_URL:=zk://localhost:2181/mesos}

# Mesos Zookeeper
: ${MESOS_STORM_STORM_ZOOKEEPER_SERVERS:=localhost}
# Trick for creating a list
__MESOS_STORM_ZOOKEEPER_SERVERS=""
for zk in ${MESOS_STORM_STORM_ZOOKEEPER_SERVERS}
do
    __MESOS_STORM_ZOOKEEPER_SERVERS="${__MESOS_STORM_ZOOKEEPER_SERVERS} - \"${zk}\""$'\n'
done

# Other mesos settings are mostly key=value
: ${MESOS_STORM_STORM_ZOOKEEPER_ROOT:=/storm}

# These are specified by the scheduler
MESOS_STORM_UI_PORT=${PORT0:-8080}
MESOS_STORM_NIMBUS_THRIFT_PORT=${PORT1:-6627}
MESOS_STORM_LOGVIEWER_PORT=${PORT2:-8000}
MESOS_STORM_MESOS_LOCAL_FILE_SERVER_PORT=${PORT3:-""}

: ${MESOS_STORM_STORM_LOCAL_DIR:=storm-local}
: ${MESOS_STORM_STORM_LOGS_DIR:=/var/log/storm}

: ${MESOS_STORM_MESOS_SUPERVISOR_SUICIDE_INACTIVE_TIMEOUT_SECS:=120}
: ${MESOS_STORM_MESOS_MASTER_FAILOVER_TIMEOUT_SECS:=2473600}

: ${MESOS_STORM_MESOS_ALLOWED_HOSTS:=}
: ${MESOS_STORM_MESOS_DISALLOWED_HOSTS:=}

: ${MESOS_STORM_MESOS_OFFER_LRU_CACHE_SIZE:=1000}
: ${MESOS_STORM_MESOS_OFFER_FILTER_SECONDS:=120}
: ${MESOS_STORM_MESOS_OFFER_EXPIRY_MULTIPLIER:=2.5}

# Worker resources
: ${MESOS_STORM_TOPOLOGY_MESOS_WORKER_CPU:=1.0}
: ${MESOS_STORM_TOPOLOGY_MESOS_WORKER_MEM_MB:=1200}
: ${MESOS_STORM_WORKER_CHILDOPTS:=-Xmx1000m}

# Supervisor resources
: ${MESOS_STORM_TOPOLOGY_MESOS_EXECUTOR_CPU:=0.1}
: ${MESOS_STORM_TOPOLOGY_MESOS_EXECUTOR_MEM_MB:=500}
: ${MESOS_STORM_SUPERVISOR_CHILDOPTS:=-Xmx256m}

# Logviewer resources
: ${MESOS_STORM_LOGVIEWER_CHILDOPTS:=-Xmx128m}
: ${MESOS_STORM_LOGVIEWER_CLEANUP_AGE_MINS:=10080}
: ${MESOS_STORM_LOGVIEWER_APPENDER_NAME:=A1}
: ${MESOS_STORM_SUPERVISOR_AUTOSTART_LOGVIEWER:=true}

# Storm build
: ${MESOS_STORM_MESOS_EXECUTOR_URI:=}

# Storm container
: ${MESOS_STORM_MESOS_CONTAINER_DOCKER_IMAGE:=mesosphere/storm}

: ${MESOS_STORM_STORM_MESSAGING_TRANSPORT:=backtype.storm.messaging.netty.Context}

: ${MESOS_STORM_MESOS_FRAMEWORK_ROLE:=*}
: ${MESOS_STORM_MESOS_FRAMEWORK_CHECKPOINT:=true}
: ${MESOS_STORM_MESOS_FRAMEWORK_SECRET_FILE:=}
: ${MESOS_STORM_MESOS_FRAMEWORK_PRINCIPAL:=}
: ${MESOS_STORM_MESOS_PREFER_RESERVED_RESOURCES:=true}
: ${MESOS_STORM_MESOS_FRAMEWORK_NAME:=Storm}


# Write configuration

printf "Writing configuration file to ${MESOS_STORM_CONFIG_LOCATION}\n"
cat > ${MESOS_STORM_CONFIG_LOCATION} <<- EndOfConfig
# Mandatory configuration
# -----------------------------------------------------------
mesos.master.url: "${MESOS_STORM_MESOS_MASTER_URL}"
storm.zookeeper.servers:
${__MESOS_STORM_ZOOKEEPER_SERVERS}
# -----------------------------------------------------------
storm.zookeeper.root: "${MESOS_STORM_STORM_ZOOKEEPER_ROOT}"
nimbus.thrift.port: ${MESOS_STORM_NIMBUS_THRIFT_PORT}
ui.port: ${MESOS_STORM_UI_PORT}

storm.local.dir: "${MESOS_STORM_STORM_LOCAL_DIR}"
storm.logs.dir: "${MESOS_STORM_STORM_LOGS_DIR}"

# Seconds to wait before supervisor to suicides if supervisor has no task to run.
mesos.supervisor.suicide.inactive.timeout.secs: ${MESOS_STORM_MESOS_SUPERVISOR_SUICIDE_INACTIVE_TIMEOUT_SECS}

# Framework failover timeout in second.
mesos.master.failover.timeout.secs: ${MESOS_STORM_MESOS_MASTER_FAILOVER_TIMEOUT_SECS}

# Allowed hosts to run topology, which takes hostname list as a whitelist.
mesos.allowed.hosts: ${MESOS_STORM_MESOS_ALLOWED_HOSTS}

# Disallowed hosts to run topology, which takes hostname list as a blacklist
mesos.disallowed.hosts: ${MESOS_STORM_MESOS_DISALLOWED_HOSTS}

# LRU cache size
mesos.offer.lru.cache.size: ${MESOS_STORM_MESOS_OFFER_LRU_CACHE_SIZE}

# Number of seconds to filter unused Mesos offers.
# These offers may be revived by the framework when needed
mesos.offer.filter.seconds: ${MESOS_STORM_MESOS_OFFER_FILTER_SECONDS}

# Offer expiry multiplier for nimbus.monitor.freq.secs
mesos.offer.expiry.multiplier: ${MESOS_STORM_MESOS_OFFER_EXPIRY_MULTIPLIER}

# Port for the local file server to bind to
mesos.local.file.server.port: ${MESOS_STORM_MESOS_LOCAL_FILE_SERVER_PORT}

# Framework role to use
mesos.framework.role: "${MESOS_STORM_MESOS_FRAMEWORK_ROLE}"

# Enabled framework checkpoint or not.
mesos.framework.checkpoint: ${MESOS_STORM_MESOS_FRAMEWORK_CHECKPOINT}

# Framework name.
mesos.framework.name: "${MESOS_STORM_MESOS_FRAMEWORK_NAME}"

# Framework principal to use to register with Mesos
mesos.framework.principal: ${MESOS_STORM_MESOS_FRAMEWORK_PRINCIPAL}

# Location of file that contains the principal's secret. Secret cannot end with a NL
mesos.framework.secret.file: ${MESOS_STORM_MESOS_FRAMEWORK_SECRET_FILE}

# Prefer reserved resources over unreserved (i.e., "*" role)
mesos.prefer.reserved.resources: ${MESOS_STORM_MESOS_PREFER_RESERVED_RESOURCES}

# Worker resources
topology.mesos.worker.cpu: ${MESOS_STORM_TOPOLOGY_MESOS_WORKER_CPU}
# Example: worker heap with 20% overhead
topology.mesos.worker.mem.mb: ${MESOS_STORM_TOPOLOGY_MESOS_WORKER_MEM_MB}
worker.childopts: "${MESOS_STORM_WORKER_CHILDOPTS}"

# Supervisor resources
topology.mesos.executor.cpu: ${MESOS_STORM_TOPOLOGY_MESOS_EXECUTOR_CPU}
# Example: supervisor memory, with 20% overhead
topology.mesos.executor.mem.mb: ${MESOS_STORM_TOPOLOGY_MESOS_EXECUTOR_MEM_MB}
supervisor.childopts: "${MESOS_STORM_SUPERVISOR_CHILDOPTS}"

# The default behavior is to launch the logviewer unless autostart is false.
# If you enable the logviewer, you'll need to add memory overhead to the
# executor for the logviewer.
logviewer.port: ${MESOS_STORM_LOGVIEWER_PORT}
logviewer.childopts: "${MESOS_STORM_LOGVIEWER_CHILDOPTS}"
logviewer.cleanup.age.mins: ${MESOS_STORM_LOGVIEWER_CLEANUP_AGE_MINS}
logviewer.appender.name: "${MESOS_STORM_LOGVIEWER_APPENDER_NAME}"
supervisor.autostart.logviewer: ${MESOS_STORM_SUPERVISOR_AUTOSTART_LOGVIEWER}

# Use the public Mesosphere Storm build
# Please note that it won't work with other distributions.
# You may want to make this empty if you use 'mesos.container.docker.image' instead.
mesos.executor.uri: "${MESOS_STORM_MESOS_EXECUTOR_URI}"

# Alternatively, use a Docker image instead of URI. If an image is specified,
# Docker will be used instead of Mesos containers.
mesos.container.docker.image: "${MESOS_STORM_MESOS_CONTAINER_DOCKER_IMAGE}"

# Use Netty to avoid ZMQ dependencies
storm.messaging.transport: "${MESOS_STORM_STORM_MESSAGING_TRANSPORT}"

EndOfConfig

Lines 20-24 are a useful bit of voodoo for writing a list. Can't recall where I originally picked it up now.