Skip to content

Instantly share code, notes, and snippets.

@duggan
Created February 3, 2017 16:13
Show Gist options
  • Save duggan/f1f3561b980f5bb06311c550efa9d634 to your computer and use it in GitHub Desktop.
Save duggan/f1f3561b980f5bb06311c550efa9d634 to your computer and use it in GitHub Desktop.
A shell script for configuring Mesos/Storm via environment variables in Docker
#!/usr/bin/env bash
#
# Write a Storm configuration from environment variables.
#
# Environment variables namespaced with `MESOS_STORM_`
set -o errexit
set -o nounset
# Location where config should be written
: ${MESOS_STORM_CONFIG_LOCATION:=/tmp/storm.yaml}
# Mesos master
: ${MESOS_STORM_MESOS_MASTER_URL:=zk://localhost:2181/mesos}
# Mesos Zookeeper
: ${MESOS_STORM_STORM_ZOOKEEPER_SERVERS:=localhost}
# Trick for creating a list
__MESOS_STORM_ZOOKEEPER_SERVERS=""
for zk in ${MESOS_STORM_STORM_ZOOKEEPER_SERVERS}
do
__MESOS_STORM_ZOOKEEPER_SERVERS="${__MESOS_STORM_ZOOKEEPER_SERVERS} - \"${zk}\""$'\n'
done
# Other mesos settings are mostly key=value
: ${MESOS_STORM_STORM_ZOOKEEPER_ROOT:=/storm}
# These are specified by the scheduler
MESOS_STORM_UI_PORT=${PORT0:-8080}
MESOS_STORM_NIMBUS_THRIFT_PORT=${PORT1:-6627}
MESOS_STORM_LOGVIEWER_PORT=${PORT2:-8000}
MESOS_STORM_MESOS_LOCAL_FILE_SERVER_PORT=${PORT3:-""}
: ${MESOS_STORM_STORM_LOCAL_DIR:=storm-local}
: ${MESOS_STORM_STORM_LOGS_DIR:=/var/log/storm}
: ${MESOS_STORM_MESOS_SUPERVISOR_SUICIDE_INACTIVE_TIMEOUT_SECS:=120}
: ${MESOS_STORM_MESOS_MASTER_FAILOVER_TIMEOUT_SECS:=2473600}
: ${MESOS_STORM_MESOS_ALLOWED_HOSTS:=}
: ${MESOS_STORM_MESOS_DISALLOWED_HOSTS:=}
: ${MESOS_STORM_MESOS_OFFER_LRU_CACHE_SIZE:=1000}
: ${MESOS_STORM_MESOS_OFFER_FILTER_SECONDS:=120}
: ${MESOS_STORM_MESOS_OFFER_EXPIRY_MULTIPLIER:=2.5}
# Worker resources
: ${MESOS_STORM_TOPOLOGY_MESOS_WORKER_CPU:=1.0}
: ${MESOS_STORM_TOPOLOGY_MESOS_WORKER_MEM_MB:=1200}
: ${MESOS_STORM_WORKER_CHILDOPTS:=-Xmx1000m}
# Supervisor resources
: ${MESOS_STORM_TOPOLOGY_MESOS_EXECUTOR_CPU:=0.1}
: ${MESOS_STORM_TOPOLOGY_MESOS_EXECUTOR_MEM_MB:=500}
: ${MESOS_STORM_SUPERVISOR_CHILDOPTS:=-Xmx256m}
# Logviewer resources
: ${MESOS_STORM_LOGVIEWER_CHILDOPTS:=-Xmx128m}
: ${MESOS_STORM_LOGVIEWER_CLEANUP_AGE_MINS:=10080}
: ${MESOS_STORM_LOGVIEWER_APPENDER_NAME:=A1}
: ${MESOS_STORM_SUPERVISOR_AUTOSTART_LOGVIEWER:=true}
# Storm build
: ${MESOS_STORM_MESOS_EXECUTOR_URI:=}
# Storm container
: ${MESOS_STORM_MESOS_CONTAINER_DOCKER_IMAGE:=mesosphere/storm}
: ${MESOS_STORM_STORM_MESSAGING_TRANSPORT:=backtype.storm.messaging.netty.Context}
: ${MESOS_STORM_MESOS_FRAMEWORK_ROLE:=*}
: ${MESOS_STORM_MESOS_FRAMEWORK_CHECKPOINT:=true}
: ${MESOS_STORM_MESOS_FRAMEWORK_SECRET_FILE:=}
: ${MESOS_STORM_MESOS_FRAMEWORK_PRINCIPAL:=}
: ${MESOS_STORM_MESOS_PREFER_RESERVED_RESOURCES:=true}
: ${MESOS_STORM_MESOS_FRAMEWORK_NAME:=Storm}
# Write configuration
printf "Writing configuration file to ${MESOS_STORM_CONFIG_LOCATION}\n"
cat > ${MESOS_STORM_CONFIG_LOCATION} <<- EndOfConfig
# Mandatory configuration
# -----------------------------------------------------------
mesos.master.url: "${MESOS_STORM_MESOS_MASTER_URL}"
storm.zookeeper.servers:
${__MESOS_STORM_ZOOKEEPER_SERVERS}
# -----------------------------------------------------------
storm.zookeeper.root: "${MESOS_STORM_STORM_ZOOKEEPER_ROOT}"
nimbus.thrift.port: ${MESOS_STORM_NIMBUS_THRIFT_PORT}
ui.port: ${MESOS_STORM_UI_PORT}
storm.local.dir: "${MESOS_STORM_STORM_LOCAL_DIR}"
storm.logs.dir: "${MESOS_STORM_STORM_LOGS_DIR}"
# Seconds to wait before supervisor to suicides if supervisor has no task to run.
mesos.supervisor.suicide.inactive.timeout.secs: ${MESOS_STORM_MESOS_SUPERVISOR_SUICIDE_INACTIVE_TIMEOUT_SECS}
# Framework failover timeout in second.
mesos.master.failover.timeout.secs: ${MESOS_STORM_MESOS_MASTER_FAILOVER_TIMEOUT_SECS}
# Allowed hosts to run topology, which takes hostname list as a whitelist.
mesos.allowed.hosts: ${MESOS_STORM_MESOS_ALLOWED_HOSTS}
# Disallowed hosts to run topology, which takes hostname list as a blacklist
mesos.disallowed.hosts: ${MESOS_STORM_MESOS_DISALLOWED_HOSTS}
# LRU cache size
mesos.offer.lru.cache.size: ${MESOS_STORM_MESOS_OFFER_LRU_CACHE_SIZE}
# Number of seconds to filter unused Mesos offers.
# These offers may be revived by the framework when needed
mesos.offer.filter.seconds: ${MESOS_STORM_MESOS_OFFER_FILTER_SECONDS}
# Offer expiry multiplier for nimbus.monitor.freq.secs
mesos.offer.expiry.multiplier: ${MESOS_STORM_MESOS_OFFER_EXPIRY_MULTIPLIER}
# Port for the local file server to bind to
mesos.local.file.server.port: ${MESOS_STORM_MESOS_LOCAL_FILE_SERVER_PORT}
# Framework role to use
mesos.framework.role: "${MESOS_STORM_MESOS_FRAMEWORK_ROLE}"
# Enabled framework checkpoint or not.
mesos.framework.checkpoint: ${MESOS_STORM_MESOS_FRAMEWORK_CHECKPOINT}
# Framework name.
mesos.framework.name: "${MESOS_STORM_MESOS_FRAMEWORK_NAME}"
# Framework principal to use to register with Mesos
mesos.framework.principal: ${MESOS_STORM_MESOS_FRAMEWORK_PRINCIPAL}
# Location of file that contains the principal's secret. Secret cannot end with a NL
mesos.framework.secret.file: ${MESOS_STORM_MESOS_FRAMEWORK_SECRET_FILE}
# Prefer reserved resources over unreserved (i.e., "*" role)
mesos.prefer.reserved.resources: ${MESOS_STORM_MESOS_PREFER_RESERVED_RESOURCES}
# Worker resources
topology.mesos.worker.cpu: ${MESOS_STORM_TOPOLOGY_MESOS_WORKER_CPU}
# Example: worker heap with 20% overhead
topology.mesos.worker.mem.mb: ${MESOS_STORM_TOPOLOGY_MESOS_WORKER_MEM_MB}
worker.childopts: "${MESOS_STORM_WORKER_CHILDOPTS}"
# Supervisor resources
topology.mesos.executor.cpu: ${MESOS_STORM_TOPOLOGY_MESOS_EXECUTOR_CPU}
# Example: supervisor memory, with 20% overhead
topology.mesos.executor.mem.mb: ${MESOS_STORM_TOPOLOGY_MESOS_EXECUTOR_MEM_MB}
supervisor.childopts: "${MESOS_STORM_SUPERVISOR_CHILDOPTS}"
# The default behavior is to launch the logviewer unless autostart is false.
# If you enable the logviewer, you'll need to add memory overhead to the
# executor for the logviewer.
logviewer.port: ${MESOS_STORM_LOGVIEWER_PORT}
logviewer.childopts: "${MESOS_STORM_LOGVIEWER_CHILDOPTS}"
logviewer.cleanup.age.mins: ${MESOS_STORM_LOGVIEWER_CLEANUP_AGE_MINS}
logviewer.appender.name: "${MESOS_STORM_LOGVIEWER_APPENDER_NAME}"
supervisor.autostart.logviewer: ${MESOS_STORM_SUPERVISOR_AUTOSTART_LOGVIEWER}
# Use the public Mesosphere Storm build
# Please note that it won't work with other distributions.
# You may want to make this empty if you use 'mesos.container.docker.image' instead.
mesos.executor.uri: "${MESOS_STORM_MESOS_EXECUTOR_URI}"
# Alternatively, use a Docker image instead of URI. If an image is specified,
# Docker will be used instead of Mesos containers.
mesos.container.docker.image: "${MESOS_STORM_MESOS_CONTAINER_DOCKER_IMAGE}"
# Use Netty to avoid ZMQ dependencies
storm.messaging.transport: "${MESOS_STORM_STORM_MESSAGING_TRANSPORT}"
EndOfConfig
@duggan
Copy link
Author

duggan commented Feb 3, 2017

Lines 20-24 are a useful bit of voodoo for writing a list. Can't recall where I originally picked it up now.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment