Sample Go stack for local dev.

README

This is a bunch of files I have locally in a `local-config` folder that I use to run & test things locally separate from our test suite.

application_message_dispatcher.yaml

transport_names:
  - vumigo_router
exposed_names:
  - bulk_message_transport
  - survey_transport
  - jsbox_transport
  - multi_survey_transport
  - opt_out_transport
  - sequential_send_transport
  - subscription_transport
  - wikipedia_ussd_transport
  - wikipedia_sms_transport

upstream_transport: vumigo_router
optout_transport: opt_out_transport

router_class: go.vumitools.api_worker.GoApplicationRouter
conversation_mappings:
  survey: survey_transport
  jsbox: jsbox_transport
  multi_survey: multi_survey_transport
  bulk_message: bulk_message_transport
  opt_out: opt_out_transport
  sequential_send: sequential_send_transport
  subscription: subscription_transport
  wikipedia_ussd: wikipedia_ussd_transport
  wikipedia_sms: wikipedia_sms_transport

redis_manager:
  key_prefix: "vumigo"
  db: 1
riak_manager:
  bucket_prefix: vumigo.

# NOTE: the order is important
middleware:
  - optout_middleware: go.vumitools.middleware.OptOutMiddleware
  - storing_middleware: vumi.middleware.message_storing.StoringMiddleware
  - logging_middleware: vumi.middleware.logging.LoggingMiddleware
  - metrics_middleware: go.vumitools.middleware.MetricsMiddleware

storing_middleware:
  redis_manager:
    key_prefix: "vumigo"
    db: 1
  riak_manager:
    bucket_prefix: vumigo.

optout_middleware:
  redis_manager:
    key_prefix: "vumigo"
    db: 1
  riak_manager:
    bucket_prefix: vumigo.
  case_sensitive: False
  optout_keywords:
    - stop

logging_middleware:
  log_level: debug

metrics_middleware:
  manager_name: application_message_dispatcher
  redis_manager:
    key_prefix: "vumigo:metrics_mw"
    db: 1

bulk_message_application.yaml

worker_name: bulk_message_application
transport_name: bulk_message_transport
metrics_prefix: bulk_message_application
redis_manager:
  key_prefix: "vumigo"
  db: 1
riak_manager:
  bucket_prefix: vumigo.
send_to:
  default:
    transport_name: "invalid broken transport"

command_dispatcher.yaml

transport_name: command_dispatcher
worker_names:
    - bulk_message_application
    - jsbox_application
    - survey_application
    - wikipedia_ussd_application
    - sequential_send_application

devnull_transport.yaml

transport_name: devnull_transport
failure_rate: 0.2
reply_rate: 0.8
ack_rate: 0.9

redis_manager:
  key_prefix: "vumigo:devnull"
  db: 1

middleware:
  - storing_mw: vumi.middleware.message_storing.StoringMiddleware
  - tagging_mw: vumi.middleware.tagger.TaggingMiddleware

tagging_mw:
  incoming:
    addr_pattern: '^(.+)$'
    tagpool_template: 'devnull'
    tagname_template: '\1'
  outgoing:
    tagname_pattern: '.*'
    msg_template: {}

storing_mw:
  riak_manager:
    bucket_prefix: vumigo.
  redis_manager:
    key_prefix: "vumigo"
    db: 1

normalize_msisdn_mw:
  country_code: '27'

event_dispatcher.yaml

transport_name: event_dispatcher

riak_manager: &RIAK
  bucket_prefix: vumigo.
redis_manager: &REDIS
  key_prefix: vumigo
  db: 1

event_handlers:
  name-of-handler: path.to.HandlerClass

account_handler_configs:
  account-key:
    - - [conversation-key, event]
      - - [name-of-handler, {}]

name-of-handler:
  event-handler: details

gtalk-transports.yaml

defaults:
  host: talk.google.com
  port: 5222
  status: "chat"
  status_message: "Vumi Go!"
  debug: 1
  middleware:
    - logging_mw: vumi.middleware.logging.LoggingMiddleware
    - storing_mw: vumi.middleware.message_storing.StoringMiddleware
    - gtalk_tagging_mw: vumi.middleware.tagger.TaggingMiddleware

  # Configure the middlewares
  logging_mw:
    log_level: debug

  gtalk_tagging_mw:
    incoming:
      addr_pattern: '^(.+\@.+)/?.*$'
      tagpool_template: 'xmpp'
      tagname_template: '\1'
    outgoing:
      tagname_pattern: '.*'
      msg_template: {}

  storing_mw:
    riak_manager:
      bucket_prefix: vumigo.
    redis_manager:
      key_prefix: "vumigo:message_store"
      db: 1

gtalk_vumigo_1:
  transport_name: gtalk_vumigo_1
  username: your@xmpp.com
  password: 'password'

vumigo_gtalk_dispatch:
  transport_names:
    - gtalk_vumigo_1
  exposed_names:
    - gtalk_vumigo
  router_class: vumi.dispatchers.base.FromAddrMultiplexRouter
  fromaddr_mappings:
    "your@xmpp.com": gtalk_vumigo_1

workers:
  gtalk_vumigo_1: vumi.transports.xmpp.XMPPTransport
  vumigo_gtalk_dispatch: vumi.dispatchers.base.BaseDispatchWorker

message_store_api.yaml

web_path: "/message_store_api/v1/"
web_port: 8077
health_path: "/health/"

riak_manager:
  bucket_prefix: vumigo.
redis_manager:
  key_prefix: "vumigo:message_store"
  db: 1

multipoll.yaml

worker_name: multi_survey_application
transport_name: multi_survey_transport
metrics_prefix: multi_survey_application
batch_size: 5
redis_manager:
  key_prefix: "vumigo"
  db: 1
riak_manager:
  bucket_prefix: vumigo.
vxpolls:
  prefix: vxpolls

opt_out.yaml

worker_name: opt_out_application
transport_name: opt_out_transport
metrics_prefix: opt_out_application
batch_size: 5

redis_manager:
  key_prefix: "vumigo"
  db: 1
riak_manager:
  bucket_prefix: vumigo.

middleware:
  - logging_middleware: vumi.middleware.logging.LoggingMiddleware

logging_middleware:
  log_level: debug

poll.yaml

worker_name: survey_application
transport_name: survey_transport
metrics_prefix: survey_application
batch_size: 2
redis_manager:
  key_prefix: "vumigo"
  db: 1
riak_manager:
  bucket_prefix: vumigo.
vxpolls:
  prefix: vxpolls

supervisord.conf

[inet_http_server]         ; inet (TCP) server disabled by default
port=127.0.0.1:7102        ; (ip_address:port specifier, *:port for all iface)

[supervisord]
logfile=./logs/supervisord.log ; (main log file;default $CWD/supervisord.log)
logfile_maxbytes=50MB       ; (max main logfile bytes b4 rotation;default 50MB)
logfile_backups=10          ; (num of main logfile rotation backups;default 10)
loglevel=info               ; (log level;default info; others: debug,warn,trace)
pidfile=./tmp/pids/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
nodaemon=false              ; (start in foreground if true;default false)
minfds=1024                 ; (min. avail startup file descriptors;default 1024)
minprocs=200                ; (min. avail process descriptors;default 200)

[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface

[supervisorctl]
serverurl=http://127.0.0.1:7102 ; use an http:// url to specify an inet socket

[program:survey_application]
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=go.apps.surveys.vumi_app.SurveyApplication
    --config=./config/poll.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:multi_survey_application]
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=go.apps.multi_surveys.vumi_app.MultiSurveyApplication
    --config=./config/multipoll.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:opt_out_application]
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=go.apps.opt_out.vumi_app.OptOutApplication
    --config=./config/opt_out.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:wikipedia_application]
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=go.apps.wikipedia.vumi_app.WikipediaApplication
    --config=./config/wikipedia.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:celery]
command=django-admin.py celeryd --loglevel=INFO
environment=DJANGO_SETTINGS_MODULE=go.settings,
            PYTHONPATH=.
stdout_logfile=./logs/celeryd.log
stderr_logfile=./logs/celeryd.err
redirect_stderr=true
autostart=false
autorestart=true
startsecs=10

[program:command_dispatcher]
numprocs=1
process_name=%(program_name)s_%(process_num)s
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=go.vumitools.api_worker.CommandDispatcher
    --config=./config/command_dispatcher.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:event_dispatcher]
numprocs=1
process_name=%(program_name)s_%(process_num)s
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=go.vumitools.api_worker.EventDispatcher
    --config=./config/event_dispatcher.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:application_message_dispatcher]
numprocs=1
process_name=%(program_name)s_%(process_num)s
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=vumi.dispatchers.base.BaseDispatchWorker
    --config=./config/application_message_dispatcher.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:bulk_message_application]
numprocs=1
process_name=%(program_name)s_%(process_num)s
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=go.apps.bulk_message.vumi_app.BulkMessageApplication
    --config=./config/bulk_message_application.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:vumigo_router]
numprocs=1
process_name=%(program_name)s_%(process_num)s
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=vumi.dispatchers.base.BaseDispatchWorker
    --config=./config/vumigo_router.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:vumigo_gtalk_transports]
numprocs=1
process_name=%(program_name)s_%(process_num)s
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=vumi.multiworker.MultiWorker
    --config=./config/gtalk-transports.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:devnull_transport]
numprocs=1
process_name=%(program_name)s_%(process_num)s
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=vumi.transports.devnull.DevNullTransport
    --config=./config/devnull_transport.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:twitter_transport]
numprocs=1
process_name=%(program_name)s_%(process_num)s
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=vumi.transports.twitter.TwitterTransport
    --config=./config/twitter_transport.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:blinkenlights_timebucket]
numprocs=1
process_name=%(program_name)s_%(process_num)s
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=vumi.blinkenlights.MetricTimeBucket
    --set-option=buckets:3
    --set-option=bucket_size:5
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:blinkenlights_aggregator]
numprocs=3
process_name=%(program_name)s_%(process_num)s
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s_%(process_num)s.pid
    vumi_worker
    --worker-class=vumi.blinkenlights.MetricAggregator
    --set-option=bucket_size:5
    --set-option=bucket:%(process_num)s
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:blinkenlights_collector]
numprocs=1
process_name=%(program_name)s_%(process_num)s
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=vumi.blinkenlights.GraphiteMetricsCollector
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

[program:message_store_api]
numprocs=1
process_name=%(program_name)s_%(process_num)s
command=twistd -n
    --pidfile=./tmp/pids/%(program_name)s.pid
    vumi_worker
    --worker-class=vumi.components.message_store_api.MessageStoreAPIWorker
    --config=./config/message_store_api.yaml
    --vhost=/develop
stdout_logfile=./logs/%(program_name)s_%(process_num)s.log
stdout_logfile_maxbytes=10MB
stdout_logfile_backups=10
stderr_logfile=./logs/%(program_name)s_%(process_num)s.err
stderr_logfile_maxbytes=10MB
stderr_logfile_backups=10
autostart=false
autorestart=true

tagpools.yaml

redis_manager:
  key_prefix: "vumigo"
  db: 1
tagpool_prefix: "tagpool_store"
pools:
  longcode:
    tags: '["default%d" % i for i in range(10001, 10001 + 1000)]'
    metadata:
      display_name: "Long code"
      delivery_class: sms
      transport_type: sms
      server_initiated: true
      msg_options:
        transport_name: smpp_transport
  xmpp:
    tags:
      - s.dehaan@gmail.com
    metadata:
      display_name: "Google Talk"
      delivery_class: gtalk
      transport_type: xmpp
      user_selects_tag: true
      server_initiated: true
      msg_options:
        transport_name: gtalk_vumigo
  devnull:
    tags: '["%d@devnull" % i for i in range(10001, 10001 + 1000)]'
    metadata:
      display_name: "/dev/null"
      delivery_class: sms
      transport_type: sms
      server_initiated: true
      msg_options:
        transport_name: devnull_transport
  twitterstream:
    tags:
      - twitter
    metadata:
      display_name: 'Twitter Stream Sampling'
      delivery_class: twitter
      transport_type: twitter
      server_initiated: true
      msg_options:
        transport_name: twitter_transport

twitter_transport.yaml

transport_name: twitter_transport
app_name: twitter_app

consumer_key: get-from-twitter
consumer_secret: get-from-twitter
access_token: get-from-twitter
access_token_secret: get-from-twitter
check_replies_interval: -1

terms:
    - terms
    - to
    - track

redis_manager:
  key_prefix: "vumigo:twitter"
  db: 1

middleware:
  - storing_mw: vumi.middleware.message_storing.StoringMiddleware
  - tagging_mw: vumi.middleware.tagger.TaggingMiddleware

tagging_mw:
  incoming:
    addr_pattern: '.*'
    tagpool_template: 'twitterstream'
    tagname_template: 'twitter'
  outgoing:
    tagname_pattern: '.*'
    msg_template: {}

storing_mw:
  riak_manager:
    bucket_prefix: vumigo.
  redis_manager:
    key_prefix: "vumigo"
    db: 1

vumigo_router.yaml

transport_names:
  - smpp_transport
  - gtalk_vumigo
  - truteq_transport
  - truteq_mama_transport
  - vumi_starcode_transport
  - safaricom_sna_smpp_transport
  - safaricom_sna_ussd_transport
  - afropinions_smpp_transport
  - switchboard_smpp_transport
  - switchboard_ussd_transport
  - pcm_transport
  - guinness_cameroun_transport
  - guinness_ghana_transport
  - guinness_kenya_transport
  - devnull_transport
  - pepsodent_ghana_transport
  - twitter_transport
exposed_names:
  - vumigo_router
middleware:
  - logging_mw: vumi.middleware.logging.LoggingMiddleware
router_class: vumi.dispatchers.base.SimpleDispatchRouter
route_mappings:
  smpp_transport:
    - vumigo_router
  gtalk_vumigo:
    - vumigo_router
  truteq_transport:
    - vumigo_router
  truteq_mama_transport:
    - vumigo_router
  vumi_starcode_transport:
    - vumigo_router
  safaricom_sna_smpp_transport:
    - vumigo_router
  safaricom_sna_ussd_transport:
    - vumigo_router
  afropinions_smpp_transport:
    - vumigo_router
  switchboard_smpp_transport:
    - vumigo_router
  switchboard_ussd_transport:
    - vumigo_router
  pcm_transport:
    - vumigo_router
  guinness_cameroun_transport:
    - vumigo_router
  guinness_ghana_transport:
    - vumigo_router
  guinness_kenya_transport:
    - vumigo_router
  devnull_transport:
    - vumigo_router
  pepsodent_ghana_transport:
    - vumigo_router
  twitter_transport:
    - vumigo_router

wikipedia.yaml

worker_name: wikipedia_ussd_application
transport_name: wikipedia_ussd_transport
metrics_prefix: wikipedia_ussd_application
sms_transport: vumigo_router
incoming_sms_transport: wikipedia_sms_transport
max_sms_content_length: 450
max_sms_unicode_length: 190
redis_manager:
  key_prefix: "vumigo"
  db: 1
riak_manager:
  bucket_prefix: vumigo.