Skip to content

Instantly share code, notes, and snippets.

@marram
Created December 2, 2014 15:25
Show Gist options
  • Save marram/f8885d2d35bc26625ae9 to your computer and use it in GitHub Desktop.
Save marram/f8885d2d35bc26625ae9 to your computer and use it in GitHub Desktop.
queue:
- name: default
rate: 500/s
bucket_size: 100
retry_parameters:
task_retry_limit: 3
min_backoff_seconds: 1
max_backoff_seconds: 5
max_doublings: 0
- name: default-shard1
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 1
max_backoff_seconds: 5
max_doublings: 0
- name: default-shard2
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 1
max_backoff_seconds: 5
max_doublings: 0
- name: default-shard3
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 1
max_backoff_seconds: 5
max_doublings: 0
- name: default-shard4
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 1
max_backoff_seconds: 5
max_doublings: 0
- name: default-shard5
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 1
max_backoff_seconds: 5
max_doublings: 0
- name: default-shard6
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 1
max_backoff_seconds: 5
max_doublings: 0
- name: default-shard7
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 1
max_backoff_seconds: 5
max_doublings: 0
- name: default-shard8
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 1
max_backoff_seconds: 5
max_doublings: 0
- name: default-shard9
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 1
max_backoff_seconds: 5
max_doublings: 0
- name: default-shard10
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 1
max_backoff_seconds: 5
max_doublings: 0
- name: unsubscribe
target: queue-workers
rate: 1/s
max_concurrent_requests: 20
bucket_size: 40
- name: chrono
target: chrono
rate: 3/s
max_concurrent_requests: 20
bucket_size: 40
- name: notifications
rate: 500/s
bucket_size: 100
retry_parameters:
task_retry_limit: 8
min_backoff_seconds: 30
max_backoff_seconds: 600
- name: notifications-shard1
rate: 500/s
bucket_size: 100
retry_parameters:
task_retry_limit: 8
min_backoff_seconds: 30
max_backoff_seconds: 600
- name: notifications-shard2
rate: 500/s
bucket_size: 100
retry_parameters:
task_retry_limit: 9
min_backoff_seconds: 30
max_backoff_seconds: 600
- name: sailthru
rate: 50/s
bucket_size: 30
max_concurrent_requests: 10
retry_parameters:
task_retry_limit: 3
# We can't name this simply "delivery". It seems that alphabetical order matters in the unit tests
# and this needs to execute AFTER the "notifications" queue.
- name: nqe-delivery
rate: 500/s
bucket_size: 100
max_concurrent_requests: 300
- name: heavy-lifting
target: al-beef
rate: 500/s
bucket_size: 100
max_concurrent_requests: 500
retry_parameters:
task_retry_limit: 3
- name: catalog-validation
rate: 30/s
bucket_size: 10
max_concurrent_requests: 30
retry_parameters:
task_retry_limit: 10
- name: sendgrid-delivery
target: nqe-bus
rate: 500/s
bucket_size: 10
retry_parameters:
task_retry_limit: 10
- name: migrations
rate: 500/s
bucket_size: 100
- name: analyzer
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 2
max_backoff_seconds: 200
max_doublings: 0
- name: analyzer-shard1
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 2
max_backoff_seconds: 200
max_doublings: 0
- name: analyzer-shard2
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 2
max_backoff_seconds: 200
max_doublings: 0
- name: analyzer-shard3
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 2
max_backoff_seconds: 200
max_doublings: 0
- name: analyzer-shard4
rate: 500/s
bucket_size: 100
retry_parameters:
min_backoff_seconds: 2
max_backoff_seconds: 200
max_doublings: 0
# A queue with evergreen workers. When a worker dies, it respawns another one.
# We put all the workers on a backend as to reduce latency for all other requests.
- name: stathat-workers
target: queue-workers
rate: 100/s
bucket_size: 10
retry_parameters:
min_backoff_seconds: 2
max_backoff_seconds: 10
- name: product-index-workers
target: queue-workers
rate: 100/s
bucket_size: 40
max_concurrent_requests: 100
retry_parameters:
min_backoff_seconds: 2
max_backoff_seconds: 100
- name: keen-workers
target: queue-workers
rate: 100/s
bucket_size: 10
retry_parameters:
min_backoff_seconds: 2
max_backoff_seconds: 10
# A pull queue where the keys will be
- name: product-index-pull
mode: pull
- name: stathat-pull
mode: pull
- name: stathat-pull-shard1
mode: pull
- name: stathat-pull-shard2
mode: pull
- name: stathat-pull-shard3
mode: pull
- name: stathat-pull-shard4
mode: pull
- name: stathat-pull-shard5
mode: pull
- name: stathat-pull-shard6
mode: pull
- name: stathat-pull-shard7
mode: pull
- name: stathat-pull-shard8
mode: pull
- name: stathat-pull-shard9
mode: pull
- name: keen-pull
mode: pull
- name: keen-pull-shard1
mode: pull
- name: keen-pull-shard2
mode: pull
- name: keen-pull-shard3
mode: pull
- name: keen-pull-shard4
mode: pull
- name: keen-pull-shard5
mode: pull
- name: keen-pull-shard6
mode: pull
- name: keen-pull-shard7
mode: pull
- name: keen-pull-shard8
mode: pull
- name: keen-pull-shard9
mode: pull
- name: event-pull
mode: pull
# Shards for event pull
- name: event-pull-gap
mode: pull
- name: event-pull-nike
mode: pull
- name: event-pull-express
mode: pull
- name: event-pull-zumiez
mode: pull
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment