Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Couchbase Elasticsearch Connector Configuration - Example
# Introduction to TOML config file format:
# https://npf.io/2014/08/intro-to-toml/
# The work of replicating a Couchbase bucket may be distributed between
# several connector worker processes, known collectively as a "group".
#
# The connector name uniquely identifies a group of connector workers.
# All workers in the group must use the same name. The group name
# is required even if there is only one worker.
#
# TIP: Changing the name invalidates the replication status checkpoint.
# To preserve the checkpoint, run the `cbes-checkpoint-backup` command
# before renaming, and `cbes-checkpoint-restore` afterwards.
[group]
name = 'landmark-example-group'
# The simplest way to deploy the connector is "static" mode. In this mode,
# each worker process is independent of the others, and must be manually
# configured to know which partitions it is responsible for.
[group.static]
# Each member in a static group must have a unique member number.
memberNumber = 1 # A value from 1 to 'totalMembers', inclusive.
# The total number of workers in the static group.
#
# TIP: To safely add or remove workers from a static group, first stop all
# running workers, then reconfigure them with the new 'totalMembers' value,
# and finally start all the workers again.
totalMembers = 1
# Performance metrics may be written to a log file and/or exposed via HTTP.
[metrics]
httpPort = 31415 # Positive = fixed, 0 = ephemeral, -1 = disable HTTP server
logInterval = '1m' # Set to '0m' to disable metrics logging
# Required if secure connections are enabled (SSL/TLS).
[truststore]
# Use the Java keytool command to create this file and import the root
# certificate from the Security section of the Couchbase admin console
# and/or the CA certificate used by Elasticsearch.
path = 'path/to/truststore'
pathToPassword = 'secrets/truststore-password.toml'
[couchbase]
hosts = ['localhost']
network = 'auto'
bucket = 'travel-sample'
# By default the connector stores replication checkpoint documents
# in the source bucket. If you'd rather store those documents in a
# separate existing bucket, put that bucket's name here.
metadataBucket = ''
# At a minimum, the Couchbase user must have the "Data DCP Reader"
# role for the source bucket, and "Data Reader" & "Data Writer"
# roles for the bucket where replication checkpoints are stored.
username = 'Administrator'
# Path to a separate TOML file with a single 'password' key.
# The base for a relative path is the connector installation directory.
pathToPassword = 'secrets/couchbase-password.toml'
# Optionally use SSL/TLS to encrypt the connection between Couchbase and
# the connector. Requires Couchbase Enterprise Edition, and requires the
# [truststore] section to be configured.
secureConnection = false
# Options for the Couchbase Database Change Protocol (DCP).
[couchbase.dcp]
compression = true
flowControlBuffer = '128mb'
persistencePollingInterval = '100ms'
[elasticsearch]
hosts = ['localhost']
username = 'elastic'
# Path to a separate TOML file with a single 'password' key.
# The base for a relative path is the connector installation directory.
pathToPassword = 'secrets/elasticsearch-password.toml'
# Optionally use SSL/TLS to encrypt the connection between Elasticsearch and
# the connector. Requires the [truststore] section to be configured. See also:
# https://www.elastic.co/guide/en/elasticsearch/reference/current/configuring-tls.html
secureConnection = false
# If connecting directly to an Amazon Elasticsearch Service, specify the AWS region.
# AWS credentials are obtained from the Default Credential Provider Chain.
# https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/credentials.html
[elasticsearch.aws]
region = ''
[elasticsearch.bulkRequestLimits]
bytes = '10mb'
actions = 1000
timeout = '1m'
concurrentRequests = 2
[elasticsearch.docStructure]
# The Elasticsearch document may optionally contain Couchbase metadata
# (cas, revision, expiry, etc). If present, this will be a top-level field
# of the Elasticsearch document. If you have configured the connector
# to put document content at the top level, choose a field name that will
# not conflict with any document fields. To omit the metadata field,
# use an empty string.
metadataFieldName = 'meta'
# false means the ES document root will have a 'doc' field whose value is
# the Couchbase document.
# true means the ES document will be identical to the Couchbase document,
# with the possible addition of the metadata field.
documentContentAtTopLevel = true
# false means ignore Couchbase counter documents.
# true means replicate them as Object nodes like {"value":<counter>}
wrapCounters = false
# The values in this section may be overridden by specific type definitions.
[elasticsearch.typeDefaults]
# Write matching documents to this index.
# Empty string means "no default".
index = ''
# Send matching documents though this pipeline.
# Empty string means "no pipeline".
pipeline = ''
# Assign this type to matching documents.
# For ES 5.x remove leading underscore!
typeName = '_doc'
# If true, ignore matching documents entirely (implies 'ignoreDeletes').
ignore = false
# If true, never delete matching documents from Elasticsearch.
ignoreDeletes = false
# Sample document type definitions for the travel-sample bucket.
# Replace these to match your own data model.
#
# At a minimum, each type must specify a 'regex' (Java regular expression)
# or 'prefix' property. The type config matches any Couchbase document
# whose IDs matches the regex or prefix.
#
# The order of type definitions is significant. If a document ID matches
# more than one type, the assigned type will be the one that occurs first
# in this configuration. If none match, the document will not be indexed.
[[elasticsearch.type]]
prefix = '_sync:' # Couchbase Sync Gateway metadata
ignore = true
[[elasticsearch.type]]
prefix = 'landmark_'
index = 'landmark'
pipeline = 'landmark-pipeline'
# When Elasticsearch rejects a document (usually due to a type mapping error)
# a rejection log entry document is written to Elasticsearch. The log entry's
# document ID is the ID of the rejected Couchbase document. The log entry's
# content has the following fields:
# "index" - (string) name of the index the connector tried to write to
# "type" - (string) document type name used for the write attempt
# "action" - (string) failed action type ("INDEX" or "DELETE")
# "error" - (string) error message received from Elasticsearch
[elasticsearch.rejectionLog]
index = 'cbes-rejects'
typeName = '_doc' # For ES 5.x remove leading underscore!
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.