Created
September 25, 2018 22:25
Star
You must be signed in to star a gist
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Prod configuration for PIV s3 Loader | |
# Sources currently supported are: | |
# 'kinesis' for reading records from a Kinesis stream | |
# 'nsq' for reading records from a NSQ topic | |
source = "kinesis" | |
# Sink is used for sending events which processing failed. | |
# Sinks currently supported are: | |
# 'kinesis' for writing records to a Kinesis stream | |
# 'nsq' for writing records to a NSQ topic | |
sink = "kinesis" | |
# The following are used to authenticate for the Amazon Kinesis sink. | |
# If both are set to 'default', the default provider chain is used | |
# (see http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) | |
# If both are set to 'iam', use AWS IAM Roles to provision credentials. | |
# If both are set to 'env', use environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY | |
aws { | |
accessKey = "iam" | |
secretKey = "iam" | |
} | |
# Config for NSQ | |
nsq { | |
channelName = "{{dummy}}" | |
host = "{{nsqHost}}" | |
# TCP port for nsqd, 4150 by default | |
port = 4150 | |
# Host name for lookupd | |
lookupHost = "{{lookupHost}}" | |
# HTTP port for nsqlookupd, 4161 by default | |
lookupPort = 4161 | |
} | |
kinesis { | |
initialPosition = LATEST | |
initialTimestamp = AT_TIMESTAMP | |
# Maximum number of records to read per GetRecords call | |
# Chosen per Josh recommendation at: | |
# https://discourse.snowplowanalytics.com/t/350k-rpm-of-throughput-with-stream-collector-kinesis/103 | |
maxRecords = 500 | |
region = "us-west-2" | |
# "appName" is used for a DynamoDB table to maintain stream state. | |
appName = "piv_sink" | |
} | |
streams { | |
# Input stream name | |
inStreamName = "stream-enricher-to-s3" | |
# Stream for events for which the storage process fails | |
outStreamName = "stream-bad-data-to-s3" | |
# Events are accumulated in a buffer before being sent to S3. | |
# The buffer is emptied whenever: | |
# - the combined size of the stored records exceeds byteLimit or | |
# - the number of stored records exceeds recordLimit or | |
# - the time in milliseconds since it was last emptied exceeds timeLimit | |
buffer { | |
byteLimit = 8 # Not supported by NSQ; will be ignored | |
recordLimit = 1 | |
timeLimit = 15 # Not supported by NSQ; will be ignored | |
} | |
} | |
s3 { | |
region = "us-west-2" | |
bucket = "piv-stream-data-prod-bucket" | |
# Format is one of lzo or gzip | |
# Note, that you can use gzip only for enriched data stream. | |
format = "lzo" | |
# Maximum Timeout that the application is allowed to fail for | |
maxTimeout = 1 | |
} | |
# Optional section for tracking endpoints | |
#monitoring { | |
# snowplow{ | |
# collectorUri = "{{collectorUri}}" | |
# collectorPort = 80 | |
# appId = "{{appName}}" | |
# method = "{{method}}" | |
# } | |
#} | |
logging { | |
level: "INFO" | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment