druid_sparse

common.runtime.properties

druid.extensions.directory=extensions
druid.extensions.loadList=["druid-s3-extensions", "druid-rabbitmq", "postgresql-metadata-storage"]

druid.zk.service.host=sv-zookeeper

druid.metadata.storage.type=postgresql
druid.metadata.storage.connector.connectURI=jdbc:<postgresurl>
druid.metadata.storage.connector.user=<postgresuser>
druid.metadata.storage.connector.password=<postgrespass>

druid.storage.type=s3
druid.storage.bucket=<s3bucket>
druid.s3.accessKey=<s3access>
druid.s3.secretKey=<s3secret>

druid.cache.type=local
druid.cache.sizeInBytes=10000000

druid.emitter=logging

overlord.cmd

java -Xmx16g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath ../config/_common:conf/druid/overlord:lib/* io.druid.cli.Main server overlord

overlord.runtime.properties

druid.service=druid/overlord
druid.port=8090

druid.indexer.queue.startDelay=PT0M
druid.indexer.runner.javaOpts=-server -Xmx20g
# -XX:+UseG1GC -XX:MaxGCPauseMillis=100
druid.indexer.fork.property.druid.processing.numThreads=8
#druid.indexer.fork.property.druid.processing.buffer.sizeBytes=536870912
#druid.indexer.fork.property.druid.server.http.numThreads=200
druid.indexer.fork.property.druid.computation.buffer.size=1000000000

task.json

{
    "type": "index_realtime",
    "spec": {
        "dataSchema": {
            "dataSource": "logdb_data",
            "parser": {
                "type": "map",
                "parseSpec": {
                    "format": "json",
                    "timestampSpec": {
                        "column": "timestamp",
                        "format": "millis"
                    },
                    "dimensionsSpec": {
                        "dimensions": ["config_id", "stream_name"],
                        "dimensionExclusions": [],
                        "spatialDimensions": []
                    }
                }
            },
            "metricsSpec": [
                {"type": "count", "name": "count"},
                {"type": "doubleSum", "name": "data_num", "fieldName": "data_num"}
            ],
            "granularitySpec": {
                "type": "uniform",
                "segmentGranularity": "HOUR",
                "queryGranularity": "NONE"
            }
        },
        "ioConfig": {
            "type": "realtime",
            "firehose": {
                "type": "timed",
                "shutoffTime": "2016-05-01T20:00:00",
                "delegate": {
                    "type": "receiver",
                    "serviceName": "whdruid_receiver"
                }
            }
        },
        "tuningConfig": {
            "type": "realtime",
            "rejectionPolicy": {
                "type": "none"
            }
        }
    }
}