Last active
February 29, 2020 01:38
-
-
Save AdheipSingh/7049a58bcca0a37b1755ea45feca4a15 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| apiVersion: druid.apache.org/v1alpha1 | |
| kind: Druid | |
| metadata: | |
| name: cluster | |
| spec: | |
| podAnnotations: | |
| iam.amazonaws.com/role: arn:aws:iam::9806916998632986329863220374:role/dsfdghjkl | |
| env: | |
| - name: AWS_REGION | |
| value: ap-south-1 | |
| common.runtime.properties: | | |
| # Zookeeper | |
| druid.zk.service.host=zk-cs.druid.svc.cluster.local | |
| druid.zk.paths.base=/druid | |
| #druid.zk.service.compress=false | |
| # Metadata Store | |
| druid.metadata.storage.type=postgresql | |
| druid.metadata.storage.connector.connectURI=jdbc:postgresql://postgres.druid.svc.cluster.local:5432/druiddb | |
| druid.metadata.storage.connector.password=admin123 | |
| druid.metadata.storage.connector.createTables=true | |
| # | |
| # Monitoring | |
| # | |
| druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] | |
| #druid.emitter=noop | |
| druid.emitter.logging.logLevel=debug | |
| # Storage type of double columns | |
| # ommiting this will lead to index double as float at the storage layer | |
| druid.indexing.doubleStorage=double | |
| # | |
| # Extensions | |
| # | |
| druid.extensions.directory=/opt/druid/extensions | |
| druid.extensions.loadList=["druid-s3-extensions","druid-kafka-indexing-service","druid-datasketches","postgresql-metadata-storage"] | |
| # Log all runtime properties on startup. Disable to avoid logging properties on startup: | |
| druid.startup.logging.logProperties=true | |
| # | |
| # Service discovery | |
| # | |
| druid.selectors.indexing.serviceName=druid/overlord | |
| druid.selectors.coordinator.serviceName=druid/coordinator | |
| druid.sql.enable=true | |
| deepStorage: | |
| spec: | |
| properties: |- | |
| druid.storage.type=s3 | |
| druid.storage.bucket=kjafflnfl | |
| druid.storage.baseKey=druid/segments | |
| druid.indexer.logs.directory=data/logs/ | |
| type: default | |
| commonConfigMountPath: /opt/druid/conf/druid/cluster/_common | |
| image: "apache/incubator-druid:0.16.0-incubating" | |
| jvm.options: |- | |
| -server | |
| -XX:MaxDirectMemorySize=10240g | |
| -Duser.timezone=UTC | |
| -Dfile.encoding=UTF-8 | |
| -Dlog4j.debug | |
| -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager | |
| nodes: | |
| brokers: | |
| affinity: | |
| nodeAffinity: | |
| requiredDuringSchedulingIgnoredDuringExecution: | |
| nodeSelectorTerms: | |
| - | |
| matchExpressions: | |
| - | |
| key: node-type | |
| operator: In | |
| values: | |
| - druid-query | |
| druid.port: 8082 | |
| extra.jvm.options: |- | |
| -Xmx1G | |
| -Xms1G | |
| nodeConfigMountPath: /opt/druid/conf/druid/cluster/query/broker | |
| nodeType: broker | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| replicas: 1 | |
| resources: | |
| limits: | |
| cpu: "1" | |
| memory: 6G | |
| requests: | |
| cpu: "1" | |
| memory: 6G | |
| runtime.properties: | | |
| druid.service=druid/broker | |
| druid.plaintextPort=8082 | |
| # HTTP server settings | |
| druid.server.http.numThreads=60 | |
| # HTTP client settings | |
| druid.broker.http.numConnections=50 | |
| druid.broker.http.maxQueuedBytes=10000000 | |
| # Processing threads and buffers | |
| druid.processing.buffer.sizeBytes=500000000 | |
| druid.processing.numMergeBuffers=6 | |
| druid.processing.numThreads=1 | |
| druid.processing.tmpDir=var/druid/processing | |
| # Query cache disabled -- push down caching and merging instead | |
| druid.broker.cache.useCache=false | |
| druid.broker.cache.populateCache=false | |
| services: | |
| - | |
| metadata: | |
| name: broker-%s-service | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 8082 | |
| targetPort: 8082 | |
| type: ClusterIP | |
| log4j.config: |- | |
| <Configuration status="WARN"> | |
| <Appenders> | |
| <Console name="logline" target="SYSTEM_OUT"> | |
| <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/> | |
| </Console> | |
| <Console name="msgonly" target="SYSTEM_OUT"> | |
| <PatternLayout pattern="%m%n"/> | |
| </Console> | |
| </Appenders> | |
| <Loggers> | |
| <Root level="info"> | |
| <AppenderRef ref="logline"/> | |
| </Root> | |
| <Logger name="org.apache.druid.java.util.emitter.core.LoggingEmitter" additivity="false" level="debug"> | |
| <AppenderRef ref="msgonly"/> | |
| </Logger> | |
| </Loggers> | |
| </Configuration> | |
| tolerations: | |
| - | |
| effect: NoSchedule | |
| key: node-role.kubernetes.io/master | |
| operator: Exists | |
| volumeClaimTemplates: | |
| - | |
| metadata: | |
| name: data-volume | |
| spec: | |
| accessModes: | |
| - ReadWriteOnce | |
| resources: | |
| requests: | |
| storage: 20Gi | |
| storageClassName: zone-a-storage | |
| volumeMounts: | |
| - | |
| mountPath: /druid/data | |
| name: data-volume | |
| securityContext: | |
| fsGroup: 1000 | |
| runAsGroup: 1000 | |
| runAsUser: 1000 | |
| coordinators: | |
| affinity: | |
| nodeAffinity: | |
| requiredDuringSchedulingIgnoredDuringExecution: | |
| nodeSelectorTerms: | |
| - | |
| matchExpressions: | |
| - | |
| key: node-type | |
| operator: In | |
| values: | |
| - druid-master | |
| druid.port: 8081 | |
| extra.jvm.options: |- | |
| -Xmx1G | |
| -Xms1G | |
| nodeConfigMountPath: /opt/druid/conf/druid/cluster/master/coordinator | |
| nodeType: coordinator | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| replicas: 1 | |
| resources: | |
| limits: | |
| cpu: "1" | |
| memory: 6G | |
| requests: | |
| cpu: "1" | |
| memory: 6G | |
| runtime.properties: | | |
| druid.service=druid/coordinator | |
| # HTTP server threads | |
| druid.coordinator.startDelay=PT30S | |
| druid.coordinator.period=PT30S | |
| druid.coordinator.merge.on=true | |
| services: | |
| - | |
| metadata: | |
| name: coordinator-%s-service | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 8081 | |
| targetPort: 8081 | |
| type: ClusterIP | |
| tolerations: | |
| - | |
| effect: NoSchedule | |
| key: node-role.kubernetes.io/master | |
| operator: Exists | |
| volumeMounts: | |
| - | |
| mountPath: /druid/data | |
| name: data-volume | |
| volumes: | |
| - | |
| emptyDir: {} | |
| name: data-volume | |
| securityContext: | |
| fsGroup: 1000 | |
| runAsGroup: 1000 | |
| runAsUser: 1000 | |
| historicals: | |
| affinity: | |
| nodeAffinity: | |
| requiredDuringSchedulingIgnoredDuringExecution: | |
| nodeSelectorTerms: | |
| - | |
| matchExpressions: | |
| - | |
| key: kubernetes.io/hostname | |
| operator: In | |
| values: | |
| - ip-1-9.ap-south-1.compute.internal | |
| druid.port: 8083 | |
| extra.jvm.options: |- | |
| -Xmx1G | |
| -Xms1G | |
| log4j.config: |- | |
| <Configuration status="WARN"> | |
| <Appenders> | |
| <Console name="logline" target="SYSTEM_OUT"> | |
| <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/> | |
| </Console> | |
| <Console name="msgonly" target="SYSTEM_OUT"> | |
| <PatternLayout pattern="%m%n"/> | |
| </Console> | |
| </Appenders> | |
| <Loggers> | |
| <Root level="info"> | |
| <AppenderRef ref="logline"/> | |
| </Root> | |
| <Logger name="org.apache.druid.java.util.emitter.core.LoggingEmitter" additivity="false" level="debug"> | |
| <AppenderRef ref="msgonly"/> | |
| </Logger> | |
| </Loggers> | |
| </Configuration> | |
| nodeType: historical | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| nodeConfigMountPath: /opt/druid/conf/druid/cluster/data/historical | |
| replicas: 1 | |
| resources: | |
| # limits: | |
| # cpu: "1" | |
| # memory: 8G | |
| # requests: | |
| # cpu: "1" | |
| # memory: 8G | |
| runtime.properties: | | |
| druid.service=druid/historical | |
| druid.plaintextPort=8083 | |
| # HTTP server threads | |
| druid.server.http.numThreads=5 | |
| # Processing threads and buffers | |
| druid.processing.buffer.sizeBytes=500000000 | |
| druid.processing.numMergeBuffers=4 | |
| druid.processing.numThreads=15 | |
| druid.processing.tmpDir=var/druid/processing | |
| # Segment storage | |
| druid.segmentCache.locations=[{"path":"var/druid/segment-cache","maxSize":300000000000}] | |
| druid.server.maxSize=300000000000 | |
| # Query cache | |
| druid.historical.cache.useCache=true | |
| druid.historical.cache.populateCache=true | |
| druid.cache.type=caffeine | |
| druid.cache.sizeInBytes=256000000 | |
| services: | |
| - | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 8083 | |
| targetPort: 8083 | |
| type: ClusterIP | |
| tolerations: | |
| - | |
| effect: NoSchedule | |
| key: node-role.kubernetes.io/master | |
| operator: Exists | |
| volumeMounts: | |
| - | |
| mountPath: /opt/apache-druid-0.16.0-incubating/var | |
| name: data-volume | |
| volumes: | |
| - | |
| hostPath: | |
| path: /data | |
| name: data-volume | |
| securityContext: | |
| fsGroup: 0 | |
| runAsGroup: 0 | |
| runAsUser: 0 | |
| middlemanagers: | |
| affinity: | |
| nodeAffinity: | |
| requiredDuringSchedulingIgnoredDuringExecution: | |
| nodeSelectorTerms: | |
| - | |
| matchExpressions: | |
| - | |
| key: kubernetes.io/hostname | |
| operator: In | |
| values: | |
| - ip-114-9.ap-south-1.compute.internal | |
| druid.port: 8091 | |
| extra.jvm.options: |- | |
| -Xmx4G | |
| -Xms4G | |
| nodeType: middleManager | |
| nodeConfigMountPath: /opt/druid/conf/druid/cluster/data/middlemanager | |
| log4j.config: |- | |
| <Configuration status="WARN"> | |
| <Appenders> | |
| <Console name="logline" target="SYSTEM_OUT"> | |
| <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/> | |
| </Console> | |
| <Console name="msgonly" target="SYSTEM_OUT"> | |
| <PatternLayout pattern="%m%n"/> | |
| </Console> | |
| </Appenders> | |
| <Loggers> | |
| <Root level="info"> | |
| <AppenderRef ref="logline"/> | |
| </Root> | |
| <Logger name="org.apache.druid.java.util.emitter.core.LoggingEmitter" additivity="false" level="info"> | |
| <AppenderRef ref="msgonly"/> | |
| </Logger> | |
| </Loggers> | |
| </Configuration> | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| ports: | |
| - | |
| containerPort: 8100 | |
| name: peon-0-pt | |
| - | |
| containerPort: 8101 | |
| name: peon-1-pt | |
| - | |
| containerPort: 8102 | |
| name: peon-2-pt | |
| - | |
| containerPort: 8103 | |
| name: peon-3-pt | |
| - | |
| containerPort: 8104 | |
| name: peon-4-pt | |
| replicas: 1 | |
| resources: | |
| limits: | |
| cpu: "2" | |
| memory: 5Gi | |
| requests: | |
| cpu: "2" | |
| memory: 5Gi | |
| runtime.properties: |- | |
| druid.service=druid/middleManager | |
| druid.plaintextPort=8091 | |
| # Number of tasks per middleManager | |
| druid.worker.capacity=4 | |
| # Task launch parameters | |
| druid.indexer.runner.javaOpts=-server -Xms1g -Xmx1g -XX:MaxDirectMemorySize=1g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+ExitOnOutOfMemoryError -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager | |
| druid.indexer.task.baseTaskDir=var/druid/task | |
| # HTTP server threads | |
| druid.server.http.numThreads=60 | |
| # Processing threads and buffers on Peons | |
| druid.indexer.fork.property.druid.processing.numMergeBuffers=2 | |
| druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100000000 | |
| druid.indexer.fork.property.druid.processing.numThreads=1 | |
| services: | |
| - | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 8091 | |
| targetPort: 8091 | |
| - | |
| name: peon-port-0 | |
| port: 8100 | |
| targetPort: 8100 | |
| - | |
| name: peon-port-1 | |
| port: 8101 | |
| targetPort: 8101 | |
| - | |
| name: peon-port-2 | |
| port: 8102 | |
| targetPort: 8102 | |
| - | |
| name: peon-port-3 | |
| port: 8103 | |
| targetPort: 8103 | |
| - | |
| name: peon-port-4 | |
| port: 8104 | |
| targetPort: 8104 | |
| type: ClusterIP | |
| tolerations: | |
| - | |
| effect: NoSchedule | |
| key: node-role.kubernetes.io/master | |
| operator: Exists | |
| volumeMounts: | |
| - | |
| mountPath: /opt/apache-druid-0.16.0-incubating/var | |
| name: data-volume | |
| volumes: | |
| - | |
| hostPath: | |
| path: /data | |
| name: data-volume | |
| securityContext: | |
| fsGroup: 0 | |
| runAsGroup: 0 | |
| runAsUser: 0 | |
| overlords: | |
| affinity: | |
| nodeAffinity: | |
| requiredDuringSchedulingIgnoredDuringExecution: | |
| nodeSelectorTerms: | |
| - | |
| matchExpressions: | |
| - | |
| key: node-type | |
| operator: In | |
| values: | |
| - druid-master | |
| druid.port: 8090 | |
| extra.jvm.options: |- | |
| -Xmx4G | |
| -Xms4G | |
| nodeType: overlord | |
| log4j.config: |- | |
| <Configuration status="WARN"> | |
| <Appenders> | |
| <Console name="logline" target="SYSTEM_OUT"> | |
| <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/> | |
| </Console> | |
| <Console name="msgonly" target="SYSTEM_OUT"> | |
| <PatternLayout pattern="%m%n"/> | |
| </Console> | |
| </Appenders> | |
| <Loggers> | |
| <Root level="info"> | |
| <AppenderRef ref="logline"/> | |
| </Root> | |
| <Logger name="org.apache.druid.java.util.emitter.core.LoggingEmitter" additivity="false" level="debug"> | |
| <AppenderRef ref="msgonly"/> | |
| </Logger> | |
| </Loggers> | |
| </Configuration> | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| nodeConfigMountPath: /opt/druid/conf/druid/cluster/master/overlord | |
| replicas: 1 | |
| resources: | |
| limits: | |
| cpu: "2" | |
| memory: 6Gi | |
| requests: | |
| cpu: "2" | |
| memory: 6Gi | |
| runtime.properties: |- | |
| druid.service=druid/overlord | |
| druid.indexer.queue.startDelay=PT30S | |
| druid.indexer.runner.type=remote | |
| druid.indexer.storage.type=metadata | |
| services: | |
| - | |
| metadata: | |
| name: overlord-%s-service | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 8090 | |
| targetPort: 8090 | |
| type: ClusterIP | |
| tolerations: | |
| - | |
| effect: NoSchedule | |
| key: node-role.kubernetes.io/master | |
| operator: Exists | |
| volumeClaimTemplates: | |
| - | |
| metadata: | |
| name: data-volume | |
| spec: | |
| accessModes: | |
| - ReadWriteOnce | |
| resources: | |
| requests: | |
| storage: 10Gi | |
| storageClassName: default | |
| volumeMounts: | |
| - | |
| mountPath: /druid/data | |
| name: data-volume | |
| securityContext: | |
| fsGroup: 1000 | |
| runAsGroup: 1000 | |
| runAsUser: 1000 | |
| routers: | |
| affinity: | |
| nodeAffinity: | |
| requiredDuringSchedulingIgnoredDuringExecution: | |
| nodeSelectorTerms: | |
| - | |
| matchExpressions: | |
| - | |
| key: node-type | |
| operator: In | |
| values: | |
| - druid-query | |
| druid.port: 8888 | |
| extra.jvm.options: |- | |
| -Xmx512m | |
| -Xms512m | |
| nodeType: router | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| nodeConfigMountPath: /opt/druid/conf/druid/cluster/query/router | |
| replicas: 1 | |
| runtime.properties: | | |
| druid.service=druid/router | |
| druid.plaintextPort=8888 | |
| # HTTP proxy | |
| druid.router.http.numConnections=50 | |
| druid.router.http.readTimeout=PT5M | |
| druid.router.http.numMaxThreads=100 | |
| druid.server.http.numThreads=100 | |
| # Service discovery | |
| druid.router.defaultBrokerServiceName=druid/broker | |
| druid.router.coordinatorServiceName=druid/coordinator | |
| # Management proxy to coordinator / overlord: required for unified web console. | |
| druid.router.managementProxy.enabled=true | |
| services: | |
| - | |
| metadata: | |
| name: router-%s-service | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 8888 | |
| targetPort: 8888 | |
| type: ClusterIP | |
| tolerations: | |
| - | |
| effect: NoSchedule | |
| key: node-role.kubernetes.io/master | |
| operator: Exists | |
| log4j.config: |- | |
| <Configuration status="WARN"> | |
| <Appenders> | |
| <Console name="logline" target="SYSTEM_OUT"> | |
| <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/> | |
| </Console> | |
| <Console name="msgonly" target="SYSTEM_OUT"> | |
| <PatternLayout pattern="%m%n"/> | |
| </Console> | |
| </Appenders> | |
| <Loggers> | |
| <Root level="info"> | |
| <AppenderRef ref="logline"/> | |
| </Root> | |
| <Logger name="org.apache.druid.java.util.emitter.core.LoggingEmitter" additivity="false" level="debug"> | |
| <AppenderRef ref="msgonly"/> | |
| </Logger> | |
| </Loggers> | |
| </Configuration> | |
| volumeClaimTemplates: | |
| - | |
| metadata: | |
| name: data-volume | |
| spec: | |
| accessModes: | |
| - ReadWriteOnce | |
| resources: | |
| requests: | |
| storage: 1Gi | |
| storageClassName: zone-a-storage | |
| volumeMounts: | |
| - | |
| mountPath: /druid/data | |
| name: data-volume | |
| securityContext: | |
| fsGroup: 1000 | |
| runAsGroup: 1000 | |
| runAsUser: 1000 | |
| startScript: /druid.sh |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment