Created
February 20, 2020 11:50
-
-
Save AdheipSingh/ca9e9226c45e9fd3c08f7eefa5a42b1e to your computer and use it in GitHub Desktop.
druid-operator.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| apiVersion: druid.apache.org/v1alpha1 | |
| kind: Druid | |
| metadata: | |
| name: my-druid | |
| spec: | |
| common.runtime.properties: |- | |
| # | |
| # Extensions | |
| # | |
| druid.extensions.directory=/opt/druid/extensions | |
| druid.extensions.loadList=[“druid-kafka-indexing-service”,”druid-datasketches”,”postgresql-metadata-storage”] | |
| # | |
| # Logging | |
| # | |
| # Log all runtime properties on startup. Disable to avoid logging properties on startup: | |
| druid.startup.logging.logProperties=true | |
| # | |
| # Service discovery | |
| # | |
| druid.selectors.indexing.serviceName=druid/overlord | |
| druid.selectors.coordinator.serviceName=druid/coordinator | |
| # | |
| # Monitoring | |
| # | |
| druid.monitoring.monitors=[“com.metamx.metrics.JvmMonitor”] | |
| druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] | |
| druid.emitter=logging | |
| druid.emitter.logging.logLevel=info | |
| druid.request.logging.type=emitter | |
| druid.request.logging.feed=requests | |
| # Storage type of double columns | |
| # ommiting this will lead to index double as float at the storage layer | |
| druid.indexer.logs.type=file | |
| druid.indexer.logs.directory=logs/tasks | |
| druid.indexing.doubleStorage=double | |
| # to protect empty values converted to nulls | |
| druid.generic.useDefaultValueForNull=false | |
| druid.serverview.type=http | |
| druid.coordinator.loadqueuepeon.type=http | |
| druid.indexer.runner.type=httpRemote | |
| druid.announcer.skipSegmentAnnouncementOnZk=true | |
| # | |
| # | |
| druid.emitter=logging | |
| druid.monitoring.monitors=["org.apache.druid.java.util.metrics.JvmMonitor"] | |
| druid.server.http.enableForwardedRequestCustomizer=true | |
| deepStorage: | |
| spec: | |
| properties: |- | |
| druid.storage.type=local | |
| druid.storage.storageDirectory=/data/segments | |
| druid.indexer.logs.type=file | |
| druid.indexer.logs.directory=/data/logs/ | |
| type: default | |
| image: apache/incubator-druid:0.16.0-incubating | |
| jvm.options: |- | |
| -server | |
| -XX:+PrintFlagsFinal | |
| -XX:MaxDirectMemorySize=10240g | |
| -XX:+UnlockExperimentalVMOptions | |
| -XX:+UseCGroupMemoryLimitForHeap | |
| -Duser.timezone=UTC | |
| -Dfile.encoding=UTF-8 | |
| -Dlog4j.debug | |
| -XX:+ExitOnOutOfMemoryError | |
| -XX:HeapDumpPath=/druid/data/logs | |
| -XX:+HeapDumpOnOutOfMemoryError | |
| -XX:+UseG1GC | |
| -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager | |
| -XX:+UnlockDiagnosticVMOptions | |
| -XX:+PrintSafepointStatistics | |
| -XX:PrintSafepointStatisticsCount=1 | |
| -XX:+PrintGCDetails | |
| -XX:+PrintGCDateStamps | |
| -XX:+PrintGCApplicationStoppedTime | |
| -XX:+PrintGCApplicationConcurrentTime | |
| -XX:+UseGCLogFileRotation | |
| -XX:NumberOfGCLogFiles=50 | |
| -XX:GCLogFileSize=50m | |
| -Xloggc:/druid/data/logs/gc.log | |
| log4j.config: |- | |
| <Configuration status="WARN"> | |
| <Appenders> | |
| <Console name="logline" target="SYSTEM_OUT"> | |
| <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/> | |
| </Console> | |
| <Console name="msgonly" target="SYSTEM_OUT"> | |
| <PatternLayout pattern="%m%n"/> | |
| </Console> | |
| </Appenders> | |
| <Loggers> | |
| <Root level="info"> | |
| <AppenderRef ref="logline"/> | |
| </Root> | |
| <Logger name="org.apache.druid.java.util.emitter.core.LoggingEmitter" additivity="false" level="info"> | |
| <AppenderRef ref="msgonly"/> | |
| </Logger> | |
| </Loggers> | |
| </Configuration> | |
| # metadataStore: | |
| # properties: |- | |
| # druid.metadata.storage.type=postgresql | |
| # druid.metadata.storage.connector.connectURI=jdbc:postgresql://myrds.us-west-2.rds.amazonaws.com:5432/druiddb | |
| # druid.metadata.storage.connector.password={"type":"environment","variable":"DB_PWD_KEY"} | |
| # druid.metadata.storage.connector.createTables=true | |
| # druid.metadata.postgres.ssl.useSSL=true | |
| # druid.metadata.postgres.ssl.sslMode=verify-ca | |
| # druid.metadata.postgres.ssl.sslRootCert=/druid/cert/rds-combined-ca-bundle.pem | |
| # type: default | |
| nodes: | |
| brokers: | |
| tolerations: | |
| - key: "node-role.kubernetes.io/master" | |
| operator: "Exists" | |
| effect: "NoSchedule" | |
| nodeSelector: | |
| node-type: druid-query | |
| druid.port: 3333 | |
| extra.jvm.options: |- | |
| -Xmx64G | |
| -Xms64G | |
| nodeType: broker | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| ports: | |
| - | |
| containerPort: 8083 | |
| name: random | |
| replicas: 1 | |
| resources: | |
| limits: | |
| cpu: "1" | |
| memory: 2Gi | |
| requests: | |
| cpu: "1" | |
| memory: 2Gi | |
| runtime.properties: |- | |
| druid.service=druid/broker | |
| druid.broker.http.numConnections=5 | |
| druid.server.http.numThreads=5 | |
| druid.processing.buffer.sizeBytes=1073741824 | |
| druid.processing.numThreads=9 | |
| services: | |
| - | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 3333 | |
| targetPort: 3333 | |
| type: ClusterIP | |
| - | |
| metadata: | |
| name: broker-%s-service | |
| spec: | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 8080 | |
| targetPort: 3333 | |
| type: ClusterIP | |
| volumeClaimTemplates: | |
| - | |
| metadata: | |
| name: data-volume | |
| spec: | |
| accessModes: | |
| - ReadWriteOnce | |
| resources: | |
| requests: | |
| storage: 20Gi | |
| storageClassName: default | |
| volumeMounts: | |
| - | |
| mountPath: /druid/data | |
| name: data-volume | |
| coordinators: | |
| tolerations: | |
| - key: "node-role.kubernetes.io" | |
| operator: "Equal" | |
| value: "master" | |
| effect: "NoSchedule" | |
| nodeSelector: | |
| node-type: druid-master | |
| druid.port: 3333 | |
| extra.jvm.options: |- | |
| -Xmx4G | |
| -Xms4G | |
| nodeType: coordinator | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| ports: | |
| - | |
| containerPort: 8083 | |
| name: random | |
| replicas: 1 | |
| resources: | |
| limits: | |
| cpu: "1" | |
| memory: 6Gi | |
| requests: | |
| cpu: "1" | |
| memory: 6Gi | |
| runtime.properties: |- | |
| druid.service=druid/coordinator | |
| druid.coordinator.startDelay=PT30S | |
| druid.coordinator.period=PT30S | |
| druid.coordinator.kill.on=true | |
| druid.coordinator.kill.period=PT2H | |
| druid.coordinator.kill.durationToRetain=PT0s | |
| druid.coordinator.kill.maxSegments=5000 | |
| services: | |
| - | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 3333 | |
| targetPort: 3333 | |
| type: ClusterIP | |
| - | |
| metadata: | |
| name: coordinator-%s-service | |
| spec: | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 8080 | |
| targetPort: 3333 | |
| type: ClusterIP | |
| volumeClaimTemplates: | |
| - | |
| metadata: | |
| name: data-volume | |
| spec: | |
| accessModes: | |
| - ReadWriteOnce | |
| resources: | |
| requests: | |
| storage: 20Gi | |
| storageClassName: default | |
| volumeMounts: | |
| - | |
| mountPath: /druid/data | |
| name: data-volume | |
| historicals: | |
| tolerations: | |
| - key: "node-role.kubernetes.io" | |
| operator: "Equal" | |
| value: "master" | |
| effect: "NoSchedule" | |
| nodeSelector: | |
| node-type: druid-data | |
| druid.port: 3333 | |
| extra.jvm.options: |- | |
| -Xmx64G | |
| -Xms64G | |
| nodeType: historical | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| ports: | |
| - | |
| containerPort: 8084 | |
| name: random | |
| readinessProbe: | |
| httpGet: | |
| path: /druid/historical/v1/readiness | |
| replicas: 2 | |
| resources: | |
| limits: | |
| cpu: "1" | |
| memory: 10Gi | |
| requests: | |
| cpu: "1" | |
| memory: 10Gi | |
| runtime.properties: |- | |
| druid.service=druid/historical | |
| druid.server.http.numThreads=10 | |
| druid.processing.buffer.sizeBytes=1073741824 | |
| druid.server.maxSize=1099511627776 | |
| druid.segmentCache.locations=[{\"path\":\"/druid/data/segments\",\"maxSize\":1099511627776}] | |
| druid.processing.numThreads=9 | |
| services: | |
| - | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 3333 | |
| targetPort: 3333 | |
| type: ClusterIP | |
| volumeClaimTemplates: | |
| - | |
| metadata: | |
| name: data-volume | |
| spec: | |
| accessModes: | |
| - ReadWriteOnce | |
| resources: | |
| requests: | |
| storage: 20Gi | |
| storageClassName: default | |
| volumeMounts: | |
| - | |
| mountPath: /druid/data | |
| name: data-volume | |
| middlemanagers: | |
| tolerations: | |
| - key: "node-role.kubernetes.io" | |
| operator: "Equal" | |
| value: "master" | |
| effect: "NoSchedule" | |
| nodeSelector: | |
| node-type: druid-data | |
| druid.port: 3333 | |
| extra.jvm.options: |- | |
| -Xmx4G | |
| -Xms4G | |
| nodeType: middleManager | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| ports: | |
| - | |
| containerPort: 8100 | |
| name: peon-0-pt | |
| - | |
| containerPort: 8101 | |
| name: peon-1-pt | |
| - | |
| containerPort: 8102 | |
| name: peon-2-pt | |
| - | |
| containerPort: 8103 | |
| name: peon-3-pt | |
| - | |
| containerPort: 8104 | |
| name: peon-4-pt | |
| - | |
| containerPort: 8105 | |
| name: peon-5-pt | |
| - | |
| containerPort: 8106 | |
| name: peon-6-pt | |
| - | |
| containerPort: 8107 | |
| name: peon-7-pt | |
| - | |
| containerPort: 8108 | |
| name: peon-8-pt | |
| - | |
| containerPort: 8109 | |
| name: peon-9-pt | |
| replicas: 2 | |
| resources: | |
| limits: | |
| cpu: "2" | |
| memory: 10Gi | |
| requests: | |
| cpu: "2" | |
| memory: 10Gi | |
| runtime.properties: |- | |
| druid.service=druid/middleManager | |
| druid.worker.capacity=4 | |
| druid.indexer.runner.javaOpts=-server -XX:MaxDirectMemorySize=10240g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/druid/data/tmp -Dlog4j.debug -XX:+UnlockDiagnosticVMOptions -XX:+PrintSafepointStatistics -XX:PrintSafepointStatisticsCount=1 -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=50 -XX:GCLogFileSize=10m -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:+UseG1GC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -XX:HeapDumpPath=/druid/data/logs/peon.%t.%p.hprof -Xms10G -Xmx10G | |
| druid.indexer.task.baseTaskDir=/druid/data/baseTaskDir | |
| druid.server.http.numThreads=10 | |
| druid.indexer.fork.property.druid.processing.buffer.sizeBytes=1 | |
| druid.indexer.fork.property.druid.processing.numMergeBuffers=1 | |
| druid.indexer.fork.property.druid.processing.numThreads=1 | |
| druid.indexer.task.hadoopWorkingPath=/druid/data/hadoop-working-path | |
| druid.indexer.task.defaultHadoopCoordinates=[\"org.apache.hadoop:hadoop-client:2.7.3\"] | |
| services: | |
| - | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 3333 | |
| targetPort: 3333 | |
| - | |
| name: peon-port-0 | |
| port: 8100 | |
| targetPort: 8100 | |
| - | |
| name: peon-port-1 | |
| port: 8101 | |
| targetPort: 8101 | |
| - | |
| name: peon-port-2 | |
| port: 8102 | |
| targetPort: 8102 | |
| - | |
| name: peon-port-3 | |
| port: 8103 | |
| targetPort: 8103 | |
| - | |
| name: peon-port-4 | |
| port: 8104 | |
| targetPort: 8104 | |
| - | |
| name: peon-port-5 | |
| port: 8105 | |
| targetPort: 8105 | |
| - | |
| name: peon-port-6 | |
| port: 8106 | |
| targetPort: 8106 | |
| - | |
| name: peon-port-7 | |
| port: 8107 | |
| targetPort: 8107 | |
| - | |
| name: peon-port-8 | |
| port: 8108 | |
| targetPort: 8108 | |
| - | |
| name: peon-port-9 | |
| port: 8109 | |
| targetPort: 8109 | |
| - | |
| name: peon-port-10 | |
| port: 8110 | |
| targetPort: 8110 | |
| type: ClusterIP | |
| volumeClaimTemplates: | |
| - | |
| metadata: | |
| name: data-volume | |
| spec: | |
| accessModes: | |
| - ReadWriteOnce | |
| resources: | |
| requests: | |
| storage: 20Gi | |
| storageClassName: default | |
| volumeMounts: | |
| - | |
| mountPath: /druid/data | |
| name: data-volume | |
| overlords: | |
| tolerations: | |
| - key: "node-role.kubernetes.io" | |
| operator: "Equal" | |
| value: "master" | |
| effect: "NoSchedule" | |
| nodeSelector: | |
| node-type: druid-master | |
| druid.port: 3333 | |
| extra.jvm.options: |- | |
| -Xmx4G | |
| -Xms4G | |
| nodeType: overlord | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| ports: | |
| - | |
| containerPort: 8083 | |
| name: random | |
| replicas: 1 | |
| resources: | |
| limits: | |
| cpu: "2" | |
| memory: 6Gi | |
| requests: | |
| cpu: "2" | |
| memory: 6Gi | |
| runtime.properties: |- | |
| druid.service=druid/overlord | |
| druid.indexer.queue.startDelay=PT30S | |
| druid.indexer.runner.type=httpRemote | |
| druid.indexer.storage.type=metadata | |
| services: | |
| - | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 3333 | |
| targetPort: 3333 | |
| type: ClusterIP | |
| - | |
| metadata: | |
| name: overlord-%s-service | |
| spec: | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 8080 | |
| targetPort: 3333 | |
| type: ClusterIP | |
| volumeClaimTemplates: | |
| - | |
| metadata: | |
| name: data-volume | |
| spec: | |
| accessModes: | |
| - ReadWriteOnce | |
| resources: | |
| requests: | |
| storage: 10Gi | |
| storageClassName: default | |
| volumeMounts: | |
| - | |
| mountPath: /druid/data | |
| name: data-volume | |
| routers: | |
| tolerations: | |
| - key: "node-role.kubernetes.io" | |
| operator: "Equal" | |
| value: "master" | |
| effect: "NoSchedule" | |
| nodeSelector: | |
| node-type: druid-query | |
| druid.port: 3333 | |
| extra.jvm.options: |- | |
| -Xmx512m | |
| -Xms512m | |
| nodeType: router | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| ports: | |
| - | |
| containerPort: 8083 | |
| name: random | |
| replicas: 1 | |
| resources: | |
| limits: | |
| cpu: "1" | |
| memory: 2Gi | |
| requests: | |
| cpu: "1" | |
| memory: 2Gi | |
| runtime.properties: | | |
| druid.service=druid/router | |
| druid.router.managementProxy.enabled=true | |
| druid.auth.unsecuredPaths=["/status"] | |
| # druid.auth.authenticatorChain=["pac4j"] | |
| # druid.auth.authenticator.pac4j.type=pac4j | |
| # druid.auth.pac4j.oidc.clientID=00000myid0000 | |
| # druid.auth.pac4j.oidc.clientSecret={"type":"environment","variable":"OKTA_CLIENT_SECRET"}, | |
| # druid.auth.pac4j.oidc.cookiePassphrase={"type":"environment","variable":"OKTA_COOKIE_PASSWORD"}, | |
| # druid.auth.pac4j.oidc.discoveryURI=https://my.okta.com/oauth2/default/.well-known/openid-configuration | |
| services: | |
| - | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 3333 | |
| targetPort: 3333 | |
| type: ClusterIP | |
| - | |
| metadata: | |
| name: router-%s-service | |
| spec: | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 8080 | |
| targetPort: 3333 | |
| type: ClusterIP | |
| volumeClaimTemplates: | |
| - | |
| metadata: | |
| name: data-volume | |
| spec: | |
| accessModes: | |
| - ReadWriteOnce | |
| resources: | |
| requests: | |
| storage: 1Gi | |
| storageClassName: default | |
| volumeMounts: | |
| - | |
| mountPath: /druid/data | |
| name: data-volume | |
| supervisormms: | |
| druid.port: 3333 | |
| extra.jvm.options: |- | |
| -Xmx1G | |
| -Xms1G | |
| nodeType: middleManager | |
| podDisruptionBudgetSpec: | |
| maxUnavailable: 1 | |
| ports: | |
| - | |
| containerPort: 8100 | |
| name: peon-0-pt | |
| - | |
| containerPort: 8101 | |
| name: peon-1-pt | |
| - | |
| containerPort: 8102 | |
| name: peon-2-pt | |
| - | |
| containerPort: 8103 | |
| name: peon-3-pt | |
| - | |
| containerPort: 8104 | |
| name: peon-4-pt | |
| - | |
| containerPort: 8105 | |
| name: peon-5-pt | |
| - | |
| containerPort: 8106 | |
| name: peon-6-pt | |
| - | |
| containerPort: 8107 | |
| name: peon-7-pt | |
| - | |
| containerPort: 8108 | |
| name: peon-8-pt | |
| - | |
| containerPort: 8109 | |
| name: peon-9-pt | |
| replicas: 2 | |
| resources: | |
| limits: | |
| cpu: "2" | |
| memory: 6Gi | |
| requests: | |
| cpu: "2" | |
| memory: 6Gi | |
| runtime.properties: |- | |
| druid.service=druid/middleManager | |
| druid.worker.capacity=4 | |
| druid.indexer.runner.javaOpts=-server -XX:MaxDirectMemorySize=10240g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Djava.io.tmpdir=/druid/data/tmp -Dlog4j.debug -XX:+UnlockDiagnosticVMOptions -XX:+PrintSafepointStatistics -XX:PrintSafepointStatisticsCount=1 -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=50 -XX:GCLogFileSize=10m -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:+UseG1GC -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -XX:HeapDumpPath=/druid/data/logs/peon.%t.%p.hprof -Xms2G -Xmx2G | |
| druid.indexer.task.baseTaskDir=/druid/data/baseTaskDir | |
| druid.server.http.numThreads=10 | |
| druid.indexer.fork.property.druid.processing.buffer.sizeBytes=1 | |
| druid.indexer.fork.property.druid.processing.numMergeBuffers=1 | |
| druid.indexer.fork.property.druid.processing.numThreads=1 | |
| druid.indexer.task.hadoopWorkingPath=/druid/data/hadoop-working-path | |
| druid.indexer.task.defaultHadoopCoordinates=[\"org.apache.hadoop:hadoop-client:2.7.3\"] | |
| services: | |
| - | |
| spec: | |
| clusterIP: None | |
| ports: | |
| - | |
| name: tcp-service-port | |
| port: 3333 | |
| targetPort: 3333 | |
| - | |
| name: peon-port-0 | |
| port: 8100 | |
| targetPort: 8100 | |
| - | |
| name: peon-port-1 | |
| port: 8101 | |
| targetPort: 8101 | |
| - | |
| name: peon-port-2 | |
| port: 8102 | |
| targetPort: 8102 | |
| - | |
| name: peon-port-3 | |
| port: 8103 | |
| targetPort: 8103 | |
| - | |
| name: peon-port-4 | |
| port: 8104 | |
| targetPort: 8104 | |
| - | |
| name: peon-port-5 | |
| port: 8105 | |
| targetPort: 8105 | |
| - | |
| name: peon-port-6 | |
| port: 8106 | |
| targetPort: 8106 | |
| - | |
| name: peon-port-7 | |
| port: 8107 | |
| targetPort: 8107 | |
| - | |
| name: peon-port-8 | |
| port: 8108 | |
| targetPort: 8108 | |
| - | |
| name: peon-port-9 | |
| port: 8109 | |
| targetPort: 8109 | |
| - | |
| name: peon-port-10 | |
| port: 8110 | |
| targetPort: 8110 | |
| type: ClusterIP | |
| volumeClaimTemplates: | |
| - | |
| metadata: | |
| name: data-volume | |
| spec: | |
| accessModes: | |
| - ReadWriteOnce | |
| resources: | |
| requests: | |
| storage: 10Gi | |
| storageClassName: default | |
| volumeMounts: | |
| - | |
| mountPath: /druid/data | |
| name: data-volume | |
| readinessProbe: | |
| httpGet: | |
| path: /status | |
| rollingDeploy: true | |
| securityContext: | |
| fsGroup: 107 | |
| runAsUser: 106 | |
| # zookeeper: | |
| # spec: | |
| # properties: |- | |
| # druid.zk.service.host=zk-0.zk-hs,zk-1.zk-hs,zk-2.zk-hs | |
| # druid.zk.paths.base=/druid | |
| # druid.zk.service.compress=false | |
| # type: default |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment