sidecar-example.yml for MiNiFi and NiFi

sidecar-example.yml

apiVersion: v1
data:
  # TODO: replace with own minifi properties
  minifi.properties: |
    # Core Properties #
    nifi.version=0.11.0
    nifi.flow.configuration.file=./conf/config.yml
    nifi.administrative.yield.duration=30 sec
    # If a component has no work to do (is "bored"), how long should we wait before checking again for work?
    nifi.bored.yield.duration=100 millis

    # Comma separated path for the extension libraries. Relative path is relative to the minifi executable.
    nifi.extension.path=../extensions/*

    # Provenance Repository #
    nifi.provenance.repository.directory.default=${MINIFI_HOME}/provenance_repository
    nifi.provenance.repository.max.storage.time=1 MIN
    nifi.provenance.repository.max.storage.size=1 MB
    nifi.flowfile.repository.directory.default=${MINIFI_HOME}/flowfile_repository
    nifi.database.content.repository.directory.default=${MINIFI_HOME}/content_repository
    nifi.provenance.repository.class.name=NoOpRepository
    nifi.content.repository.class.name=DatabaseContentRepository

    #nifi.remote.input.secure=true
    #nifi.security.need.ClientAuth=
    #nifi.security.client.certificate=
    #nifi.security.client.private.key=
    #nifi.security.client.pass.phrase=
    #nifi.security.client.ca.certificate=
    #nifi.security.use.system.cert.store=

    # Optional username/password used to authenticate against NiFi in RemoteProcessorGroups (i.e. Site-to-site configurations)
    #nifi.rest.api.user.name=admin
    #nifi.rest.api.password=password

    # State storage configuration #
    ## The default state storage can be overridden by specifying a controller service instance
    ## that implements CoreComponentStateManagementProvider
    ## (e.g. an instance of RocksDbPersistableKeyValueStoreService or UnorderedMapPersistableKeyValueStoreService)
    #nifi.state.management.provider.local=
    #nifi.state.manangement.provider.local.path=
    ## To make the default state storage persist every state change, set this to true
    ## this comes at a performance penalty, but makes sure no state is lost even on unclean shutdowns
    #nifi.state.management.provider.local.always.persist=true
    ## To change the frequency at which the default state storage is persisted, modify the following
    #nifi.state.management.provider.local.auto.persistence.interval=1 min

    # Disk space watchdog #
    ## Stops MiNiFi FlowController activity (excluding C2), when the available disk space on either of the repository
    ## volumes go below stop.threshold, checked every interval, then restarts when the available space on all
    ## repository volumes reach at least restart.threshold.
    #minifi.disk.space.watchdog.enable=true
    #minifi.disk.space.watchdog.interval=15 sec
    #minifi.disk.space.watchdog.stop.threshold=100 MB
    #minifi.disk.space.watchdog.restart.threshold=150 MB

    ## Enabling C2 Uncomment each of the following options
    ## define those with missing options
    #nifi.c2.enable=true
    ## define protocol parameters
    ## The default is RESTSender.
    ## Alternatively, you may use CoapProtocol if that extension is built.
    #nifi.c2.agent.protocol.class=RESTSender
    #nifi.c2.agent.protocol.class=CoapProtocol
    #nifi.c2.agent.coap.host=
    #nifi.c2.agent.coap.port=
    ## base URL of the c2 server,
    ## very likely the same base url of rest urls
    #nifi.c2.flow.base.url=
    #nifi.c2.rest.url=
    #nifi.c2.rest.url.ack=
    nifi.c2.root.classes=DeviceInfoNode,AgentInformation,FlowInformation
    ## Minimize heartbeat payload size by excluding agent manifest from the heartbeat
    nifi.c2.full.heartbeat=false
    ## heartbeat 4 times a second
    #nifi.c2.agent.heartbeat.period=250
    ## define parameters about your agent
    #nifi.c2.agent.class=
    #nifi.c2.agent.identifier=
    ## define metrics reported
    nifi.c2.root.class.definitions=metrics
    nifi.c2.root.class.definitions.metrics.name=metrics
    nifi.c2.root.class.definitions.metrics.metrics=typedmetrics
    nifi.c2.root.class.definitions.metrics.metrics.typedmetrics.name=RuntimeMetrics
    nifi.c2.root.class.definitions.metrics.metrics.queuemetrics.name=QueueMetrics
    nifi.c2.root.class.definitions.metrics.metrics.queuemetrics.classes=QueueMetrics
    nifi.c2.root.class.definitions.metrics.metrics.typedmetrics.classes=ProcessMetrics,SystemInformation
    nifi.c2.root.class.definitions.metrics.metrics.processorMetrics.name=ProcessorMetric
    nifi.c2.root.class.definitions.metrics.metrics.processorMetrics.classes=GetFileMetrics

    ## Specify multiple heartbeat handlers.
    ## HeartbeatLogger logs the heartbeats on TRACE for debugging.
    #nifi.c2.agent.heartbeat.reporter.classes=HeartbeatLogger

    ## enable the controller socket provider on port 9998
    ## off by default. C2 must be enabled to support these
    #controller.socket.host=localhost
    #controller.socket.port=9998


    #JNI properties
    nifi.framework.dir=${MINIFI_HOME}/minifi-jni/lib
    nifi.nar.directory=${MINIFI_HOME}/minifi-jni/nars
    nifi.nar.deploy.directory=${MINIFI_HOME}/minifi-jni/nardeploy
    nifi.nar.docs.directory=${MINIFI_HOME}/minifi-jni/nardocs
    # must be comma separated
    nifi.jvm.options=-Xmx1G
    nifi.python.processor.dir=${MINIFI_HOME}/minifi-python/

  config.yml: |
    Flow Controller:
      name: minifi-logging
    Processors:
    - id: 94b8e610-b4ed-1ec9-b26f-c839931bf3e2
      name: TailFile
      class: org.apache.nifi.processors.standard.TailFile
      max concurrent tasks: 1
      scheduling strategy: TIMER_DRIVEN
      scheduling period: 5 sec
      auto-terminated relationships list: []
      Properties:
        File to Tail: /nifi-logs/nifi-app.log
        tail-mode: Single file
        Lookup frequency: 1 min
    - id: 261e8cf1-71ba-af86-fb2b-bc95764f91f8
      name: MergeContent
      class: org.apache.nifi.processors.standard.MergeContent
      max concurrent tasks: 1
      scheduling strategy: EVENT_DRIVEN
      auto-terminated relationships list:
        - original
      Properties:
        # TODO optmize merging strategy for your own preferences
        Attribute Strategy: Keep Only Common Attributes
        Maximum number of Bins: 100
        Minimum Group Size: 0
        Max Bin Age: 1 min
        Minimum Number of Entries: 1000000
        Maximum Group Size: 1000000
        Maximum Number of Entries: 1000000
        Merge Strategy: Bin-Packing Algorithm
    - id: 69335770-ee29-11eb-9a03-0242ac130003
      name: CompressContent
      class: org.apache.nifi.processors.standard.CompressContent
      max concurrent tasks: 1
      scheduling strategy: EVENT_DRIVEN
      auto-terminated relationships list:
        - failure
      Properties:
        Compression Level: 6
        Compression Format: gzip
        UpdateFileName: false
    - id: fe198bd9-2a1c-316e-0000-000000000000
      name: PutS3Object
      class: org.apache.nifi.minifi.azure.processors.PutS3Object
      max concurrent tasks: 1
      scheduling strategy: EVENT_DRIVEN
      auto-terminated relationships list:
        - success
      Properties:
        Bucket: test_bucket  # TODO: replace with own S3 bucket name
        AWS Credentials Provider service: AWSCredentialsService
    Controller Services:
    - name: AWSCredentialsService
      id: 2094d776-2006-4d02-9bb9-28eac9d0fc95
      class: org.apache.nifi.minifi.aws.controllers.AWSCredentialsService
      Properties:
        Access Key: access_key  # TODO: Replace with own AWS access key id
        Secret Key: secret_key  # TODO: Replace with own AWS secret key
    Connections:
    - id: 99f617e7-49a1-6078-8534-26af7d56ca08
      name: TailFile/success/MergeContent
      source name: TailFile
      source relationship names:
      - success
      destination name: MergeContent
      max work queue size: 1000000
      max work queue data size: 10 MB
      flowfile expiration: 0 sec
      queue prioritizer class: ''
    - id: 24d6be1e-ee29-11eb-9a03-0242ac130003
      name: MergeContent/merged/CompressContent
      source name: MergeContent
      source relationship names:
      - merged
      destination name: CompressContent
      max work queue size: 1000000
      max work queue data size: 10 MB
      flowfile expiration: 0 sec
      queue prioritizer class: ''
    - id: 67ea5c91-446a-393b-6274-b6fae2f475a2
      name: CompressContent/success/PutS3Object
      source name: CompressContent
      source relationship names:
      - success
      destination name: PutS3Object
      max work queue size: 1000000
      max work queue data size: 10 MB
      flowfile expiration: 0 sec
      queue prioritizer class: ''
    Remote Process Groups: []
kind: ConfigMap
metadata:
  labels:
    k8s-app: minifi-log-collection
  name: minifi-log-collection-config
  namespace: default
---
apiVersion: v1
kind: Pod
metadata:
  name: log-collection-minifi-pod
  namespace: default
spec:
  containers:
  - name: nifi
    image: apache/nifi:latest
    volumeMounts:
    - name: nifi-logs
      mountPath: /opt/nifi/nifi-current/logs
  - name: sidecar-minifi
    image: container.repo.cloudera.com/cloudera/apacheminificpp:latest
    volumeMounts:
    - name: nifi-logs
      mountPath: /nifi-logs
    - name: minificonfig
      mountPath: /opt/minifi/minifi-current/conf/config.yml
      subPath: config.yml
    - name: minificonfig
      mountPath: /opt/minifi/minifi-current/conf/minifi.properties
      subPath: minifi.properties
  volumes:
  - name: nifi-logs
    emptyDir: {}
  - configMap:
      defaultMode: 420
      name: minifi-log-collection-config
    name: minificonfig