vector-0.12.1-config

vector.toml

[sources.internal]
  type = "internal_metrics"

[sources.kafka_in]
  type = "kafka"
  bootstrap_servers = "kafka-all-broker.kafka.svc.cluster.local:29092"
  group_id = "gcs-shipper"
  auto_offset_reset = "beginning"
  topics = ["syslog-norm"]

[transforms.all_transforms]
  type = "remap"
  inputs = ["kafka_in"]
  drop_on_error = true
  source= '''
    parsed = parse_json!(.message)
    .timestamp = .@timestamp
    del(.@timestamp)
    del(.source_type)
    del(.structured_data)
    .timeuploaded = now()
    .procId = int(.procId) ?? 0
    .total = int(.total) ?? 0
  '''

[sinks.gcs]
  type = "gcp_cloud_storage"
  inputs = ["all_transforms"]
  bucket = "logs"
  compression = "none"

  # Auth
  credentials_path = "/creds/credentials.json"

  # Buffering
  buffer.type = "memory"
  buffer.max_size = 10240000

  # Encoding
  encoding.codec = "ndjson"

  # File Naming
  key_prefix = "%F/%H/" # optional, default

  # Storage settings
  storage_class = "STANDARD"
  batch.max_bytes = 268435456

  # Healthcheck
  healthcheck.enabled = false # optional, default

[sinks.prometheus]
  type = "prometheus_exporter"
  inputs = ["internal"]
  address = "0.0.0.0:9598"