hreidar

ingress1_1  | Apr 19 14:13:00.007 TRACE sink{component_kind="sink" component_name=ingress1_to_dedup component_type=http}: vector::sinks::util::sink: Service ready; Sending batch.
ingress1_1  | Apr 19 14:13:00.007 TRACE sink{component_kind="sink" component_name=ingress1_to_dedup component_type=http}: vector::sinks::util::sink: Submitting service request. in_flight_requests=1
ingress1_1  | Apr 19 14:13:00.009 DEBUG http: vector::internal_events::http_client: Sending HTTP request. uri=http://127.0.0.1:9001/ method=POST version=HTTP/1.1 headers={"content-type": "application/json", "user-agent": "Vector/0.12.2 (v0.12.2 x86_64-unknown-linux-gnu 2021-03-30)", "accept-encoding": "identity"} body=[4561 bytes]
ingress1_1  | Apr 19 14:13:00.009 TRACE http: hyper::client::pool: take? ("http", 127.0.0.1:9001): expiration = Some(90s)
ingress1_1  | Apr 19 14:13:00.009 DEBUG http: hyper::client::pool: reuse idle connection for ("http", 127.0.0.1:9001)
ingress1_1  | Apr 19 14:13:00.011 TRACE encode_headers: hyper::proto::h1::

hreidar

## inputs
[sources.from_kafka]
  type = "kafka"
  bootstrap_servers = "k8s-c4-m1:30656,k8s-c4-m2:30656,k8s-c4-m3:30656"
  group_id = "app1-test"
  topics = ["APP1.TEST"]
  auto_offset_reset = "beginning"

[sources.metrics]
  type = "internal_metrics"

hreidar

[sources.in_from_kafka]
  type = "kafka"
  bootstrap_servers = "k8s-c4-m1:30656,k8s-c4-m2:30656,k8s-c4-m3:30656"
  group_id = "app1-test"
  topics = ["APP1.TEST"]
  auto_offset_reset = "beginning"

[sources.metrics]
  type = "internal_metrics"

hreidar

# vector elastic sink config
[sinks.eck-c1-ip_leases]
  type = "elasticsearch"
  inputs = ["ipv4-json-rename"]
  healthcheck = true
  request.concurrency = "auto"
  host = "https://eck-c1.my.org:9200"
  tls.verify_hostname = false
  index = "ip_leases-%Y-%m"
  auth.strategy = "basic"

hreidar

# HELP vector_open_files open_files
# TYPE vector_open_files gauge
vector_open_files 1
# HELP vector_internal_metrics_cardinality internal_metrics_cardinality
# TYPE vector_internal_metrics_cardinality counter
vector_internal_metrics_cardinality 34
# HELP vector_checksum_errors_total checksum_errors_total
# TYPE vector_checksum_errors_total counter
vector_checksum_errors_total{file="/nfsmount/cnr02/ipv6_leased.log"} 1
# HELP vector_files_resumed_total files_resumed_total

hreidar

Jan 15 11:04:41.410 DEBUG source{component_kind="source" component_name=cnr02-ipv4 component_type=file}:file_server: file_source::file_server: event_throughput=0.000/sec bytes_throughput=0.000/sec ratios={"discovery": 0.010746088, "other": 0.00015594058, "reading": 0.000008748576, "sending": 0.000047970625, "sleeping": 0.98904186}
Jan 15 11:04:42.329 DEBUG vector::internal_events::file::source: Files checkpointed. count=2 duration_ms=2
Jan 15 11:04:42.331 DEBUG vector::internal_events::file::source: Files checkpointed. count=0 duration_ms=5
Jan 15 11:04:42.353 DEBUG vector::internal_events::file::source: Files checkpointed. count=62 duration_ms=4
Jan 15 11:04:42.355 DEBUG vector::internal_events::file::source: Files checkpointed. count=53 duration_ms=5
Jan 15 11:04:42.356 DEBUG vector::internal_events::file::source: Files checkpointed. count=40 duration_ms=6
Jan 15 11:04:42.384 DEBUG vector::internal_events::file::source: Files checkpointed. count=45 duration_ms=3
Jan 15 11:04:42.385 DEBUG vector::internal_ev

hreidar

Sep 22 11:25:23.014  INFO source{name=cnr02-ipv4 type=file}:file_server: file_source::file_server: Found file to watch. path="/nfsmount/cnr02/ip_leased.log" file_position=0
Sep 23 00:00:03.989  WARN source{name=cnr01-ipv6 type=file}:file_server: file_source::file_server: Ignoring file smaller than fingerprint_bytes file="/nfsmount/cnr01/ipv6_leased.log"
Sep 23 00:00:04.088  INFO source{name=cnr01-ipv4 type=file}:file_server: file_source::file_server: Found file to watch. path="/nfsmount/cnr01/ip_leased.log" file_position=0
Sep 23 00:00:22.458  INFO source{name=cnr01-ipv6 type=file}:file_server: file_source::file_server: Found file to watch. path="/nfsmount/cnr01/ipv6_leased.log" file_position=0
Sep 23 00:14:22.401  INFO vector: Shutting down.
Sep 23 00:14:23.084  INFO vector: Log level "debug" is enabled.
Sep 23 00:14:23.089  INFO vector: Loading configs. path=["/etc/vector/vector.toml"]
Sep 23 00:14:23.107  INFO vector: Vector is starting. version="0.10.0" git_version="v0.9.0-377-g0f0311a" released="Wed, 22 

hreidar

filebeat:
    prospectors:
        -
            paths:
                - "/nfsmount/cnr01/ip_leased.log"
            input_type: log
            document_type: ipv4_lease_log
            fields_under_root: true
            fields:
                origin: cnr01

hreidar

[sources.p2-logs-from_ingress]
  type = "kafka"
  bootstrap_servers = "k8s-c4-m1:30656,k8s-c4-m2:30656,k8s-c4-m3:30656"
  group_id = "p2-vector-parser"
  topics = ["P2_OSS_AND_IP.R"]


# split messages to swimlanes 
[transforms.splitting_types]
  inputs = ["p2-logs-from_ingress"]