Created
December 2, 2020 00:26
-
-
Save KannarFr/48c7ccc581ac430900f2e2cb551e068e to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Given | |
``` | |
[sources.files] | |
type = "file" | |
include = ["/home/kannar/git/cc/conf/vector/logs_cargo.test"] | |
ignore_older = 86400 # seconds | |
start_at_beginning = true | |
oldest_first = true | |
# Fingerprinting | |
fingerprinting.fingerprint_bytes = 256 # bytes, relevant when strategy = "checksum" | |
fingerprinting.ignored_header_bytes = 0 # bytes, relevant when strategy = "checksum" | |
fingerprinting.strategy = "checksum" | |
# Multiline | |
multiline.condition_pattern = "^[\\s]+" # required | |
multiline.mode = "continue_through" # required | |
multiline.start_pattern = "^[^\\s]" # required | |
multiline.timeout_ms = 1000 # required, milliseconds | |
[transforms.files_purifier] | |
type = "remove_fields" | |
inputs = ["files"] | |
# Commented fields are kept. | |
fields = [ | |
"file", | |
"source_type", | |
"host" | |
] | |
[transforms.files_copy] | |
# General | |
type = "add_fields" | |
inputs = ["files_purifier"] | |
overwrite = true | |
fields.timestamp_micros = "{{ timestamp }}" | |
[transforms.files_logs_types_definer] | |
type = "coercer" | |
inputs = ["files_copy"] | |
drop_unspecified = false | |
types.timestamp_micros = "timestamp" # example | |
[transforms.foo] | |
type = "coercer" | |
inputs = ["files_logs_types_definer"] | |
drop_unspecified = false | |
types.timestamp_micros = "int" | |
[sinks.stdout] | |
type = "console" | |
inputs = ["foo"] | |
target = "stdout" | |
encoding = "json" | |
``` | |
with file containing | |
``` | |
[2020-11-30T17:10:47.685+0000] [Payara 4.1] [INFO] [] [com.clevercloud.ccapi.amqp.EventsListener] [tid: _ThreadID=127 _ThreadName=EventsListener - Thread for clever.api.events] [timeMillis: 1606756247685] [levelValue: 800] [[ | |
[2020-11-30 17:10:47] Received event CLEVER_TOOLS_REQUEST from user_28d4f02c-2b22-4140-a255-8ce638eb07c3]] | |
[2020-11-30T17:11:16.799+0000] [Payara 4.1] [INFO] [] [com.clevercloud.ccapi.amqp.EventsListener] [tid: _ThreadID=127 _ThreadName=EventsListener - Thread for clever.api.events] [timeMillis: 1606756276799] [levelValue: 800] [[ | |
[2020-11-30 17:11:16] Received event CLEVER_TOOLS_REQUEST from user_6cc65066-c000-41d9-92c0-f42faf9277bf]] | |
[2020-11-30T17:11:30.668+0000] [Payara 4.1] [INFO] [] [com.clevercloud.ccapi.helpers.user.UsersHelper] [tid: _ThreadID=36 _ThreadName=http-thread-pool::http-listener-1(4)] [timeMillis: 1606756290668] [levelValue: 800] [[ | |
Found 1 owner with 0 apps.]] | |
``` | |
outputs | |
``` | |
kannar@pond ~/git/cc/vector (git)-[pulsar-avro] % ./target/debug/vector -c /home/kannar/git/cc/rubydeployer/data/conf/vector.toml [0][1:14:57 - 02/12/2020] | |
Dec 02 01:15:32.772 INFO vector::app: Log level is enabled. level="vector=info,codec=info,file_source=info,tower_limit=trace,rdkafka=info" | |
Dec 02 01:15:32.773 INFO vector::app: Loading configs. path=[("/home/kannar/git/cc/rubydeployer/data/conf/vector.toml", None)] | |
Dec 02 01:15:32.805 INFO vector::topology: Running healthchecks. | |
Dec 02 01:15:32.805 INFO vector::topology: Starting source. name="files" | |
Dec 02 01:15:32.805 INFO vector::topology::builder: Healthcheck: Passed. | |
Dec 02 01:15:32.806 INFO vector::topology: Starting transform. name="oui" | |
Dec 02 01:15:32.806 INFO vector::topology: Starting transform. name="files_purifier" | |
Dec 02 01:15:32.806 INFO source{component_kind="source" component_name=files component_type=file}: vector::sources::file: Starting file server. include=["/home/kannar/git/cc/conf/vector/logs_cargo.test"] exclude=[] | |
Dec 02 01:15:32.806 INFO vector::topology: Starting transform. name="files_logs_types_definer" | |
Dec 02 01:15:32.807 INFO vector::topology: Starting transform. name="files_copy" | |
Dec 02 01:15:32.807 INFO vector::topology: Starting sink. name="stdout" | |
Dec 02 01:15:32.807 INFO vector: Vector has started. version="0.11.0" git_version="v0.9.0-1537-g9a7105a" released="Tue, 01 Dec 2020 05:20:13 +0000" arch="x86_64" | |
Dec 02 01:15:32.807 INFO vector::app: API is disabled, enable by setting `api.enabled` to `true` and use commands like `vector top`. | |
Dec 02 01:15:32.809 INFO source{component_kind="source" component_name=files component_type=file}:file_server: file_source::checkpointer: Loaded checkpoint data. | |
Dec 02 01:15:32.810 INFO source{component_kind="source" component_name=files component_type=file}:file_server: vector::internal_events::file::source: Found new file to watch. path="/home/kannar/git/cc/conf/vector/logs_cargo.test" | |
{"message":"[2020-11-30T17:10:47.685+0000] [Payara 4.1] [INFO] [] [com.clevercloud.ccapi.amqp.EventsListener] [tid: _ThreadID=127 _ThreadName=EventsListener - Thread for clever.api.events] [timeMillis: 1606756247685] [levelValue: 800] [[\n [2020-11-30 17:10:47] Received event CLEVER_TOOLS_REQUEST from user_28d4f02c-2b22-4140-a255-8ce638eb07c3]]","timestamp":"2020-12-02T00:15:32.828477862Z"} | |
{"message":"[2020-11-30T17:11:16.799+0000] [Payara 4.1] [INFO] [] [com.clevercloud.ccapi.amqp.EventsListener] [tid: _ThreadID=127 _ThreadName=EventsListener - Thread for clever.api.events] [timeMillis: 1606756276799] [levelValue: 800] [[\n [2020-11-30 17:11:16] Received event CLEVER_TOOLS_REQUEST from user_6cc65066-c000-41d9-92c0-f42faf9277bf]]","timestamp":"2020-12-02T00:15:32.829823396Z"} | |
^CDec 02 01:15:33.668 INFO vector: Vector has stopped. | |
Dec 02 01:15:33.672 INFO vector::topology: Shutting down... Waiting on running components. remaining_components="files_copy, files_logs_types_definer, oui, files, stdout, files_purifier" time_remaining="59 seconds left" | |
{"message":"[2020-11-30T17:11:30.668+0000] [Payara 4.1] [INFO] [] [com.clevercloud.ccapi.helpers.user.UsersHelper] [tid: _ThreadID=36 _ThreadName=http-thread-pool::http-listener-1(4)] [timeMillis: 1606756290668] [levelValue: 800] [[\n Found 1 owner with 0 apps.]]","timestamp":"2020-12-02T00:15:33.687141509Z"} | |
``` | |
timestamp_micros field is not well parsed | |
``` | |
Dec 02 01:17:47.388 DEBUG transform{component_kind="transform" component_name=oui component_type=coercer}: vector::internal_events::coercer: Could not convert types. field=timestamp_micros error=IntParseError { s: "2020-12-02T00:17:47.380249922Z", source: ParseIntError { kind: InvalidDigit } } rate_limit_secs=30 | |
``` |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment