Vector Config to Parse Postgres CSV Logs and Send to Logflare

vector_pg_csv_parser.toml

# Logs

[sources.pg_query_log]
    type = "file"
    include = [ "/logs/postgresql.csv" ]
    read_from = "end"
 
[sources.pg_query_log.multiline]
    start_pattern = "^20[0-9][0-9]-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9].[0-9]{3} UTC,\""
    mode = "halt_before"
    condition_pattern = "^20[0-9][0-9]-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9].[0-9]{3} UTC,\""
    timeout_ms = 1000

[transforms.csv_to_object]
    type = "remap"
    inputs = [ "pg_query_log" ]
    source = """
    csv_data = parse_csv!(.message)
    .parsed.timestamp = csv_data[0]
    .parsed.user_name = csv_data[1]
    .parsed.database_name = csv_data[2]
    .parsed.process_id = to_int(csv_data[3]) ?? null
    .parsed.connection_from = csv_data[4]
    .parsed.session_id = csv_data[5]
    .parsed.session_line_num = to_int(csv_data[6]) ?? null
    .parsed.command_tag = csv_data[7]
    .parsed.session_start_time = csv_data[8]
    .parsed.virtual_transaction_id = csv_data[9]
    .parsed.transaction_id = to_int(csv_data[10]) ?? null
    .parsed.error_severity = csv_data[11]
    .parsed.sql_state_code = csv_data[12]
    .parsed.message = csv_data[13]
    .parsed.detail = csv_data[14]
    .parsed.hint = csv_data[15]
    .parsed.internal_query = csv_data[16]
    .parsed.internal_query_pos = to_int(csv_data[17]) ?? null
    .parsed.context = csv_data[18]
    .parsed.query = csv_data[19]
    .parsed.query_pos = to_int(csv_data[20]) ?? null
    .parsed.location = csv_data[0]
    .parsed.application_name = csv_data[21]
    .parsed.backend_type = csv_data[22]
    .parsed.leader_pid = to_int(csv_data[23]) ?? null
    .parsed.query_id = to_int(csv_data[24]) ?? null

    z_ts = replace(.parsed.timestamp, " UTC", "Z")
    iso8601_ts = replace(z_ts, " ", "T")

    .timestamp = iso8601_ts
    
    .parsed_from = .message
    .message = .parsed.message

    del(.parsed.message)
    """
    
[sinks.http_logs]
    type = "http"
    inputs = ["csv_to_object"]
    encoding.codec = "json"
    compression = "none"
    uri = "https://api.logflare.app/logs/vector?api_key=XXXXX&source=XXXXXX"

Hi!

Could you provide log_line_prefix?

AFAIK the log_line_prefix does not affect the Postgres CSV logs as everything is already included.

thanks