agup006

[PARSER]
    Name   apache
    Format regex
    Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
    Time_Key time
    Time_Format %d/%b/%Y:%H:%M:%S %z

[PARSER]
    Name   apache2
    Format regex

agup006

 # Taken from https://docs.k3s.io/installation/airgap#prepare-the-images-directory-and-k3s-binary
  runAsRoot mkdir -p /var/lib/rancher/k3s/agent/images/ "$BINARY_DIR"/
  runAsRoot cp -fv "$CALYPTIA_ROOT/k3s/k3s-airgap-images-$ARCH.tar" /var/lib/rancher/k3s/agent/images/
  runAsRoot chmod a+r /var/lib/rancher/k3s/agent/images/*.tar
  local k3s_exe=k3s
  if [[ "$ARCH" != "amd64" ]]; then
    k3s_exe="k3s-$ARCH"
  fi
  runAsRoot cp -f "$CALYPTIA_ROOT/k3s/$k3s_exe" "$BINARY_DIR"/
  runAsRoot chmod 755 "$BINARY_DIR/$k3s_exe"

agup006

function cb_filter(tag, timestamp, record)
    record.tag = tag
    return 1, timestamp, record
end

agup006

#!/bin/bash
set -eu

# Make sure to do this first for Openshift
if [[ "${SKIP_SA:-no}" != "no" ]]; then
    echo "Skipping service account creation"
fi

NAMESPACE=calyptia

agup006

function process(tag, timestamp, record)
    -- Splitting the log by tabs to extract the fields
    local fields = {}
    for field in string.gmatch(record["log"], "([^\t]+)") do
        table.insert(fields, field)
    end

    -- Extracting fields from the split log
    local date = fields[1]
    local time = fields[2]

agup006

# This is a sample configuration for Calyptia Fluent Bit
# This reads logs from the server and routes them to standard output

[INPUT]
  tag             dummy.4aaf0fd6-0f0c-47a3-92b1-7432382f5f3f
  Name            dummy
  rate            1
  dummy           {"message":"hello world"}
  samples         0
  start_time_sec  -1

agup006

[INPUT]
    Name         winevtlog
    Channels     Setup,Windows PowerShell
    Interval_Sec 1
    DB           winevtlog.sqlite

[OUTPUT]
    Name   stdout
    Match  *

agup006

Description: EC2 Instance with Calyptia Core
Mappings:
  RegionsAmis:
    us-east-1:
      '111': ami-0e7c24109670b64ec
    us-east-2:
      '111': ami-014087e40c89090ec
Parameters:
  Version:
    AllowedValues:

agup006

Methodology

The following gist contains all files needed to take raw NGINX access logs and then conver them into a suitable format that is ready to ingest within OpenSearch. These components include the following:

• Mock NGINX logs
• Fluent Bit configuration file
• Fluent Bit parsers.conf file (This is the default and a user would not require to define)

Important Considerations and Notes

1. When converting to the proper format we use a lua script to perform this inline. To reduce complexity we do this in a single line, however a user may wish to abstract this and add as a seperate file
2. TraceID and SpanID are set as the same values seen in the schema and are not dynamically generated
3. timestamp is not seen in Standard Output and instead is only seen when sending the data to OpenSearch

agup006

local dquote = ('"'):byte()

local function extract_simple(record, start)
  local index = record:find(',', start)
  local stop_index
  local next_offset
  if index ~= nil then
    stop_index = index - 1
    next_offset = index + 1
  end