Skip to content

Instantly share code, notes, and snippets.

@kan6004
Last active April 1, 2021 22:57
Show Gist options
  • Star 3 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save kan6004/8fc90bec059515e081bf to your computer and use it in GitHub Desktop.
Save kan6004/8fc90bec059515e081bf to your computer and use it in GitHub Desktop.
fluentd + nxlog conf
## built-in TCP input
## $ echo <json> | fluent-cat <tag>
<source>
type forward
</source>
# HTTP input
# http://localhost:8888/<tag>?json=<json>
<source>
type http
port 8888
bind 0.0.0.0
</source>
## nginx input
<source>
type tail
path /usr/local/nginx/logs/access.log
format nginx
tag nginx.access
pos_file /usr/local/nginx/logs/nginx.pos
</source>
## output nginx access log to Elasticsearch
<match nginx.access>
type elasticsearch
type_name access_log
host localhost
port 9200
logstash_format true
include_tag_key true
tag_key @log_name
</match>
## output Linux syslog to Elasticsearch
<match src.var.log.messages>
type elasticsearch
type_name messages
host localhost
port 9200
logstash_format true
include_tag_key true
tag_key @log_name
</match>
## output Windows EventLog to Elasticsearch
<match windows.eventlog>
type elasticsearch
type_name windows_event_log
host localhost
port 9200
logstash_format true
include_tag_key true
tag_key @log_name
</match>
## parse Windows resource statistics
<match windows.stats>
type copy
<store>
type map
tag "map.windows.stats.cpu-processor-time"
time time
record {"value" => record["stat"]["Processor"]["processor_time"], "stat" => "cpu-processor-time", "hostname" => record["Hostname"]}
</store>
<store>
type map
tag "map.windows.stats.cpu-user-time"
time time
record {"value" => record["stat"]["Processor"]["user_time"], "stat" => "cpu-user-time", "hostname" => record["Hostname"]}
</store>
<store>
type map
tag "map.windows.stats.cpu-privileged-time"
time time
record {"value" => record["stat"]["Processor"]["privileged_time"], "stat" => "cpu-privileged-time", "hostname" => record["Hostname"]}
</store>
<store>
type map
tag "map.windows.stats.cpu-interrupt_time"
time time
record {"value" => record["stat"]["Processor"]["interrupt_time"], "stat" => "cpu-interrupt-time", "hostname" => record["Hostname"]}
</store>
<store>
type map
tag "map.windows.stats.disk-time"
time time
record {"value" => record["stat"]["PhysicalDisk"]["disk_time"], "stat" => "disk-time", "hostname" => record["Hostname"]}
</store>
<store>
type map
tag "map.windows.stats.disk-queue-length"
time time
record {"value" => record["stat"]["PhysicalDisk"]["current_disk_queue_length"], "stat" => "disk-queue-length", "hostname" => record["Hostname"]}
</store>
<store>
type map
tag "map.windows.stats.memory-available-mbytes"
time time
record {"value" => record["stat"]["Memory"]["available_mbytes"], "stat" => "memory-available-mbytes", "hostname" => record["Hostname"]}
</store>
<store>
type map
tag "map.windows.stats.memory-pages-per-sec"
time time
record {"value" => record["stat"]["Memory"]["pages_per_sec"], "stat" => "memory-pages-per-sec", "hostname" => record["Hostname"]}
</store>
<store>
type map
tag "map.windows.stats.network-bytes-total-per-sec"
time time
record {"value" => record["stat"]["NetworkInterface"]["bytes_total_per_sec"], "stat" => "network-bytes-total-per-sec", "hostname" => record["Hostname"]}
</store>
</match>
## output Windows resource statistics to Elasticsearch
<match map.windows.stats.**>
type elasticsearch
type_name windows_stats
host localhost
port 9200
logstash_format true
include_tag_key true
tag_key @log_name
</match>
## This is a sample configuration file. See the nxlog reference manual about the
## configuration options. It should be installed locally and is also available
## online at http://nxlog.org/nxlog-docs/en/nxlog-reference-manual.html
## Please set the ROOT to the folder your nxlog was installed into,
## otherwise it will not start.
#define ROOT C:\Program Files\nxlog
define ROOT C:\Program Files (x86)\nxlog
Moduledir %ROOT%\modules
CacheDir %ROOT%\data
Pidfile %ROOT%\data\nxlog.pid
SpoolDir %ROOT%\data
LogFile %ROOT%\data\nxlog.log
################################################################################
## Extensions
################################################################################
<Extension json>
Module xm_json
</Extension>
################################################################################
## Inputs
################################################################################
#<Input internal>
# Module im_internal
# Exec $Message = replace($Message, "&", "_");
# Exec $raw_event = "json=" + to_json();
#</Input>
<Input eventlog>
Module im_msvistalog
Query <QueryList>\
<Query Id="0">\
<Select Path="Application">*</Select>\
<Select Path="System">*</Select>\
</Query>\
</QueryList>
Exec $Message = replace($Message, "&", "_");
Exec $raw_event = "json=" + to_json();
</Input>
<Input stats>
Module im_file
File 'C:\work\stats.log'
InputType LineBased
SavePos TRUE
</Input>
################################################################################
## Outputs
################################################################################
#<Output out_internal>
# Module om_http
# ContentType application/x-www-form-urlencoded
# URL http://<fluentd in_http ip>:8888/nxlog.internal
#</Output>
<Output out_eventlog>
Module om_http
ContentType application/x-www-form-urlencoded
URL http://<fluentd in_http ip>:8888/windows.eventlog
</Output>
<Output out_stats>
Module om_http
ContentType application/x-www-form-urlencoded
URL http://<fluentd in_http ip>:8888/windows.stats
</Output>
################################################################################
## Routes
################################################################################
#<Route 1>
# Path internal => out_internal
#</Route>
<Route 2>
Path eventlog => out_eventlog
</Route>
<Route 3>
Path stats => out_stats
</Route>
################################################################################
## For Debug
################################################################################
#<Extension fileop>
# Module xm_fileop
#</Extension>
#<Output out_file>
# Module om_file
# File "C:\Program Files (x86)\\nxlog\data\\nxlog_output.log"
#</Output>
#<Route debug>
# Path in => out_file
#</Route>
function Get-PerfCounter ($counter, $parameter, $instance = $null) {
$pc = $null
if ($counter -eq "Network Interface") {
$pc = @()
$networkAdapters = Get-WmiObject Win32_NetworkAdapter
$networkAdapters | ForEach-Object {
if ($_.MACAddress -ne $null) {
$pc += new-object System.Diagnostics.PerformanceCounter($counter, $parameter, $_.Name)
}
}
} else {
if ($instance -eq $null) {
$pc = new-object System.Diagnostics.PerformanceCounter($counter, $parameter)
} else {
$pc = new-object System.Diagnostics.PerformanceCounter($counter, $parameter, $instance)
}
}
$pc
}
function Get-PerfValue ($perfValues) {
$result = 'json={'
$result += '"Hostname":"' + [Net.DNS]::GetHostName() + '",'
$result += '"stat":{'
$count = @{"counter" = 0; "parameter" = 0}
foreach ($counter in $perfValues.keys) {
$count["counter"] += 1
$count["parameter"] = 0
$result += '"' + $counter + '":{'
foreach ($parameter in $perfValues[$counter].keys) {
$count["parameter"] += 1
if ($count["parameter"] -eq $perfValues[$counter].keys.count) {
$result += '"' + $parameter + '":' + $perfValues[$counter][$parameter]
} else {
$result += '"' + $parameter + '":' + $perfValues[$counter][$parameter] + ','
}
}
if ($count["counter"] -eq $perfValues.keys.count) {
$result += '}'
} else {
$result += '},'
}
}
$result += '}}'
$result
}
function main () {
$INTERVAL_SEC = 30
# パフォーマンスカウンタの取得
$perfCounters = @{
"Memory" = @{
"available_mbytes" = Get-PerfCounter "Memory" "Available MBytes";
"pages_per_sec" = Get-PerfCounter "Memory" "Pages/sec"
};
"NetworkInterface" = @{
"bytes_total_per_sec" = Get-PerfCounter "Network Interface" "Bytes Total/sec"
};
"PhysicalDisk" = @{
"disk_time" = Get-PerfCounter "PhysicalDisk" "% Disk Time" "_Total";
"current_disk_queue_length" = Get-PerfCounter "PhysicalDisk" "Current Disk Queue Length" "_Total"
};
"Processor" = @{
"processor_time" = Get-PerfCounter "Processor" "% Processor Time" "_Total";
"user_time" = Get-PerfCounter "Processor" "% User Time" "_Total";
"privileged_time" = Get-PerfCounter "Processor" "% Privileged Time" "_Total";
"interrupt_time" = Get-PerfCounter "Processor" "% Interrupt Time" "_Total"
}
}
# パフォーマンス値の収集
while($true) {
$perfValues = @{
"Memory" = @{
"available_mbytes" = 0;
"pages_per_sec" = 0
};
"NetworkInterface" = @{
"bytes_total_per_sec" = 0
};
"PhysicalDisk" = @{
"disk_time" = 0;
"current_disk_queue_length" = 0
};
"Processor" = @{
"processor_time" = 0;
"user_time" = 0;
"privileged_time" = 0;
"interrupt_time" = 0
}
}
foreach ($counter in $perfCounters.keys) {
foreach ($parameter in $perfCounters[$counter].keys) {
if ($counter -eq "Network Interface") {
foreach ($pc in $perfCounters[$counter][$parameter]) {
$perfValues[$counter][$parameter] += $pc.NextValue()
}
} else {
$perfValues[$counter][$parameter] = $perfCounters[$counter][$parameter].NextValue()
}
}
}
Get-PerfValue($perfValues) | Out-File "C:\work\stats.log" -encoding utf8 -append
Sleep -Second $INTERVAL_SEC
}
}
main
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment