Skip to content

Instantly share code, notes, and snippets.

@guyboertje
Created January 31, 2019 14:29
Show Gist options
  • Save guyboertje/2b1134cbbfed46eca7fb5598812f099e to your computer and use it in GitHub Desktop.
Save guyboertje/2b1134cbbfed46eca7fb5598812f099e to your computer and use it in GitHub Desktop.
Logstash xml file processing
<?xml version="1.0" encoding="UTF-8" ?>
<ExecutionLogModel>
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="initializing..." shapetype="initializing..." shapelabel="" shapeextendedinfo="">
<Message>Executing Process Performance Management- Past(Child)-STAGE (Continuation f_0_0)</Message>
</LogEvent>
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="shape10" shapetype="Try/Catch" shapelabel="" shapeextendedinfo="">
<Message>Executing Try/Catch Shape continuation as f_0_0 with 1 document(s).</Message>
</LogEvent>
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="shape10" shapetype="Try/Catch" shapelabel="" shapeextendedinfo="">
<Message>Shape executed successfully in 82 ms.</Message>
</LogEvent>
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="shape13" shapetype="Map" shapelabel="" shapeextendedinfo="Performance Management- Past Year(groovy to FF)">
<Message>Executing Map with 1 document(s).</Message>
</LogEvent>
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="shape13" shapetype="Map" shapelabel="" shapeextendedinfo="Performance Management- Past Year(groovy to FF)">
<Message>Shape executed successfully in 2 ms.</Message>
</LogEvent>
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="shape14" shapetype="Return Documents" shapelabel="Success" shapeextendedinfo="">
<Message>Adding 1 document(s) to Return Documents return store.</Message>
</LogEvent>
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="shape14" shapetype="Return Documents" shapelabel="Success" shapeextendedinfo="">
<Message>Shape executed successfully in 1 ms.</Message>
</LogEvent>
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="cleanup..." shapetype="cleanup..." shapelabel="" shapeextendedinfo="">
<Message>Returning 1 document(s) in Return Documents return store.</Message>
</LogEvent>
<LogEvent level="INFO" time="2018-07-10T04:12:26Z" shapename="cleanup..." shapetype="cleanup..." shapelabel="" shapeextendedinfo="">
<Message>Process execution completed normally.</Message>
</LogEvent>
</ExecutionLogModel>
{
"shapelabel" => "",
"@timestamp" => 2019-01-31T14:25:40.653Z,
"@version" => "1",
"time" => "2018-07-10T04:12:26Z",
"shapename" => "initializing...",
"Message" => "Executing Process Performance Management- Past(Child)-STAGE (Continuation f_0_0)",
"level" => "INFO",
"shapeextendedinfo" => "",
"path" => "/elastic/tmp/testing/logs/logs_sample.xml",
"shapetype" => "initializing...",
"host" => "Elastics-MacBook-Pro.local"
}
{
"shapelabel" => "",
"@timestamp" => 2019-01-31T14:25:40.653Z,
"@version" => "1",
"time" => "2018-07-10T04:12:26Z",
"shapename" => "shape10",
"Message" => "Executing Try/Catch Shape continuation as f_0_0 with 1 document(s).",
"level" => "INFO",
"shapeextendedinfo" => "",
"path" => "/elastic/tmp/testing/logs/logs_sample.xml",
"shapetype" => "Try/Catch",
"host" => "Elastics-MacBook-Pro.local"
}
{
"shapelabel" => "",
"@timestamp" => 2019-01-31T14:25:40.653Z,
"@version" => "1",
"time" => "2018-07-10T04:12:26Z",
"shapename" => "shape10",
"Message" => "Shape executed successfully in 82 ms.",
"level" => "INFO",
"shapeextendedinfo" => "",
"path" => "/elastic/tmp/testing/logs/logs_sample.xml",
"shapetype" => "Try/Catch",
"host" => "Elastics-MacBook-Pro.local"
}
{
"shapelabel" => "",
"@timestamp" => 2019-01-31T14:25:40.653Z,
"@version" => "1",
"time" => "2018-07-10T04:12:26Z",
"shapename" => "shape13",
"Message" => "Executing Map with 1 document(s).",
"level" => "INFO",
"shapeextendedinfo" => "Performance Management- Past Year(groovy to FF)",
"path" => "/elastic/tmp/testing/logs/logs_sample.xml",
"shapetype" => "Map",
"host" => "Elastics-MacBook-Pro.local"
}
{
"shapelabel" => "",
"@timestamp" => 2019-01-31T14:25:40.653Z,
"@version" => "1",
"time" => "2018-07-10T04:12:26Z",
"shapename" => "shape13",
"Message" => "Shape executed successfully in 2 ms.",
"level" => "INFO",
"shapeextendedinfo" => "Performance Management- Past Year(groovy to FF)",
"path" => "/elastic/tmp/testing/logs/logs_sample.xml",
"shapetype" => "Map",
"host" => "Elastics-MacBook-Pro.local"
}
{
"shapelabel" => "Success",
"@timestamp" => 2019-01-31T14:25:40.653Z,
"@version" => "1",
"time" => "2018-07-10T04:12:26Z",
"shapename" => "shape14",
"Message" => "Adding 1 document(s) to Return Documents return store.",
"level" => "INFO",
"shapeextendedinfo" => "",
"path" => "/elastic/tmp/testing/logs/logs_sample.xml",
"shapetype" => "Return Documents",
"host" => "Elastics-MacBook-Pro.local"
}
{
"shapelabel" => "Success",
"@timestamp" => 2019-01-31T14:25:40.653Z,
"@version" => "1",
"time" => "2018-07-10T04:12:26Z",
"shapename" => "shape14",
"Message" => "Shape executed successfully in 1 ms.",
"level" => "INFO",
"shapeextendedinfo" => "",
"path" => "/elastic/tmp/testing/logs/logs_sample.xml",
"shapetype" => "Return Documents",
"host" => "Elastics-MacBook-Pro.local"
}
{
"shapelabel" => "",
"@timestamp" => 2019-01-31T14:25:40.653Z,
"@version" => "1",
"time" => "2018-07-10T04:12:26Z",
"shapename" => "cleanup...",
"Message" => "Returning 1 document(s) in Return Documents return store.",
"level" => "INFO",
"shapeextendedinfo" => "",
"path" => "/elastic/tmp/testing/logs/logs_sample.xml",
"shapetype" => "cleanup...",
"host" => "Elastics-MacBook-Pro.local"
}
{
"shapelabel" => "",
"@timestamp" => 2019-01-31T14:25:40.653Z,
"@version" => "1",
"time" => "2018-07-10T04:12:26Z",
"shapename" => "cleanup...",
"Message" => "Process execution completed normally.",
"level" => "INFO",
"shapeextendedinfo" => "",
"path" => "/elastic/tmp/testing/logs/logs_sample.xml",
"shapetype" => "cleanup...",
"host" => "Elastics-MacBook-Pro.local"
}
input {
file {
sincedb_path => "/elastic/tmp/testing/confs/test-jk-xmlpretty.sincedb"
path => "/elastic/tmp/testing/logs/logs_sample.xml"
mode => "read"
file_completed_action => "log"
file_completed_log_path => "/elastic/tmp/testing/confs/test-jk-xmlpretty-done.txt"
delimiter => "øhåyø"
}
}
filter {
xml {
remove_namespaces => true
store_xml => true
target => "[log_event]"
source => "[message]"
force_array => false
force_content => false
}
split {
field => "[log_event][LogEvent]"
remove_field => ["message"]
}
ruby {
code => '
event.remove("[log_event]")["LogEvent"].each{|k,v| event.set(k,v)}
'
}
}
output { stdout { codec => "rubydebug"} }
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment