NAB artificial data logstash conf

nab_logstash.conf

input {
	stdin {
  }
}

filter {
     csv {
        columns => ["timestamp","value"]
		separator => ","
        convert => { 'value' => 'float' }
	}
     date {
        match => [ "timestamp", "YYYY-MM-dd HH:mm:ss" ]
        locale => en
        remove_field => "timestamp"
     }
}

output {
    stdout { codec => dots }
    elasticsearch {
        index => "${INDEX_NAME}"
        user => "${ES_AUTH_UN}"
        password => "${ES_AUTH_PW}"
     	  hosts => ["${ES_HOST}"]
    }
}

simple_execute.sh

#!/usr/bin/env bash

for i in for i in $(find ~/Projects/NAB/data -name '*.csv')
do
    if test -f "$i"
    then
        file_name=$(basename $i)
        dirty_name=${file_name%.*}
        export INDEX_NAME=$(echo $dirty_name | tr '[:upper:]' '[:lower:]')
        cat $i | ./logstash -f nab_logstash.conf
    fi
done

NAB datasets can be seen here: https://github.com/numenta/NAB/tree/master/data