logstash-pg-es.conf

input {
  jdbc {
    jdbc_driver_library => "drivers/postgresql-42.2.5.jar"
    jdbc_driver_class => "org.postgresql.Driver"
    jdbc_connection_string => "${DATABASE_URL}"
    jdbc_user => "postgres"
    schedule => "* * * * *"
    statement => "SELECT * from rss_items where id > :sql_last_value order by id asc limit 100000"
    use_column_value => true
    tracking_column => "id"
    last_run_metadata_path => "last_state/rss_items"
  }
  jdbc {
    jdbc_driver_library => "drivers/postgresql-42.2.5.jar"
    jdbc_driver_class => "org.postgresql.Driver"
    jdbc_connection_string => "${DATABASE_URL}"
    jdbc_user => "postgres"
    schedule => "* * * * *"
    statement => "SELECT * from tweets where id > :sql_last_value order by id asc limit 100000"
    use_column_value => true
    tracking_column => "id"
    last_run_metadata_path => "last_state/tweets"
  }
  jdbc {
    jdbc_driver_library => "drivers/postgresql-42.2.5.jar"
    jdbc_driver_class => "org.postgresql.Driver"
    jdbc_connection_string => "${DATABASE_URL}"
    jdbc_user => "postgres"
    schedule => "* * * * *"
    statement => "SELECT * from facebook_posts where id > :sql_last_value order by id asc limit 100000"
    use_column_value => true
    tracking_column => "id"
    last_run_metadata_path => "last_state/facebook_posts"
  }
  jdbc {
    jdbc_driver_library => "drivers/postgresql-42.2.5.jar"
    jdbc_driver_class => "org.postgresql.Driver"
    jdbc_connection_string => "${DATABASE_URL}"
    jdbc_user => "postgres"
    schedule => "* * * * *"
    statement => "SELECT * from youtube_videos where id > :sql_last_value order by id asc limit 10000"
    use_column_value => true
    tracking_column => "id"
    last_run_metadata_path => "last_state/youtube_videos"
  }
}
output {
  elasticsearch {
    hosts => ["localhost:9200"]
    index => "news"
    id => "elasticsearch_plugin"
  }
}