derickson

// polling code adapted from: https://gist.github.com/benbjurstrom/00cdfdb24e39c59c124e812d5effa39a


// Settings for polling Gmail
const GMAIL_LABEL = "STATUSREPORT";
const PAGE_SIZE = 25;

// Settings for Elasticsearch
const ELASTICSEARCH_HOST = "https://xxxxxxxx.elastic.cloud:443";
const ELASTIC_API_KEY_BASE64 = "xxxxxxxx==";

derickson

## Install the required packages
## pip install -qU elasticsearch openai
import os
from elasticsearch import Elasticsearch
from openai import OpenAI
es_client = Elasticsearch(
    "undefined",
    api_key=os.environ["ES_API_KEY"]
)
      

derickson

## Example of pulling data from DC Capital bikeshare to Elasticsearch in real time
## HTTP Poller -> XML Splitting -> Elasticsearch

input {
  ## pull data from Capital Bikeshare every 60 seconds
  http_poller {
    urls => {
      bikeshare_dc => "https://www.capitalbikeshare.com/data/stations/bikeStations.xml"
    }
    request_timeout => 30

derickson

// npm install elasticsearch
// setup nodejs client for elasticsearch
// documentation: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/index.html
var elasticsearch = require('elasticsearch');
var EsClient = new elasticsearch.Client({
  host: 'localhost:9200',
  log: 'info'
});

derickson

## 1) Pull all records from index named fedeastsql

POST /_xpack/sql?format=txt
{
  "query": "SELECT * FROM fedeastsql"
}


## 2) List all columns and datatype of this index

derickson

#!/bin/python3

import json
import moment

from elasticsearch import helpers
from elasticsearch import Elasticsearch

esConnString = 'http://elastic:changeme@localhost:9200'
esTo = Elasticsearch([esConnString],request_timeout=100)

derickson

PUT my_index
{
    "mappings": {
        "properties": {
            "securityTags": { 
                "type": "keyword",
                "fields": {
                    "length": { 
                        "type":     "token_count",
                        "analyzer": "standard"

derickson

#!/usr/bin/python
# -*- coding: utf-8 -*-

import json


def prettyPrint(doc):
	print(json.dumps(doc, indent=4, sort_keys=True))

derickson

{
    "bool": {
        "should": [
            [
                {
                    "bool": {
                        "must": [
                            [
                                {
                                    "term": {

derickson

### EXAMPLE SEARCH: Has Spotify run in the last 30 seconds?
GET /topbeat-*/_search
{
  "query": {
    "bool": {
      "must": [
        {"term": {
          "proc.name": {
            "value": "Spotify"