David Erickson derickson
derickson
/ logstash-bikehare-status.conf
Last active
March 10, 2021 14:26
Logstash config for http_poller pulling of DC Capital bikeshare data and doing an XML split of the contents.
View logstash-bikehare-status.conf
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ## Example of pulling data from DC Capital bikeshare to Elasticsearch in real time | |
| ## HTTP Poller -> XML Splitting -> Elasticsearch | |
| input { | |
| ## pull data from Capital Bikeshare every 60 seconds | |
| http_poller { | |
| urls => { | |
| bikeshare_dc => "https://www.capitalbikeshare.com/data/stations/bikeStations.xml" | |
| } | |
| request_timeout => 30 |
derickson
/ mongoToES.js
Created
July 16, 2015 15:18
Example of NodeJS Loop of Mongo to Elasticsearch
View mongoToES.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // npm install elasticsearch | |
| // setup nodejs client for elasticsearch | |
| // documentation: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/index.html | |
| var elasticsearch = require('elasticsearch'); | |
| var EsClient = new elasticsearch.Client({ | |
| host: 'localhost:9200', | |
| log: 'info' | |
| }); |
derickson
/ elasticsearch_sql.json
Created
August 7, 2018 18:46
View elasticsearch_sql.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ## 1) Pull all records from index named fedeastsql | |
| POST /_xpack/sql?format=txt | |
| { | |
| "query": "SELECT * FROM fedeastsql" | |
| } | |
| ## 2) List all columns and datatype of this index |
derickson
/ ingestBeer.py
Last active
July 18, 2018 13:58
View ingestBeer.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/python3 | |
| import json | |
| import moment | |
| from elasticsearch import helpers | |
| from elasticsearch import Elasticsearch | |
| esConnString = 'http://elastic:changeme@localhost:9200' | |
| esTo = Elasticsearch([esConnString],request_timeout=100) |
derickson
/ abac with covery query proposal
Created
October 11, 2017 16:55
View abac with covery query proposal
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| PUT my_index | |
| { | |
| "mappings": { | |
| "properties": { | |
| "securityTags": { | |
| "type": "keyword", | |
| "fields": { | |
| "length": { | |
| "type": "token_count", | |
| "analyzer": "standard" |
derickson
/ combinatorialSet.py
Created
August 2, 2017 14:36
View combinatorialSet.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/python | |
| # -*- coding: utf-8 -*- | |
| import json | |
| def prettyPrint(doc): | |
| print(json.dumps(doc, indent=4, sort_keys=True)) | |
derickson
/ bigBool.json
Created
August 2, 2017 14:28
View bigBool.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "bool": { | |
| "should": [ | |
| [ | |
| { | |
| "bool": { | |
| "must": [ | |
| [ | |
| { | |
| "term": { |
derickson
/ watcher examples .txt
Last active
September 1, 2016 15:03
View watcher examples .txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ### EXAMPLE SEARCH: Has Spotify run in the last 30 seconds? | |
| GET /topbeat-*/_search | |
| { | |
| "query": { | |
| "bool": { | |
| "must": [ | |
| {"term": { | |
| "proc.name": { | |
| "value": "Spotify" |
View spectator.html
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <html> | |
| <head> | |
| <script src="../dist/aframe.min.js"></script> | |
| <script> | |
| AFRAME.registerComponent('spectator',{ | |
| 'schema': { | |
| canvas: { | |
| type: 'string', | |
| default: '' |
View splitHelper.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| var ycsbUtil = {}; | |
| ycsbUtil.dropAndSplitYCSB = function (shardCount, chunksPerShard) { | |
| db.getMongo().getDB( 'ycsb' ).dropDatabase(); | |
| db.adminCommand( { "enablesharding" : "ycsb" } ) ; | |
| db.adminCommand( { "shardcollection" : "ycsb.usertable", "key" : { "_id" : 1 } } ) ; | |
| var splitCount = shardCount * chunksPerShard; | |
| var splitSize = Math.ceil( 10000 / splitCount; ) | |
| for(var i=1; i < splitCount; ++i){ |
NewerOlder