David Erickson derickson
derickson
/ logstash-bikehare-status.conf
Last active Mar 10, 2021
Logstash config for http_poller pulling of DC Capital bikeshare data and doing an XML split of the contents.
View logstash-bikehare-status.conf
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ## Example of pulling data from DC Capital bikeshare to Elasticsearch in real time | |
| ## HTTP Poller -> XML Splitting -> Elasticsearch | |
| input { | |
| ## pull data from Capital Bikeshare every 60 seconds | |
| http_poller { | |
| urls => { | |
| bikeshare_dc => "https://www.capitalbikeshare.com/data/stations/bikeStations.xml" | |
| } | |
| request_timeout => 30 |
View mongoToES.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| // npm install elasticsearch | |
| // setup nodejs client for elasticsearch | |
| // documentation: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/index.html | |
| var elasticsearch = require('elasticsearch'); | |
| var EsClient = new elasticsearch.Client({ | |
| host: 'localhost:9200', | |
| log: 'info' | |
| }); |
derickson
/ elasticsearch_sql.json
Created Aug 7, 2018
View elasticsearch_sql.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ## 1) Pull all records from index named fedeastsql | |
| POST /_xpack/sql?format=txt | |
| { | |
| "query": "SELECT * FROM fedeastsql" | |
| } | |
| ## 2) List all columns and datatype of this index |
derickson
/ ingestBeer.py
Last active Jul 18, 2018
View ingestBeer.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/python3 | |
| import json | |
| import moment | |
| from elasticsearch import helpers | |
| from elasticsearch import Elasticsearch | |
| esConnString = 'http://elastic:changeme@localhost:9200' | |
| esTo = Elasticsearch([esConnString],request_timeout=100) |
View abac with covery query proposal
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| PUT my_index | |
| { | |
| "mappings": { | |
| "properties": { | |
| "securityTags": { | |
| "type": "keyword", | |
| "fields": { | |
| "length": { | |
| "type": "token_count", | |
| "analyzer": "standard" |
derickson
/ combinatorialSet.py
Created Aug 2, 2017
View combinatorialSet.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/usr/bin/python | |
| # -*- coding: utf-8 -*- | |
| import json | |
| def prettyPrint(doc): | |
| print(json.dumps(doc, indent=4, sort_keys=True)) | |
derickson
/ bigBool.json
Created Aug 2, 2017
View bigBool.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "bool": { | |
| "should": [ | |
| [ | |
| { | |
| "bool": { | |
| "must": [ | |
| [ | |
| { | |
| "term": { |
derickson
/ watcher examples .txt
Last active Sep 1, 2016
View watcher examples .txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| ### EXAMPLE SEARCH: Has Spotify run in the last 30 seconds? | |
| GET /topbeat-*/_search | |
| { | |
| "query": { | |
| "bool": { | |
| "must": [ | |
| {"term": { | |
| "proc.name": { | |
| "value": "Spotify" |
View spectator.html
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| <html> | |
| <head> | |
| <script src="../dist/aframe.min.js"></script> | |
| <script> | |
| AFRAME.registerComponent('spectator',{ | |
| 'schema': { | |
| canvas: { | |
| type: 'string', | |
| default: '' |
View splitHelper.js
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| var ycsbUtil = {}; | |
| ycsbUtil.dropAndSplitYCSB = function (shardCount, chunksPerShard) { | |
| db.getMongo().getDB( 'ycsb' ).dropDatabase(); | |
| db.adminCommand( { "enablesharding" : "ycsb" } ) ; | |
| db.adminCommand( { "shardcollection" : "ycsb.usertable", "key" : { "_id" : 1 } } ) ; | |
| var splitCount = shardCount * chunksPerShard; | |
| var splitSize = Math.ceil( 10000 / splitCount; ) | |
| for(var i=1; i < splitCount; ++i){ |
NewerOlder