Skip to content

Instantly share code, notes, and snippets.

@derickson
derickson / mongoToES.js
Created Jul 16, 2015
Example of NodeJS Loop of Mongo to Elasticsearch
View mongoToES.js
// npm install elasticsearch
// setup nodejs client for elasticsearch
// documentation: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/index.html
var elasticsearch = require('elasticsearch');
var EsClient = new elasticsearch.Client({
host: 'localhost:9200',
log: 'info'
});
@derickson
derickson / logstash-bikehare-status.conf
Last active Feb 13, 2019
Logstash config for http_poller pulling of DC Capital bikeshare data and doing an XML split of the contents.
View logstash-bikehare-status.conf
## Example of pulling data from DC Capital bikeshare to Elasticsearch in real time
## HTTP Poller -> XML Splitting -> Elasticsearch
input {
## pull data from Capital Bikeshare every 60 seconds
http_poller {
urls => {
bikeshare_dc => "https://www.capitalbikeshare.com/data/stations/bikeStations.xml"
}
request_timeout => 30
View elasticsearch_sql.json
## 1) Pull all records from index named fedeastsql
POST /_xpack/sql?format=txt
{
"query": "SELECT * FROM fedeastsql"
}
## 2) List all columns and datatype of this index
View ingestBeer.py
#!/bin/python3
import json
import moment
from elasticsearch import helpers
from elasticsearch import Elasticsearch
esConnString = 'http://elastic:changeme@localhost:9200'
esTo = Elasticsearch([esConnString],request_timeout=100)
View abac with covery query proposal
PUT my_index
{
"mappings": {
"properties": {
"securityTags": {
"type": "keyword",
"fields": {
"length": {
"type": "token_count",
"analyzer": "standard"
View combinatorialSet.py
#!/usr/bin/python
# -*- coding: utf-8 -*-
import json
def prettyPrint(doc):
print(json.dumps(doc, indent=4, sort_keys=True))
View bigBool.json
{
"bool": {
"should": [
[
{
"bool": {
"must": [
[
{
"term": {
View watcher examples .txt
### EXAMPLE SEARCH: Has Spotify run in the last 30 seconds?
GET /topbeat-*/_search
{
"query": {
"bool": {
"must": [
{"term": {
"proc.name": {
"value": "Spotify"
@derickson
derickson / spectator.html
Last active Aug 26, 2016
A-Frame spectator camera example code
View spectator.html
<html>
<head>
<script src="../dist/aframe.min.js"></script>
<script>
AFRAME.registerComponent('spectator',{
'schema': {
canvas: {
type: 'string',
default: ''
@derickson
derickson / splitHelper.js
Last active Dec 24, 2015
YCSB splittin code for mongodb
View splitHelper.js
var ycsbUtil = {};
ycsbUtil.dropAndSplitYCSB = function (shardCount, chunksPerShard) {
db.getMongo().getDB( 'ycsb' ).dropDatabase();
db.adminCommand( { "enablesharding" : "ycsb" } ) ;
db.adminCommand( { "shardcollection" : "ycsb.usertable", "key" : { "_id" : 1 } } ) ;
var splitCount = shardCount * chunksPerShard;
var splitSize = Math.ceil( 10000 / splitCount; )
for(var i=1; i < splitCount; ++i){
You can’t perform that action at this time.