Skip to content

Instantly share code, notes, and snippets.

@derickson
derickson / mongoToES.js
Created Jul 16, 2015
Example of NodeJS Loop of Mongo to Elasticsearch
View mongoToES.js
// npm install elasticsearch
// setup nodejs client for elasticsearch
// documentation: https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/index.html
var elasticsearch = require('elasticsearch');
var EsClient = new elasticsearch.Client({
host: 'localhost:9200',
log: 'info'
});
View not_indexing_fields
DELETE /storedvsunstoredtest
## notice I disable the _all text index and the index: no setting on field2
PUT /storedvsunstoredtest
{
"settings": {
"number_of_replicas": 0,
"number_of_shards": 3
},
"mappings": {
View gridfsBinaryTest.py
## import the necessary libraries
from io import BytesIO
from pymongo import MongoClient
import gridfs
import random
from math import ceil
## delete all versions of the file
def deleteFile(db, fs, name):
oldFile = db['fs.files'].find_one({"filename":name})
@derickson
derickson / stddev.js
Created Mar 11, 2014
standard deviation MongoDB aggregation pipline using method from http://en.wikipedia.org/wiki/Standard_deviation#Rapid_calculation_methods
View stddev.js
// standard deviation MongoDB aggregation pipline
// using method from http://en.wikipedia.org/wiki/Standard_deviation#Rapid_calculation_methods
var parts = db.stat.aggregate( { $group: { _id: null,
s0: {$sum:1},
s1:{$sum:"$stat"},
s2:{$sum:{$multiply:["$stat","$stat"]}} }} ).result[0];
var stddev = Math.sqrt( parts.s0 * parts.s2 - Math.pow( parts.s1, 2) ) / parts.s0
@derickson
derickson / splitHelper.js
Last active Dec 24, 2015
YCSB splittin code for mongodb
View splitHelper.js
var ycsbUtil = {};
ycsbUtil.dropAndSplitYCSB = function (shardCount, chunksPerShard) {
db.getMongo().getDB( 'ycsb' ).dropDatabase();
db.adminCommand( { "enablesharding" : "ycsb" } ) ;
db.adminCommand( { "shardcollection" : "ycsb.usertable", "key" : { "_id" : 1 } } ) ;
var splitCount = shardCount * chunksPerShard;
var splitSize = Math.ceil( 10000 / splitCount; )
for(var i=1; i < splitCount; ++i){
@derickson
derickson / rtw.xqy
Last active Dec 17, 2015
Example of a simple XQuery recursive typeswitch for simple data conversion
View rtw.xqy
xquery version "1.0-ml";
(:~
@author derickson
@date 2013-05-22
Example of a Recursive Typeswitch for simple data conversion
Works in MarkLogic Server 6+
:)
@derickson
derickson / kml.xqy
Created May 2, 2013
geo code for guest blog post
View kml.xqy
xquery version "1.0-ml";
declare namespace ns = "http://www.marklogic.com/MLU/logbook";
declare variable $bbox as xs:string? := xdmp:get-request-field("BBOX", ());
xdmp:set-response-content-type("text/kml; charset=utf-8"),
<kml xmlns="http://www.opengis.net/kml/2.2" xmlns:gx="http://www.google.com/kml/ext/2.2">
<Document>
{
View insertFence.xqy
xquery version "1.0-ml";
import module namespace mkf = "http://derickson/kmlalert/model/m-kf" at "/model/m-kmlfence.xqy";
import module namespace lk = "http://derickson/kmlalert/lib/kml" at "/lib/l-kml.xqy";
declare namespace kml ="http://www.opengis.net/kml/2.2";
declare namespace gx ="http://www.google.com/kml/ext/2.2";
mkf:insert-fence(
lk:get-google-map("https://maps.google.com/maps/ms?msid=2041XXXXXX Your URL GOES HERE")
)
View dehex.xqy
xquery version("1.0-ml");
let $d := fn:doc("/wireshark.xml")
for $p in $d//packet
let $hex := fn:string($p//proto[@name eq "fake-field-wrapper"]/field[@name eq "data"]/@value)
let $len := fn:string-length($hex)
return
fn:string-join((
for $i in (1 to $len)
@derickson
derickson / color-cycle.xqy
Last active Dec 12, 2015
Cycle light colors
View color-cycle.xqy
xquery version "1.0-ml";
import module namespace lh = "http://derickson/lib/l-mlhue"
at "/lib/l-mlhue.xqy";
for $i in (1 to 10)
for $state in (
$lh:RED, $lh:MAGENTA, $lh:PURPLE,
$lh:BLUE, $lh:GREEN, $lh:YELLOW,
$lh:ORANGE
You can’t perform that action at this time.