simple collection schema

_.md

[ Launch: simple collection schema ] 6347614 by enjalot

config.json

{"description":"simple collection schema","endpoint":"","display":"div","public":true,"require":[],"fileconfigs":{"inlet.js":{"default":true,"vim":false,"emacs":false,"fontSize":12},"test.json":{"default":true,"vim":false,"emacs":false,"fontSize":12},"console.js":{"default":true,"vim":false,"emacs":false,"fontSize":12},"_.md":{"default":true,"vim":false,"emacs":false,"fontSize":12},"config.json":{"default":true,"vim":false,"emacs":false,"fontSize":12}},"fullscreen":false,"play":false,"loop":false,"restart":false,"autoinit":true,"pause":true,"loop_type":"period","bv":false,"nclones":15,"clone_opacity":0.4,"duration":3000,"ease":"linear","dt":0.01,"thumbnail":"http://i.imgur.com/mzu3ays.png"}

console.js

var display = d3.select("#display");
display.style("overflow", "scroll")

tributary.console = function() {
  var str = "";
  for(var i = 0; i < arguments.length; i++) {
    str += JSON.stringify(arguments[i], null, 2) + " ";
  }
  var span = display.append("span").text(str)
  span.append("br")
  return span
}

inlet.js

var data = tributary.test;

var schemas = data.map(function(datum) {
  var walked = walkObject(datum)
  tributary.console(walked)
  console.log("walked", walked)
  return walked;
})

tributary.console()
var counts = countSchemas(schemas);

var cspan = tributary.console(counts);
cspan.style("white-space", "pre-wrap")

function countSchemas(schemas) {
  var counts = {key: "counts", count: schemas.length, children: []};
  //TODO: use crossfilter? performance
  var schema;
  for(var i = 0, l = schemas.length; i < l; i++) {
    schema = schemas[i];
    
    //loop thru the keys, increment counts
    var keys = _.keys(schema);
    keys.forEach(function(key) {
      //TODO: break up the object paths (foo.bar.baz) and do their counts
      var ind = indexOfFn(counts.children, function(c) { return c.key === key });
      if(~ind) {
        counts.children[ind].count++;
      } else {
        counts.children.push({ key: key, count: 1 });
      }
    })
  }
  counts.children.sort(function(a,b) { return b.count - a.count });
  return counts;
}

//recursively walk down an object until we have key,type pairs
//that are groupable
function walkObject(obj, res, path) {
  if(!res) res = {};
  var keys = _.keys(obj);
  keys.forEach(function(key) {
    var kpath;
    if(!path) { kpath = key; }
    else { kpath = path + "." + key; }

    var val = obj[key];
    var type = checkType(val);
    set(res, kpath, {$type: type })
    if(type === "object") {
      walkObject(val, res, kpath);
    }
    //} else {

    //res[key] = { type: type };
    //}
  })
  return res;
}


function walkObjectOLD(obj, res, path) {
  if(!res) res = {};
  var keys = _.keys(obj);
  keys.forEach(function(key) {
    var kpath;
    if(!path) { kpath = key; }
    else { kpath = path + "." + key; }

    var val = obj[key];
    var type = checkType(val);
    if(type === "object") {
      walkObject(val, res, kpath);
    } else {
      res[kpath] = type;
    }
  })
  return res;
}

function checkType(value) {
  if(value.length >=0 && typeof(value) !== "string") {
    return "array";
  }
  return typeof(value);
}

function indexOfFn (xs, fn) {
  for (var i = 0, l = xs.length; i < l; i++) {
    if (fn(xs[i])) return i;
  }
  return -1;
}

function set(obj, path, value) {
  var keys = path.split(".");
  console.log("path", path)
  var val = obj;
  for(var i = 0; i < keys.length; i++) {
    var key = keys[i];
    
    if(!val[key]) val[key] = {};
    if(i === keys.length - 1) {
      console.log("last key", key, val, value)
      val[key] = value;
    }
    val = val[key]

  }
}

test.json

[
  { "_id": "a", "name": "ian", "age": 27 },
  { "_id": "b", "name": "sarah" },
  { "_id": "c", "name": "andreas", "cool": "yes" },
  { "_id": "d", "name": "brian", "boss": true },
  { "_id": "e", "name": "nate", "skills": ["everything", "javascript"] },
  { "_id": "f", "name": "randal", "product": { "word": "bond" } },
  { "_id": "d", "name": "joseph", "skills": ["OT"] },
  { "_id": "f", "name": "eric", "product": { "mack": "daddy" } }
]