MongoDB connector for Data Studio - aggregation cache (getSchema.gs)

getSchema.gs

// ==== getSchema ====
// https://developers.google.com/datastudio/connector/build#define_the_fields_with_getschema
// legacy response - https://developers.google.com/datastudio/connector/reference#getschema
var RECCOUNT = 'Record_Count';

function schemaFromSample(samples, _fields) {
  var names = [];
  var dataTypes = {};
  
  var solveType = function (doc, k, deep, kbase) {
    var _k = k.replace(/[.]/g, '__')
    if (kbase)
      _k = kbase + '__' + _k

    if (names.indexOf(_k) !== -1)
      return;
    
    var val = objResolve(doc, k.replace(/__/g, '.'))
    var t = typeof (val)

    if (['number', 'string', 'boolean'].indexOf(t) !== -1) {
      names.push(_k);
      dataTypes[_k] = t.toUpperCase();
      return;
    }
    
    if (deep && t == 'object' && val) {
      Object.keys(val).forEach(function(subk){
        solveType (val, subk, deep-1, _k)
      })
    }

  }

  var doc
  for (doc in samples) {
    doc = samples[doc];
    
    if (_fields && _fields.length) {
      _fields.forEach(function(k){ 
        var ak = k.replace(/[*]*$/, '')
        var adeep = k.length - ak.length

        if (!ak) {
          Object.keys(doc).forEach(function(k){ solveType(doc, k, adeep) })
          return
        }
          
        solveType(doc, ak, adeep)
      })
      continue;
    }

    Object.keys(doc).forEach(function(k){ solveType(doc, k, 1) })
  }

  if (_fields && _fields.length) {
    _fields.forEach(function(k){ 

      if (!/[*]$/.test(k)) {
        var _k = k.replace(/[.]/g, '__')

        if (names.indexOf(_k) !== -1)
          names.push(_k);
      }
    })
  }
  
  // https://developers.google.com/datastudio/connector/reference#field
  // https://developers.google.com/datastudio/connector/semantics#semantic-type-detection
  return (names).map(function(k) {
    var fld = {name: k, dataType: dataTypes[k] || 'STRING'}
    
    return fld;
  })
}

function schemaFromConfig (schemaFields) {
  return schemaFields.map(function(nt) {
    // split name:type
    var ntsegs = nt.split(':');
    
    var field = {name: ntsegs[0].replace(/[.]/g, '__'), dataType: (ntsegs[1] || 'STRING').toUpperCase()};
    
    return field;
  });
}

function refreshRawSchema(request, renew) {
  // get schema with cache
  var rconfig = request.configParams
  var cs = configService(rconfig);
  var schemaFields = cs.list('schemaFields')
  var schema = schemaFromConfig(schemaFields)
  
  if (schema.length && !schema.find(function(sch) { return (/[*]$/.test(sch.name)); }))
    return schema;

  var process = function () {
    // dynamic schema by fetching sample data
    // var pipeline = cs.pipeline(request.dateRange); 
    var pipeline = cs.pipeline(); 
    
    pipeline.push({$limit: 20}); // limit sample

    var samples = cs.fetchData(pipeline);
    
    return schemaFromSample(samples, schemaFields); 
  }
  
  var schemaCache = cs.cache('schema', {verbose: true});
  
  return schemaCache.race(process, {force: renew});
}

function refreshSchema (request, renew) {
  var schema = refreshRawSchema(request, renew);

  if (!schema.find(function(fld){ return fld.name == RECCOUNT}))
    schema.push({name: RECCOUNT, dataType: 'NUMBER'});

  return schema;
}

function getSchema(request) {

  return {schema: refreshSchema(request, true)};
}