Atomic

db.q

\c 30 300

/ KDB built-in types

/ default values
STORAGE_VALUES:(0b;0Ng;0x00;0h;0i;0;0e;0.0;" ";`;0Np;2000.01m;2000.01.01;2000.01.01T00:00:00.000;0Nn;00:00;00:00:00;00:00:00.000;0)
STORAGE_VALUES,:enlist each STORAGE_VALUES

/ char aliases
STORAGE_CHARS:"bgxhijefcspmdznuvtr" / scalar char aliases
STORAGE_CHARS,:upper STORAGE_CHARS / list char aliases

/ friendly names
STORAGE_NAMES:`boolean`guid`byte`short`int`long`real`float`char`symbol`timestamp`month`date`datetime`timespan`minute`second`time`ref
STORAGE_NAMES,:{`$,[;"Vector"] each string x} STORAGE_NAMES
STORAGE_TYPES:type each STORAGE_VALUES

attrsSystem:{[attrEntityId]
  / in-memory table that contains the definition of the core built-in attributes in the database
  :flip`entityId`entityDt`entityName`attrType`attrIndex`attrFullText`entityDoc!flip
    ((0N;attrEntityId;`entityDt;"r";1b;0b;"This entity's datatype. Every object in the system gets its id from this table. Every object has a type");
    (0N;attrEntityId;`entityName;"s";1b;0b;"Unique identifier among all entities in the system. Used as a symbolic entity Id");
    (0N;attrEntityId;`attrType;"c";1b;0b;"kdb+ storage type char; see http://code.kx.com/wiki/Reference/Datatypes");
    (0N;attrEntityId;`attrIndex;"b";1b;0b;"1b if designer expects to filter or sort on this attribute; 0b if not. Defaults to 1b");
    (0N;attrEntityId;`attrFullText;"b";1b;0b;"1b if attribute value should be indexed for full-text search; 0b if not. Defaults to 1b");
    (0N;attrEntityId;`entityDoc;"C";1b;0b;"Documentation string");
     / First-class transactions
    (0N;attrEntityId;`txTimestamp;"p";1b;0b;"The timestamp associated with this transaction.");
     / Datatype meta-model
    (0N;attrEntityId;`dtNamespace;"s";1b;0b;"Datatype's namespace");
    (0N;attrEntityId;`dtName;"C";1b;0b;"Datatype's local name. Names within a given namespace are unique");
    (0N;attrEntityId;`dtParent;"r";1b;0b;"Datatype's parent datatype. Only populated in datatype hierarchies");
    (0N;attrEntityId;`dtList;"r";1b;0b;"Datatype's list datatype");
    (0N;attrEntityId;`dtComponent;"r";1b;0b;"Datatype's component datatype");
    (0N;attrEntityId;`dtFields;"R";1b;0b;"Datatype's fields");
     / Security model
    (0N;attrEntityId;`userName;"s";1b;0b;"User");
    (0N;attrEntityId;`groupId;"r";1b;0b;"Group");
    (0N;attrEntityId;`roleName;"s";1b;0b;"Role name. Unique across the system");
    (0N;attrEntityId;`roleMapEntryRole;"r";1b;0b;"Reference to a RoleMap entity");
    (0N;attrEntityId;`roleMapEntryUsers;"R";1b;0b;"References to User entities");
    (0N;attrEntityId;`roleMapEntryGroups;"R";1b;0b;"References to Group entities");
    (0N;attrEntityId;`roleMapEntries;"R";1b;0b;"References to RoleMapEntry entities"))}

/ Core Datatypes: Datatype + Attribute + User + Group + Role + RoleMapEntry + RoleMap
dtsSystem:{[attrEntityId;dtEntityId]
  / in-memory table that provides fast access to all datatype definitions in the database
  namespace:`$"http://www.appian.com/ae/types/2009";
  :flip`entityId`entityDt`entityName`dtNamespace`dtName`dtParent`dtFields`entityDoc!flip
    ((dtEntityId;dtEntityId;`dtDt;namespace;"Datatype";0N;`entityDt`dtNamespace`dtName`dtParent`dtList`dtComponent`dtFields;"The datatype for datatypes");
     (attrEntityId;dtEntityId;`attrDt;namespace;"Attribute";0N;`entityDt`entityName`attrType`attrIndex`attrFullText;"The attribute datatype");
     (0N;dtEntityId;`userDt;namespace;"User";0N;enlist`userName;"The user datatype");
     (0N;dtEntityId;`groupDt;namespace;"Group";0N;enlist`groupId;"The group datatype");
     (0N;dtEntityId;`roleDt;namespace;"Role";0N;enlist`roleName;"The role datatype");
     (0N;dtEntityId;`roleMapEntryDt;namespace;"RoleMapEntry";0N;`roleMapEntryRole`roleMapEntryUsers`roleMapEntryGroups;"The rolemap entry datatype");
     (0N;dtEntityId;`roleMapDt;namespace;"RoleMap";0N;enlist`roleMapEntries;"The rolemap datatype"))}

initHdb:{[]
  / start at entity & transaction 1
  nextEntityId:1; nextTxId:1;
  / negative numbers are temp ids used to specify relationships between entities before those entities are persisted
  attrEntityId:-1; dtEntityId:-2;

  / build core attributes + datatypes
  A:attrsSystem[attrEntityId];
  D:dtsSystem[attrEntityId;dtEntityId];

  / allocate & assign real entity ids to data
  dataIds:(A`entityId),D`entityId;
  allocation:allocateNewEntityIds[nextEntityId;dataIds];
  eIds:value idMappings:allocation`idMappings;
  A:update entityId:(count A)#eIds,entityDt:idMappings[entityDt] from A;
  D:update entityId:(count A)_eIds,entityDt:idMappings[entityDt] from D;
  / overrides symbolic ids with entity ids
  D[`dtFields]:A[`entityId]@(A`entityName)?D`dtFields;

  / create EVT tables
  data:joinTables[A;D];
  evtTables:toEvtTables[data;nextTxId];

  nextId:allocation`nextId;
  / keyed access by entityName + O(1) lookup by entityId
  A:update `u#entityId from `entityName xkey A;
  D:update `u#entityId from `entityName xkey D;

  :`idMappings`nextId`nextTxId`newAttributes`newDatatypes`tables!(idMappings;nextId;nextTxId+1;A;D;evtTables)}

joinTables:{ / needed when x & y are tables with different schemas; result is a list of dictionaries
  countX:count x;
  countY:count y;
  joined:(countX+countY)#();
  joined[til countX]:x;
  joined[countX+til countY]:y;
  :joined}

toEvtTables:{[data;nextTxId]
  / figure out ALL distinct attribute names being written to
  k:targetAttrs[data];
  / mask of where each attribute appears
  w:wTargetAttrs[data; k];
  eIds:data[;`entityId];
  / get ids by attribute
  e:eIds@w;
  / get values by attribute
  v:(flip data[;k])@'w;
  / create in-memory EV tables keyed by attribute name
  :k!{[e;v;t;w]flip`e`v`t!(e[w];v[w];t)}[e;v;nextTxId] each til count k}

targetAttrs:{[data]:(distinct raze key each data) except `entityId}
wTargetAttrs:{[data;targetAttrs]:where each (count each data)>/:(key each data)?\:/:targetAttrs}

allocateNewEntityIds:{[nextEntityId;dataIds]
  / verify that ids being referenced are valid
  eIds:(count dataIds)#0N; / initialize ids
  mNull:null dataIds; / find nulls i.e. new entities to be created
  wNull:where mNull;
  mNotNull:not mNull; / note: m -> mask
  wNotNull:where mNotNull; / explicitly specified ids, can be negative (for tmp ids) or positive (for existing ids)
  listToString:{:trim raze" ",/:string x};
  if[any not nextEntityId>dataIds@wNotNull; '"Invalid id in (",listToString[dataIds@wNotNull],"). Cannot be greater than nextEntityId (",string[nextEntityId],")"];

  / allocate new ids
  eIds[wNull]:nextEntityId+til count wNull;
  nextEntityId+:count wNull;

  / allocate temporary ids
  wTmp:where mNotNull&dataIds<0;
  tmpIds:distinct dataIds@wTmp;
  tmpIdsMapping:tmpIds!nextEntityId+til count tmpIds;
  eIds[wTmp]:nextEntityId+til count wTmp;
  nextEntityId+:count wTmp;
  eIds[wTmp]:tmpIdsMapping@dataIds[wTmp];
  :`nextId`idMappings!(nextEntityId;dataIds!eIds)}

loadTpcchSchema:{[nextEntityId;nextTxId]

  data:.j.k each read0 `:tpcch.json;

  / allocate & assign real entity ids to data
  dataIds:data[;`entityId];
  allocation:allocateNewEntityIds[nextEntityId;dataIds];
  eIds:value idMappings:allocation`idMappings;
  data[;`entityId]:eIds;

  k:targetAttrs[data];  w:wTargetAttrs[data; k];
  t:k!w; / entityName to indices where those appear in data
  / attrType is used to cast/validate data being received
  attrTypes:DB[`newAttributes;([] entityName:k);`attrType];
  0N!STORAGE_NAMES[STORAGE_CHARS?attrTypes]; / this is just for info purposes
  / TODO1: cast all fields of type r and R to symbols, then look their ids against the DB[`tables;`entityName] table
  / TODO2: cast all numeric fields to exact numberic type
  break;

  evtTables:toEvtTables[data;nextTxId];
  A:data@wA:where{`attrType in key x} each data; / attributes
  D:data@wD:where{`dtName in key x} each data; / datatypes

  nextId:allocation`nextId;
  / keyed access by entityName + O(1) lookup by entityId
  A:update `u#entityId from `entityName xkey A;
  D:update `u#entityId from `entityName xkey D;

  :`idMappings`nextId`nextTxId`newAttributes`newDatatypes`tables!(idMappings;nextId;nextTxId+1;A;D;evtTables)}

DB:0N!initHdb[];
TPC:0N!loadTpcchSchema[DB`nextId;DB`nextTxId];

\
attrsTpch:{[]
  :flip`entityId`entityDt`entityName`attrType`attrIndex`attrFullText`entityDoc!flip
    ((0N;`attrDt;`s_name;"C";1b;0b;"Supplier Name");
    (0N;`attrDt;`s_nationkey;"r";1b;0b;"Supplier Nation Key");
    (0N;`attrDt;`n_name;"C";1b;0b;"Nation Name");
    (0N;`attrDt;`n_regionkey;"r";1b;0b;"Nation Region Key");
    (0N;`attrDt;`r_name;"C";1b;0b;"Region Name"))}

dtsTpch:{[]
  namespace:`tpch;
  :flip`entityId`entityDt`entityName`dtNamespace`dtName`dtParent`dtFields`entityDoc!flip
    ((0N;`dtDt;`SUPPLIER;namespace;"Supplier";0N;`s_name`s_nationkey;"The Supplier datatype");
    (0N;`dtDt;`NATION;namespace;"Nation";0N;`n_name`n_regionkey;"The Nation datatype");
    (0N;`dtDt;`REGION;namespace;"Region";0N;enlist `r_name;"The Region datatype"))}