Skip to content

Instantly share code, notes, and snippets.

@dongyuwei
Last active May 15, 2019 09:22
Show Gist options
  • Save dongyuwei/21fd7c4e843beae7f70224dd10a4085d to your computer and use it in GitHub Desktop.
Save dongyuwei/21fd7c4e843beae7f70224dd10a4085d to your computer and use it in GitHub Desktop.
flatbuffer for english dictionary

flatbuffer Scheme: dictionary.fbs

table Word {
  frequency:long;
  ipa:string;
  translation:[string];
}


table Dictionary {
    keys:[string];
    values:[Word];
}

root_type Dictionary;

json data: dictionary.json

{
  "keys": ["downbeat"],
  "values": [
    {
      "frequency": 236548,
      "translation": [
        "n. 下拍(乐队指挥向下的手势);停滞",
        "adj. 忧郁的;悲观的;不强烈的"
      ],
      "ipa": "daʊnˈbit"
    }
  ]
}

generate js code and binary data from scheme and json text:

cd flatbuffers/samples && ../Debug/flatc --js dictionary.fbs --binary dictionary.json

@dongyuwei
Copy link
Author

some inspiration: google/flatbuffers#4289

@dongyuwei
Copy link
Author

dictionary_generated.js

// automatically generated by the FlatBuffers compiler, do not modify

/**
 * @constructor
 */
function Word() {
  /**
   * @type {flatbuffers.ByteBuffer}
   */
  this.bb = null;

  /**
   * @type {number}
   */
  this.bb_pos = 0;
}

/**
 * @param {number} i
 * @param {flatbuffers.ByteBuffer} bb
 * @returns {Word}
 */
Word.prototype.__init = function(i, bb) {
  this.bb_pos = i;
  this.bb = bb;
  return this;
};

/**
 * @param {flatbuffers.ByteBuffer} bb
 * @param {Word=} obj
 * @returns {Word}
 */
Word.getRootAsWord = function(bb, obj) {
  return (obj || new Word).__init(bb.readInt32(bb.position()) + bb.position(), bb);
};

/**
 * @returns {flatbuffers.Long}
 */
Word.prototype.frequency = function() {
  var offset = this.bb.__offset(this.bb_pos, 4);
  return offset ? this.bb.readInt64(this.bb_pos + offset) : this.bb.createLong(0, 0);
};

/**
 * @param {flatbuffers.Encoding=} optionalEncoding
 * @returns {string|Uint8Array|null}
 */
Word.prototype.ipa = function(optionalEncoding) {
  var offset = this.bb.__offset(this.bb_pos, 6);
  return offset ? this.bb.__string(this.bb_pos + offset, optionalEncoding) : null;
};

/**
 * @param {number} index
 * @param {flatbuffers.Encoding=} optionalEncoding
 * @returns {string|Uint8Array}
 */
Word.prototype.translation = function(index, optionalEncoding) {
  var offset = this.bb.__offset(this.bb_pos, 8);
  return offset ? this.bb.__string(this.bb.__vector(this.bb_pos + offset) + index * 4, optionalEncoding) : null;
};

/**
 * @returns {number}
 */
Word.prototype.translationLength = function() {
  var offset = this.bb.__offset(this.bb_pos, 8);
  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
};

/**
 * @param {flatbuffers.Builder} builder
 */
Word.startWord = function(builder) {
  builder.startObject(3);
};

/**
 * @param {flatbuffers.Builder} builder
 * @param {flatbuffers.Long} frequency
 */
Word.addFrequency = function(builder, frequency) {
  builder.addFieldInt64(0, frequency, builder.createLong(0, 0));
};

/**
 * @param {flatbuffers.Builder} builder
 * @param {flatbuffers.Offset} ipaOffset
 */
Word.addIpa = function(builder, ipaOffset) {
  builder.addFieldOffset(1, ipaOffset, 0);
};

/**
 * @param {flatbuffers.Builder} builder
 * @param {flatbuffers.Offset} translationOffset
 */
Word.addTranslation = function(builder, translationOffset) {
  builder.addFieldOffset(2, translationOffset, 0);
};

/**
 * @param {flatbuffers.Builder} builder
 * @param {Array.<flatbuffers.Offset>} data
 * @returns {flatbuffers.Offset}
 */
Word.createTranslationVector = function(builder, data) {
  builder.startVector(4, data.length, 4);
  for (var i = data.length - 1; i >= 0; i--) {
    builder.addOffset(data[i]);
  }
  return builder.endVector();
};

/**
 * @param {flatbuffers.Builder} builder
 * @param {number} numElems
 */
Word.startTranslationVector = function(builder, numElems) {
  builder.startVector(4, numElems, 4);
};

/**
 * @param {flatbuffers.Builder} builder
 * @returns {flatbuffers.Offset}
 */
Word.endWord = function(builder) {
  var offset = builder.endObject();
  return offset;
};

/**
 * @constructor
 */
function Dictionary() {
  /**
   * @type {flatbuffers.ByteBuffer}
   */
  this.bb = null;

  /**
   * @type {number}
   */
  this.bb_pos = 0;
}

/**
 * @param {number} i
 * @param {flatbuffers.ByteBuffer} bb
 * @returns {Dictionary}
 */
Dictionary.prototype.__init = function(i, bb) {
  this.bb_pos = i;
  this.bb = bb;
  return this;
};

/**
 * @param {flatbuffers.ByteBuffer} bb
 * @param {Dictionary=} obj
 * @returns {Dictionary}
 */
Dictionary.getRootAsDictionary = function(bb, obj) {
  return (obj || new Dictionary).__init(bb.readInt32(bb.position()) + bb.position(), bb);
};

/**
 * @param {number} index
 * @param {flatbuffers.Encoding=} optionalEncoding
 * @returns {string|Uint8Array}
 */
Dictionary.prototype.keys = function(index, optionalEncoding) {
  var offset = this.bb.__offset(this.bb_pos, 4);
  return offset ? this.bb.__string(this.bb.__vector(this.bb_pos + offset) + index * 4, optionalEncoding) : null;
};

/**
 * @returns {number}
 */
Dictionary.prototype.keysLength = function() {
  var offset = this.bb.__offset(this.bb_pos, 4);
  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
};

/**
 * @param {number} index
 * @param {Word=} obj
 * @returns {Word}
 */
Dictionary.prototype.values = function(index, obj) {
  var offset = this.bb.__offset(this.bb_pos, 6);
  return offset ? (obj || new Word).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
};

/**
 * @returns {number}
 */
Dictionary.prototype.valuesLength = function() {
  var offset = this.bb.__offset(this.bb_pos, 6);
  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
};

/**
 * @param {flatbuffers.Builder} builder
 */
Dictionary.startDictionary = function(builder) {
  builder.startObject(2);
};

/**
 * @param {flatbuffers.Builder} builder
 * @param {flatbuffers.Offset} keysOffset
 */
Dictionary.addKeys = function(builder, keysOffset) {
  builder.addFieldOffset(0, keysOffset, 0);
};

/**
 * @param {flatbuffers.Builder} builder
 * @param {Array.<flatbuffers.Offset>} data
 * @returns {flatbuffers.Offset}
 */
Dictionary.createKeysVector = function(builder, data) {
  builder.startVector(4, data.length, 4);
  for (var i = data.length - 1; i >= 0; i--) {
    builder.addOffset(data[i]);
  }
  return builder.endVector();
};

/**
 * @param {flatbuffers.Builder} builder
 * @param {number} numElems
 */
Dictionary.startKeysVector = function(builder, numElems) {
  builder.startVector(4, numElems, 4);
};

/**
 * @param {flatbuffers.Builder} builder
 * @param {flatbuffers.Offset} valuesOffset
 */
Dictionary.addValues = function(builder, valuesOffset) {
  builder.addFieldOffset(1, valuesOffset, 0);
};

/**
 * @param {flatbuffers.Builder} builder
 * @param {Array.<flatbuffers.Offset>} data
 * @returns {flatbuffers.Offset}
 */
Dictionary.createValuesVector = function(builder, data) {
  builder.startVector(4, data.length, 4);
  for (var i = data.length - 1; i >= 0; i--) {
    builder.addOffset(data[i]);
  }
  return builder.endVector();
};

/**
 * @param {flatbuffers.Builder} builder
 * @param {number} numElems
 */
Dictionary.startValuesVector = function(builder, numElems) {
  builder.startVector(4, numElems, 4);
};

/**
 * @param {flatbuffers.Builder} builder
 * @returns {flatbuffers.Offset}
 */
Dictionary.endDictionary = function(builder) {
  var offset = builder.endObject();
  return offset;
};

/**
 * @param {flatbuffers.Builder} builder
 * @param {flatbuffers.Offset} offset
 */
Dictionary.finishDictionaryBuffer = function(builder, offset) {
  builder.finish(offset);
};

// Exports for Node.js and RequireJS
this.Word = Word;
this.Dictionary = Dictionary;

@dongyuwei
Copy link
Author

deserialize binary file and get the dictionary data :samplebinary2.js

var assert = require("assert");
var flatbuffers = require("../js/flatbuffers").flatbuffers;
var { Dictionary, Word } = require("./dictionary_generated.js");
var fs = require("fs");

function main() {
  var data = new Uint8Array(fs.readFileSync("./dictionary.bin"));
  var buf = new flatbuffers.ByteBuffer(data);

  var dict = Dictionary.getRootAsDictionary(buf);
  console.log("dict.keysLength", dict.keysLength());
  var word = dict.values(0);
  console.log(
    "word",
    word.ipa(),
    word.translationLength(),
    word.translation(0),
    word.translation(1)
  );

  console.log("dict.valuesLength", dict.valuesLength());
  console.log("The FlatBuffer was successfully created and verified!");
}

main();

@dongyuwei
Copy link
Author

@dongyuwei
Copy link
Author

flatc -b myschema.fbs mydata.json
This will generate the binary file mydata_wire.bin which can be loaded as before.
https://github.com/google/flatbuffers/blob/master/docs/source/CppUsage.md

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment