JSONログから雑にデータ構造を調べるための糞スクリプト。同じキーで異なる型は想定外

json2bq.rb

# coding: utf-8
require 'json'

def get_bqtype(k, obj)
  return case obj
    when TrueClass, FalseClass then "BOOLEAN"
    when String then "STRING"
    when Float then "FLOAT"
    when Fixnum, Bignum 
      return "TIMESTAMP" if ["timestamp", "_ts"].include?(k)
      return "INTEGER"
    when Hash then "RECORD"
    when Array
      abort "Array not support"
    else "???"
  end
end
def seek(buf, obj)
  abort "obj is not Hash" if obj.class != Hash
  obj.each_pair do |k,v|
    item = {
      :name => k,
      :type => get_bqtype(k, v)
    }
    buf << item
    if v.class == Hash
      item[:fields] = []
      seek(item[:fields], v)
    end
  end
end

buf = []
obj = JSON.parse($stdin.read)
seek(buf, obj)

puts JSON.pretty_generate(buf)

merge_struct.rb

# encoding: utf-8

require 'json'

def merge(omap, smap)
  if !smap.instance_of?(Hash) || !omap.instance_of?(Hash)
    abort "instance is not Hash"
  end
  smap.each_pair do |k, v|
    if v.instance_of?(Hash)
      omap[k] = omap[k] || {}
      merge(omap[k], v)
    else
      if omap[k] == nil || omap[k].to_s == ""
        omap[k] = v
      end
    end
  end
end

map = {}
$stdin.each_line do |line|
  obj = JSON.load(line)
  merge(map, obj)
end

puts JSON.pretty_generate(map)

merge_struct2.rb

# encoding: utf-8

require 'json'

def is_bool?(obj)
  return obj.class == TrueClass || obj.class == FalseClass
end

def merge(omap, smap)
  if !smap.instance_of?(Hash) || !omap.instance_of?(Hash)
    abort "instance is not Hash"
  end
  smap.each_pair do |k, v|
    if v.instance_of?(Hash)
      omap[k] = omap[k] || {}
      merge(omap[k], v)
    else
      if omap[k] == nil
        omap[k] = v
      elsif v.class != NilClass && omap[k].class != v.class && !(is_bool?(v) && is_bool?(omap[k]))
        p v.class
        p omap[k].class
        abort "dif class"
      elsif omap[k].to_s == ""
        omap[k] = v
      end
    end
  end
end

map = {}
$stdin.each_line do |line|
  obj = JSON.load(line)
  merge(map, obj)
end

puts JSON.pretty_generate(map)

ruby merge.rb < event-core.json | ruby json2bq.rb
こんな感じで使う。