Parsing and extracting headwords and part-of-speech from GCIDE and save them to GDBM

gcide_extract.rb

require "nokogiri"
require "json"
require 'gdbm'

class LiPosFromGcideExtractor
  def parse_each_file(filename) 
    File.open(filename, "r:ISO-8859-1") do |file|
      chunks = file.read
        .split(/\n\n/)
        .select{|chunk| chunk =~ /^[<\[]\w/}    
      
      chunks.each do |chunk|
        doc = Nokogiri::XML(chunk)
        ent = doc.css("ent").map{|n| n.text}.join(" ")
        pos = doc.css("pos").map{|n| n.text}.join(" ")
        if pos != "" and ent != ""
          @db[ent] = pos.split(/\s+/).map{|p| p.gsub(/\./, "")}.join(" ")
      end
      end
    end
  end

  def extract
    @db = GDBM.new("gcide_li_pos.db")
    for i in "A".."Z"
      parse_each_file("CIDE.#{i}")
    end
    @db.close
  end
end

(LiPosFromGcideExtractor.new).extract