Created
March 15, 2015 07:26
-
-
Save muziyoshiz/e9fa61a4167143c01fec to your computer and use it in GitHub Desktop.
LineDecoder Usage Sample
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
module Embulk | |
module Parser | |
class LineDecoderSampleParserPlugin < ParserPlugin | |
Plugin.register_parser("line-decoder-sample", self) | |
def self.transaction(config, &control) | |
# configuration code: | |
parser_task = config.load_config(Java::LineDecoder::DecoderTask) | |
task = { | |
"decoder_task" => DataSource.from_java(parser_task.dump) | |
# "property1" => config.param("property1", :string), | |
# "property2" => config.param("property2", :integer, default: 0), | |
} | |
columns = [ | |
Column.new(0, "line", :string), | |
# Column.new(0, "example", :string), | |
# Column.new(1, "column", :long), | |
# Column.new(2, "name", :double), | |
] | |
yield(task, columns) | |
end | |
def init | |
# initialization code: | |
# @property1 = task["property1"] | |
# @property2 = task["property2"] | |
@decoder_task = task.param("decoder_task", :hash).load_task(Java::LineDecoder::DecoderTask) | |
end | |
def run(file_input) | |
decoder = Java::LineDecoder.new(file_input.instance_eval { @java_file_input }, @decoder_task) | |
while decoder.nextFile | |
while line = decoder.poll | |
page_builder.add([line]) | |
end | |
end | |
# while file = file_input.next_file | |
# file.each do |buffer| | |
# # parsering code | |
# record = ["col1", 2, 3.0] | |
# page_builder.add(record) | |
# end | |
# end | |
page_builder.finish | |
end | |
end | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment