frsyuki

require 'msgpack'

class PageSchema
  def initialize(*keys)
    @struct = Struct.new(*keys)
    @struct.class_eval do
      def to_msgpack(arg=nil)
        packer = (arg.is_a?(MessagePack::Packer) ? arg : MessagePack::Packer.new(arg))
        packer.write_array_header(self.size)
        self.each_pair {|k,v| packer.write(k).write(v) }

frsyuki

/*
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

frsyuki

LOG:  statement: create table dt (data text);
LOG:  statement: insert into dt (data) values ('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa');
LOG:  statement: insert into dt select * from dt;
LOG:  statement: insert into dt select * from dt;
LOG:  statement: insert into dt select * from dt;
LOG:  statement: insert into dt select * from dt;
LOG:  statement: insert into dt select * from dt;
LOG:  statement: insert into dt select * from dt;
LOG:  statement: insert into dt select * from dt;
LOG:  statement: insert into dt select * from dt;

frsyuki

public class X {
    public static void main(String[] args) {
        System.out.println("doit: "+doit());
    }

    public static int doit() {
        try {
            System.out.println("try");
            throw new RuntimeException("exception");

frsyuki

frsyuki

Embulk-plugin-input-pcapng-files書いた にコメント。

本来的にはinputプラグインではなくFile input内のparser/decoderプラグインとして造るべきでは？

その通りで、本来はparserプラグインでしょう。Java側にはParserプラグインを書くAPIがあるのですが、まだそれをJRubyで書く仕組みが無いのです。書きます。 （https://github.com/embulk/embulk/issues/33）

処理対象のファイル一覧→スレッドへの分配あたりを再開発してる感

ファイルを読み込むタイプのプラグインはスレッド数を考えなくても良くて、基本的にファイル数 = タスク数にすればOKです（example_input.rbがまぎらわしいかもしれませんが…）。なぜなら、設定ファイルのexec:のところにスレッド数を書けるからです。

frsyuki

def self.transaction(config, &control)
  task = {
    'charset' => 'utf-8',
    'newline' => 'CRLF',  # or LF
  }
  # ...
  yield(task)
end

frsyuki

reqirb(main):001:0> require 'time'
=> true
irb(main):002:0> Time.strptime("2015-02-24 21:02:57.321", "%Y-%m-%d %H:%m:%S.%N").strftime("%Y-%m-%d %H:%m:%S.%N")
=> "2015-02-24 21:02:57.321000000"
irb(main):003:0> Time.strptime("1424842137.321", "%s.%N").strftime("%Y-%m-%d %H:%m:%S.%N")
=> "2015-02-24 21:02:57.000000000"
irb(main):004:0> RUBY_VERSION
=> "2.1.3"

frsyuki

package org.embulk.input;

import java.util.List;
import java.util.ArrayList;
import com.google.common.base.Optional;
import org.embulk.config.CommitReport;
import org.embulk.config.Config;
import org.embulk.config.ConfigDefault;
import org.embulk.config.ConfigDiff;
import org.embulk.config.ConfigSource;

frsyuki