sakama

package org.embulk.output.hoge;

import okhttp3.mockwebserver.MockWebServer;
import org.embulk.spi.OutputPlugin;
import org.embulk.test.TestingEmbulk;

import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;

sakama

public final class Test {
  public Test(String hoge) {

  }
}

sakama

ERROR 1297 (HY000) at line 1207: Got temporary error 410 'REDO log files overloaded (decrease TimeBetweenLocalCheckpoints or increase NoOfFragmentLogFiles)' from NDBCLUSTER
#NoOfFragmentLogFiles=4にしていたのをNoOfFragmentLogFiles=16(デフォルト値)に戻した

ERROR 1114 (HY000) at line 1213: The table 'sample_table' is full
#→通常のMySQLでも出るが一旦以下のようにして回避
#innodb_data_file_path=ibdata1:10M:autoextend→ダメだった
#config.iniの[NDBD_DEFAULT]セクション内のDataMemory、IndexMemoryを増やす必要がある
#IndexMemory=100M
#DataMemory=1024M
#※本来はndb_size.pl(http://dev.mysql.com/doc/refman/5.1/ja/mysql-cluster-utilities-ndb-size.html)や

sakama

CDH5(Cloudera Hadoop 5のインストール)

下準備

# cat /etc/system-release
Amazon Linux AMI release 2015.03

タイムゾーンの変更
# date

sakama

$ git clone git@github.com:treasure-data/embulk-input-marketo.git
$ cd embulk-input-marketo
$ ./gradlew gem
$ cd /path/to/config.yml
$ embulk run -I /path/to/embulk-input-marketo/lib config.yml

sakama

{
 "kind": "bigquery#table",
 "etag": "\"ABCDEFGHIJKLMNOPQRSTU\"",
 "id": "abcdefg:demodatasets.demotable",
 "selfLink": "https://www.googleapis.com/bigquery/v2/projects/vibrant-grammar-856/datasets/demodatasets/tables/demotable",
 "tableReference": {
  "projectId": "abcdefg",
  "datasetId": "demodatasets",
  "tableId": "demotable"
 },

sakama

$ embulk run -I ~/data/embulk-output-td/lib gcs_td.yml
2017-06-23 18:14:44.697 +0900: Embulk v0.8.25
2017-06-23 18:14:57.034 +0900 [INFO] (0001:transaction): Loaded plugin embulk-input-gcs (0.2.3)
2017-06-23 18:14:57.102 +0900 [INFO] (0001:transaction): Loaded plugin embulk/output/td from a load path
2017-06-23 18:14:59.183 +0900 [INFO] (0001:transaction): Using local thread executor with max_threads=8 / output tasks 4 = input tasks 2 * 2
2017-06-23 18:14:59.238 +0900 [INFO] (0001:transaction): td-client version: 0.7.37
2017-06-23 18:14:59.242 +0900 [INFO] (0001:transaction): Reading configuration file: /Users/someone/.td/td.conf
org.embulk.exec.PartialExecutionException: java.lang.NoClassDefFoundError: com/fasterxml/jackson/datatype/jsonorg/JsonOrgModule
	at org.embulk.exec.BulkLoader$LoaderState.buildPartialExecuteException(org/embulk/exec/BulkLoader.java:375)

sakama

Embulk Meetup #3 LTタイトル(各5〜maxで10分予定)

https://eventdots.jp/event/618872

• EmbulkでMySQLからBigQueryへ1000個のテーブルをコピーした話 by @kazukgw
• MySQLからBigQueryの同期を差分更新にしたら4倍高速になった話 by 塩﨑健弘
• とあるマーケティング部隊でのembulkの活用事例 by @grimrose
• embulk-input-redashを作ってみた @ariarijp
• Embulkを利用した開発でちょっと困ったこと by @katsuyan121
• Embulk資料集の紹介 @hiroysato

sakama

##Example Data

> db.version();
3.0.10

> db.test.find();
{
  "_id" : ObjectId("573a9d37efb32df843124db9"),
  "account" : 32864,
  "time" : ISODate("2015-01-25T13:23:15Z"),

sakama

private Schema getFieldSchema() {
    ImmutableList.Builder<Column> columns = ImmutableList.builder();
    columns.add(new Column(0, "record", Types.JSON));
    return new Schema(columns.build());
}