digdag embulk example - Move Large volumes of data using Embulk from any data store to another.

digdag.md

hoge.dig

timezone: UTC

_export:
  my_path: "/tmp/hoge/csv"

+step1:
  embulk>: ./config.yml

config.yaml

in:
  type: file
  path_prefix: ${my_path} # <-- これ
  decoders:
  - {type: gzip}
  parser:
    charset: UTF-8
    newline: CRLF
    type: csv
    delimiter: ','
    quote: '"'
    escape: '"'
    null_string: 'NULL'
    trim_if_not_quoted: false
    skip_header_lines: 1
    allow_extra_columns: false
    allow_optional_columns: false
    columns:
    - {name: id, type: long}
    - {name: account, type: long}
    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
    - {name: purchase, type: timestamp, format: '%Y%m%d'}
    - {name: comment, type: string}
out: {type: stdout}

digdag run hoge --rerun
2016-06-29 17:30:22 +0900: Digdag v0.8.2
2016-06-29 17:30:24 +0900 [WARN] (main): Reusing the last session time 2016-06-29T00:00:00+00:00.
2016-06-29 17:30:24 +0900 [INFO] (main): Using session .digdag/status/20160629T000000+0000.
2016-06-29 17:30:24 +0900 [INFO] (main): Starting a new session project id=1 workflow name=hoge session_time=2016-06-29T00:00:00+00:00
2016-06-29 17:30:24 +0900 [INFO] (0018@+hoge+step1): embulk>: ./config.yml
2016-06-29 17:30:29.404 +0900: Embulk v0.8.9
2016-06-29 17:30:31.673 +0900 [INFO] (0001:transaction): Listing local files at directory '/tmp/hoge/csv' filtering filename by prefix ''
2016-06-29 17:30:31.681 +0900 [INFO] (0001:transaction): Loading files [/tmp/hoge/csv/sample_01.csv.gz]