console.log

# embulk preview ./test_access.yml
2015-11-11 13:37:01.020 +0900: Embulk v0.7.8
2015-11-11 13:37:01.844 +0900 [INFO] (preview): Loaded plugin embulk-filter-column (0.3.1)
2015-11-11 13:37:01.860 +0900 [INFO] (preview): Loaded plugin embulk-parser-apache-custom-log (0.3.0)
2015-11-11 13:37:01.871 +0900 [INFO] (preview): Listing local files at directory '.' filtering filename by prefix 'test_access.log'
2015-11-11 13:37:01.875 +0900 [INFO] (preview): Loading files [test_access.log]
2015-11-11 13:37:01.938 +0900 [INFO] (preview): LogFormat : %h %v %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i" "%{Cookie}i" "%{Set-cookie}o" %D %{X_VIASSL}i
2015-11-11 13:37:01.938 +0900 [INFO] (preview): RegExp    : (.*) ([^\s]*) (.*) \[([^\]]+)\] "(.*)" ([1-9]\d{2}) (-?\d+|-) "(.*)" "(.*)" "(.*)" "(.*)" (-?\d+|-) (.*)
2015-11-11 13:37:01.938 +0900 [INFO] (preview): replacement : 13
+-------------------------+
|  request-time:timestamp |
+-------------------------+
| 2015-11-10 01:59:59 UTC |
| 2015-11-10 02:00:00 UTC |
+-------------------------+
# embulk run ./test_access.yml
2015-11-11 13:37:09.165 +0900: Embulk v0.7.8
2015-11-11 13:37:10.420 +0900 [INFO] (transaction): Loaded plugin embulk-filter-column (0.3.1)
2015-11-11 13:37:10.446 +0900 [INFO] (transaction): Loaded plugin embulk-parser-apache-custom-log (0.3.0)
2015-11-11 13:37:10.460 +0900 [INFO] (transaction): Listing local files at directory '.' filtering filename by prefix 'test_access.log'
2015-11-11 13:37:10.465 +0900 [INFO] (transaction): Loading files [test_access.log]
2015-11-11 13:37:10.520 +0900 [INFO] (transaction): {done:  0 / 1, running: 0}
2015-11-11 13:37:10.565 +0900 [INFO] (task-0000): LogFormat : %h %v %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i" "%{Cookie}i" "%{Set-cookie}o" %D %{X_VIASSL}i
2015-11-11 13:37:10.565 +0900 [INFO] (task-0000): RegExp    : (.*) ([^\s]*) (.*) \[([^\]]+)\] "(.*)" ([1-9]\d{2}) (-?\d+|-) "(.*)" "(.*)" "(.*)" "(.*)" (-?\d+|-) (.*)
2015-11-11 13:37:10.565 +0900 [INFO] (task-0000): replacement : 13
Nov 10-NOV-2015 2        "01:59:59 AM" %>s Nov "%{Referer}i" "%{User-Agent}i" "%{Cookie}i" "%{Set-cookie}o" 11/10/15 %{X_VIASSL}i
Nov 10-NOV-2015 2        "02:00:00 AM" %>s Nov "%{Referer}i" "%{User-Agent}i" "%{Cookie}i" "%{Set-cookie}o" 11/10/15 %{X_VIASSL}i
2015-11-11 13:37:10.600 +0900 [INFO] (transaction): {done:  1 / 1, running: 0}
2015-11-11 13:37:10.611 +0900 [INFO] (main): Committed.
2015-11-11 13:37:10.612 +0900 [INFO] (main): Next config diff: {"in":{"last_path":"test_access.log"},"out":{}}

test_access.log

123.123.123.123 example.jp - [10/Nov/2015:10:59:59 +0900] "POST /" 200 1182 "http://example.jp/" "M" "op" "-" 45215 off
234.234.234.234 example.jp - [10/Nov/2015:11:00:00 +0900] "GET /test.js HTTP/1.1" 200 95 "http://example.jp/" "M" "_ga=" "-" 429 off

test_access.yml

in:
  type: file
  path_prefix: test_access.log
#  decoders:
#  - type: commons-compress
#    format: bzip2
  parser:
    type: apache-custom-log
####    format: "%v %{X-Forwarded-For}i %l %u %t \"%m %U%q %H\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O %D"
    format: "%h %v %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%{Cookie}i\" \"%{Set-cookie}o\" %D %{X_VIASSL}i"

# LogFormat "%h %v %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%{Cookie}i\" \"%{Set-cookie}o\" %D %{X_VIASSL}i" urchin
# %h             remote-host
# %v             request-server-name
# %u             request-user
# %t             request-time
# %r             request-line
# %s             response-status
# %b             response-bytes
# %{Referer}i    request-header-Referer
# %{User-Agent}i request-header-User-Agent
# %{Cookie}i     request-header-Cookie
# %{Set-cookie}o response-header-Set-cookie
# %D             request-process-time-us
# %{X_VIASSL}i   reequest-header-x_viassl
 
filters:
  - type: column
    columns:
##      - {name: remote-host, type: string}
#####      - {name: request-server-name, type: string}
#####      - {name: request-user, type: string}
      - {name: request-time, type: timestamp}
##      - {name: response-bytes, type: long}
##      - {name: request-line, type: string}
##      - {name: response-status, type: long}
##      - {name: response-bytes, type: long}
##      - {name: request-header-Referer, type: string}
##      - {name: request-header-User-Agent, type: string}
###      - {name: request-header-Cookie, type: string}
#####      - {name: response-header-Set-cookie, type: string}
##      - {name: request-process-time-us, type: long}
##      - {name: request-header-X_VIASSL, type: string}
out: {type: stdout}