task-limit-exceeded

db_load.dig

for_each>:
    year: ["2015", "2016"]
    month: ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"]
_do:
  +monthly_repeat:
    _export:
      year_month: "${year}-${month}"
      start_of_month: "${moment(year_month).startOf('month').format('YYYY-MM-DD')}"
      end_of_month: "${moment(start_of_month).endOf('month').format('DD')}"

    loop>: ${end_of_month}
    _do:
      +db_load:
        _export:
          start_date: "${moment(start_of_month).add(i, 'days').format('YYYY-MM-DD')}"
          next_date: "${moment(start_date).add(1, 'days').format('YYYY-MM-DD')}"
          table_suffix: "${moment(start_date).format('YYYYMMDD')}"

        +db1:
          _export:
            database: 'database_1'
          _parallel: true

          +users:
            _export:
              table: 'users'
              select: 'id, status, 〜'

            +job:
              embulk>: tasks/db/embulk/mysql_bigquery.yml

          +hoge:
            _export:
              table: 'hoge'
              select: '*'

            +job:
              embulk>: tasks/db/embulk/mysql_bigquery.yml

        +db2:
          _export:
            database: 'database_2'

          +fuga_events:
            _export:
              table: 'fuga_events'
              select: 'id, category, 〜'

            +job:
              embulk>: tasks/db/embulk/mysql_bigquery.yml

          +fuga_events_activities:
            _export:
              table: 'fuga_events_activities'
              select: 'id, category, 〜'

            +job:
              embulk>: tasks/db/embulk/mysql_bigquery.yml

mysql_bigquery.yml

in:
  type: mysql
  host: ${mysql_host}
  user: ${mysql_user}
  password: '${mysql_password}'
  database: ${database}
  table: ${table}
  select: '${select}'
  where: "created_at >= '${start_date} 00:00:00' AND created_at < '${next_date} 00:00:00'"
  connect_timeout: 0
  socket_timeout: 0
out:
  type: bigquery
  mode: replace
  auth_method: json_key
  json_keyfile: ${json_keyfile}
  project: ${project_id}
  dataset: ${dataset}
  auto_create_table: true
  table: ${table}_${table_suffix}
  allow_quoted_newlines: true
  timeout_sec: 3000
  open_timeout_sec: 3000