Export CloudWatch Logs with Step Functions

config.yml

BucketName: sakamaki-neko-inu #CloudWatch Logsログデータを保管するS3バケット名
TimeoutSeconds: 82800 #ステートマシンタイムアウト設定

DescribeExportTask.py

import boto3
import logging

logs_client = boto3.client('logs')
logger = logging.getLogger()
logger.setLevel(logging.INFO)

def lambda_handler(event, context):
    task_id = event['iterator']['task_id']

    #エクスポートタスク ステータス取得
    response = logs_client.describe_export_tasks(
        taskId=task_id,
    )
    status_code = response['exportTasks'][0]['status']['code']

    logger.info('Task ID : ' + task_id)
    logger.info('Status code : ' + status_code)

    return {
        'status_code': status_code
    }

DescribeLogGroups.py

import boto3
import jmespath
import logging

logs_client = boto3.client('logs')
logger = logging.getLogger()
logger.setLevel(logging.INFO)

def lambda_handler(event, context):
    #CloudWatch Logsロググループ名取得（ロググループ名をリストに格納）
    response = logs_client.describe_log_groups()
    log_groups = jmespath.search('logGroups[].logGroupName',response)

    logger.info('Log list : '.join(log_groups))

    return {
        'element_num':len(log_groups),
        'log_groups':log_groups
    }

ExportLogGroup.py

import os
import boto3
import datetime
import logging

logs_client = boto3.client('logs')
logger = logging.getLogger()
logger.setLevel(logging.INFO)

def lambda_handler(event, context):
    export_bucket = os.environ['EXPORT_BUCKET']
    index = event['iterator']['index']
    count = event['describe_log_groups']['element_num']
    target_log_group = event['describe_log_groups']['log_groups']

    today = datetime.date.today()                    #実行日取得（例：2019-09-12）
    yesterday = today - datetime.timedelta(days=1)   #前日取得（例：2019-09-11）
    #出力日時（from）取得（例：2019-09-11 00:00:00）
    from_time = datetime.datetime(year=today.year, month=today.month, day=yesterday.day, hour=0, minute=0,second=0)
    #出力日時（to）取得（例：2019-09-11 23:59:59.999999）
    to_time = datetime.datetime(year=today.year, month=today.month, day=yesterday.day, hour=23, minute=59,second=59,microsecond=999999)

    #エポック時刻取得(float型)
    epoc_from_time = from_time.timestamp()
    epoc_to_time = to_time.timestamp()
    #エポック時刻をミリ秒にしint型にキャスト（create_export_taskメソッドにintで渡すため）
    m_epoc_from_time = int(epoc_from_time * 1000)
    m_epoc_epoc_to_time = int(epoc_to_time * 1000)

    #CloudWatch Logsエクスポート
    response = logs_client.create_export_task(
        logGroupName = target_log_group[index],
        fromTime = m_epoc_from_time,
        to = m_epoc_epoc_to_time,
        destination = export_bucket,
        destinationPrefix = yesterday.strftime('%Y%m%d')
    )

    logger.info('Target log group : ' + target_log_group[index])
    logger.info('Task ID : ' + response['taskId'])

    index += 1

    return {
        'index':index,
        'end_flg':count == index,
        'task_id':response['taskId']
    }

serverless.yml

service: TestService

plugins:
  - serverless-step-functions     #StepFunctionsプラグイン

provider:
  name: aws
  runtime: python3.7
  stage: dev
  region: ap-northeast-1
  #Lambda関数にCloudWatch Logs権限付与
  iamRoleStatements:
    - Effect: Allow
      Resource: "arn:aws:logs:ap-northeast-1:*:*:*"
      Action:
        - "logs:*"

functions:
  DescribeLogGroups:
    handler: DescribeLogGroups.lambda_handler
    name: DescribeLogGroups
  ExportLogGroup:
    handler: ExportLogGroup.lambda_handler
    name: ExportLogGroup
    timeout: 900
    environment:
      #CloudWatch Logs出力先バケット
      EXPORT_BUCKET: ${file(./config.yml):BucketName}
  DescribeExportTask:
    handler: DescribeExportTask.lambda_handler
    name: DescribeExportTask

#ステートマシン定義
stepFunctions:
  stateMachines:
    ExportCloudWatchLogs:
      name: ExportCloudWatchLogs
      #CloudEatchEventで起動
      events:
        - schedule: cron(0 16 * * ? *)
      definition:
        StartAt: Configure
        TimeoutSeconds: ${file(./config.yml):TimeoutSeconds}
        States:
          Configure:
            Type: Pass
            Result:
              index: 0
            ResultPath: "$.iterator"
            Next: "DescribeLogGroups"
          DescribeLogGroups:
            Type: Task
            Resource:
              Fn::GetAtt: [DescribeLogGroupsLambdaFunction, Arn]
            ResultPath: "$.describe_log_groups"
            Next: "ExportLogGroup"
          ExportLogGroup:
            Type: Task
            Resource:
              Fn::GetAtt: [ExportLogGroupLambdaFunction, Arn]
            ResultPath: "$.iterator"
            Next: "DescribeExportTask"
          DescribeExportTask:
            Type: Task
            Resource:
              Fn::GetAtt: [DescribeExportTaskLambdaFunction, Arn]
            ResultPath: "$.describe_export_task"
            Next: "IsExportTask"
          IsExportTask:
            Type: Choice
            Choices:
              - Variable: "$.describe_export_task.status_code"
                StringEquals: "COMPLETED"
                Next: "IsComplete"
              - Or:
                - Variable: "$.describe_export_task.status_code"
                  StringEquals: "PENDING"
                - Variable: "$.describe_export_task.status_code"
                  StringEquals: "RUNNING"
                Next: "WaitSeconds"
            Default: "Fail"
          WaitSeconds:
            Type: Wait
            Seconds: 1
            Next: DescribeExportTask
          IsComplete:
            Type: Choice
            Choices:
              - Variable: "$.iterator.end_flg"
                BooleanEquals: true
                Next: "Succeed"
            Default: "ExportLogGroup"
          Succeed:
            Type: Succeed
          Fail:
            Type: Fail

resources:
  Resources:
    #CloudWatch Logsを出力するS3バケット作成
    S3Bucket:
      Type: AWS::S3::Bucket
      Properties:
        BucketName: ${file(./config.yml):BucketName}
    #CloudWatch LogsをエクスポートできるようにS3バケットポリシー設定
    BucketPolicy:
      Type: AWS::S3::BucketPolicy
      Properties:
        Bucket: { Ref: S3Bucket }
        PolicyDocument:
          Version: "2012-10-17"
          Statement:
            -
              Action:
                - "s3:GetBucketAcl"
              Effect: "Allow"
              Resource: "arn:aws:s3:::${file(./config.yml):BucketName}"
              Principal:
                Service: "logs.ap-northeast-1.amazonaws.com"
            -
              Action:
                - "s3:PutObject"
              Effect: "Allow"
              Resource: "arn:aws:s3:::${file(./config.yml):BucketName}/*"
              Principal:
                Service: "logs.ap-northeast-1.amazonaws.com"
              Condition:
                StringEquals:
                  s3:x-amz-acl: "bucket-owner-full-control"