Skip to content

Instantly share code, notes, and snippets.

@sakamaki-kazuyoshi
Last active April 20, 2024 00:12
Show Gist options
  • Save sakamaki-kazuyoshi/0c1d71f8b514a437fc736cea3ef6e1f2 to your computer and use it in GitHub Desktop.
Save sakamaki-kazuyoshi/0c1d71f8b514a437fc736cea3ef6e1f2 to your computer and use it in GitHub Desktop.
Export CloudWatch Logs with Step Functions
BucketName: sakamaki-neko-inu #CloudWatch Logsログデータを保管するS3バケット名
TimeoutSeconds: 82800 #ステートマシンタイムアウト設定
import boto3
import logging
logs_client = boto3.client('logs')
logger = logging.getLogger()
logger.setLevel(logging.INFO)
def lambda_handler(event, context):
task_id = event['iterator']['task_id']
#エクスポートタスク ステータス取得
response = logs_client.describe_export_tasks(
taskId=task_id,
)
status_code = response['exportTasks'][0]['status']['code']
logger.info('Task ID : ' + task_id)
logger.info('Status code : ' + status_code)
return {
'status_code': status_code
}
import boto3
import jmespath
import logging
logs_client = boto3.client('logs')
logger = logging.getLogger()
logger.setLevel(logging.INFO)
def lambda_handler(event, context):
#CloudWatch Logsロググループ名取得(ロググループ名をリストに格納)
response = logs_client.describe_log_groups()
log_groups = jmespath.search('logGroups[].logGroupName',response)
logger.info('Log list : '.join(log_groups))
return {
'element_num':len(log_groups),
'log_groups':log_groups
}
import os
import boto3
import datetime
import logging
logs_client = boto3.client('logs')
logger = logging.getLogger()
logger.setLevel(logging.INFO)
def lambda_handler(event, context):
export_bucket = os.environ['EXPORT_BUCKET']
index = event['iterator']['index']
count = event['describe_log_groups']['element_num']
target_log_group = event['describe_log_groups']['log_groups']
today = datetime.date.today() #実行日取得(例:2019-09-12)
yesterday = today - datetime.timedelta(days=1) #前日取得(例:2019-09-11)
#出力日時(from)取得(例:2019-09-11 00:00:00)
from_time = datetime.datetime(year=today.year, month=today.month, day=yesterday.day, hour=0, minute=0,second=0)
#出力日時(to)取得(例:2019-09-11 23:59:59.999999)
to_time = datetime.datetime(year=today.year, month=today.month, day=yesterday.day, hour=23, minute=59,second=59,microsecond=999999)
#エポック時刻取得(float型)
epoc_from_time = from_time.timestamp()
epoc_to_time = to_time.timestamp()
#エポック時刻をミリ秒にしint型にキャスト(create_export_taskメソッドにintで渡すため)
m_epoc_from_time = int(epoc_from_time * 1000)
m_epoc_epoc_to_time = int(epoc_to_time * 1000)
#CloudWatch Logsエクスポート
response = logs_client.create_export_task(
logGroupName = target_log_group[index],
fromTime = m_epoc_from_time,
to = m_epoc_epoc_to_time,
destination = export_bucket,
destinationPrefix = yesterday.strftime('%Y%m%d')
)
logger.info('Target log group : ' + target_log_group[index])
logger.info('Task ID : ' + response['taskId'])
index += 1
return {
'index':index,
'end_flg':count == index,
'task_id':response['taskId']
}
service: TestService
plugins:
- serverless-step-functions #StepFunctionsプラグイン
provider:
name: aws
runtime: python3.7
stage: dev
region: ap-northeast-1
#Lambda関数にCloudWatch Logs権限付与
iamRoleStatements:
- Effect: Allow
Resource: "arn:aws:logs:ap-northeast-1:*:*:*"
Action:
- "logs:*"
functions:
DescribeLogGroups:
handler: DescribeLogGroups.lambda_handler
name: DescribeLogGroups
ExportLogGroup:
handler: ExportLogGroup.lambda_handler
name: ExportLogGroup
timeout: 900
environment:
#CloudWatch Logs出力先バケット
EXPORT_BUCKET: ${file(./config.yml):BucketName}
DescribeExportTask:
handler: DescribeExportTask.lambda_handler
name: DescribeExportTask
#ステートマシン定義
stepFunctions:
stateMachines:
ExportCloudWatchLogs:
name: ExportCloudWatchLogs
#CloudEatchEventで起動
events:
- schedule: cron(0 16 * * ? *)
definition:
StartAt: Configure
TimeoutSeconds: ${file(./config.yml):TimeoutSeconds}
States:
Configure:
Type: Pass
Result:
index: 0
ResultPath: "$.iterator"
Next: "DescribeLogGroups"
DescribeLogGroups:
Type: Task
Resource:
Fn::GetAtt: [DescribeLogGroupsLambdaFunction, Arn]
ResultPath: "$.describe_log_groups"
Next: "ExportLogGroup"
ExportLogGroup:
Type: Task
Resource:
Fn::GetAtt: [ExportLogGroupLambdaFunction, Arn]
ResultPath: "$.iterator"
Next: "DescribeExportTask"
DescribeExportTask:
Type: Task
Resource:
Fn::GetAtt: [DescribeExportTaskLambdaFunction, Arn]
ResultPath: "$.describe_export_task"
Next: "IsExportTask"
IsExportTask:
Type: Choice
Choices:
- Variable: "$.describe_export_task.status_code"
StringEquals: "COMPLETED"
Next: "IsComplete"
- Or:
- Variable: "$.describe_export_task.status_code"
StringEquals: "PENDING"
- Variable: "$.describe_export_task.status_code"
StringEquals: "RUNNING"
Next: "WaitSeconds"
Default: "Fail"
WaitSeconds:
Type: Wait
Seconds: 1
Next: DescribeExportTask
IsComplete:
Type: Choice
Choices:
- Variable: "$.iterator.end_flg"
BooleanEquals: true
Next: "Succeed"
Default: "ExportLogGroup"
Succeed:
Type: Succeed
Fail:
Type: Fail
resources:
Resources:
#CloudWatch Logsを出力するS3バケット作成
S3Bucket:
Type: AWS::S3::Bucket
Properties:
BucketName: ${file(./config.yml):BucketName}
#CloudWatch LogsをエクスポートできるようにS3バケットポリシー設定
BucketPolicy:
Type: AWS::S3::BucketPolicy
Properties:
Bucket: { Ref: S3Bucket }
PolicyDocument:
Version: "2012-10-17"
Statement:
-
Action:
- "s3:GetBucketAcl"
Effect: "Allow"
Resource: "arn:aws:s3:::${file(./config.yml):BucketName}"
Principal:
Service: "logs.ap-northeast-1.amazonaws.com"
-
Action:
- "s3:PutObject"
Effect: "Allow"
Resource: "arn:aws:s3:::${file(./config.yml):BucketName}/*"
Principal:
Service: "logs.ap-northeast-1.amazonaws.com"
Condition:
StringEquals:
s3:x-amz-acl: "bucket-owner-full-control"
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment