Building a pip package with AWS Codebuild and capturing EventBridge notifications

.gitignore

env
*.pyc
__pycache__
tmp
setenv-priv.sh

.PIP_BUILDER_DEMO.md

hello

app.props

AppName=pip-builder-demo

delete_stack.sh

#!/usr/bin/env bash

. app.props

aws cloudformation delete-stack --stack-name $AppName

deploy_stack.sh

#!/usr/bin/env bash

. app.props

echo "SlackWebhookUrl: $SLACK_WEBHOOK_URL"

aws cloudformation deploy --stack-name $AppName --template-file stack.json --capabilities CAPABILITY_NAMED_IAM --parameter-overrides SlackWebhookUrl=$SLACK_WEBHOOK_URL

dump_logs.py

from botocore.exceptions import ClientError

import boto3, os, re, sys, time

def fetch_log_events(logs, kwargs):
    events, token = [], None
    while True:
        if token:
            kwargs["nextToken"]=token
        resp=logs.filter_log_events(**kwargs)
        events+=resp["events"]
        if "nextToken" in resp:
            token=resp["nextToken"]
        else:
            break
    return sorted(events,
                  key=lambda x: x["timestamp"])

if __name__=="__main__":
    try:
        if len(sys.argv) < 3:
            raise RuntimeError("please enter lambda name, window")
        lambdaname, window = sys.argv[1:3]
        if not re.search("^\\d+$", window):
            raise RuntimeError("window is invalid")
        window=int(window)
        logs=boto3.client("logs")
        starttime=int(1000*(time.time()-window))
        loggroupname="/aws/lambda/%s" % lambdaname
        kwargs={"logGroupName": loggroupname,
                "startTime": starttime,
                "interleaved": True}
        events=fetch_log_events(logs, kwargs)
        for event in events:
            msg=re.sub("\\r|\\n", "", event["message"])
            print (msg)
    except RuntimeError as error:
        print ("Error: %s" % str(error))
    except ClientError as error:
        print ("Error: %s" % str(error))

empty_bucket.py

from botocore.exceptions import ClientError

import boto3, os, re, sys

def hungarorise(text):
    return "".join([tok.capitalize()
                    for tok in re.split("\\-|\\_", text)])

def fetch_outputs(cf, stackname):
    outputs={}
    for stack in cf.describe_stacks()["Stacks"]:
        if (stack["StackName"].startswith(stackname) and
            "Outputs" in stack):
            for output in stack["Outputs"]:
                outputs[output["OutputKey"]]=output["OutputValue"]
    return outputs

if __name__=="__main__":
    try:
        props=dict([tuple(row.split("="))
                    for row in open("app.props").read().split("\n")
                    if row!=''])
        stackname=props["AppName"]
        cf=boto3.client("cloudformation")
        outputs=fetch_outputs(cf, stackname)
        bucketkey=hungarorise("app-bucket")
        if bucketkey not in outputs:
            raise RuntimeError("bucket not found")
        bucketname=outputs[bucketkey]
        s3=boto3.client("s3")
        paginator=s3.get_paginator("list_objects_v2")
        pages=paginator.paginate(Bucket=bucketname)
        for struct in pages:
            if "Contents" in struct:
                for obj in struct["Contents"]:
                    print (obj["Key"])
                    s3.delete_object(Bucket=bucketname,
                                     Key=obj["Key"])
    except RuntimeError as error:
        print ("Error: %s" % str(error))
    except ClientError as error:
        print ("Error: %s" % str(error))

format_stack.py

import json, os

if __name__=="__main__":
    if not os.path.exists("tmp"):
        os.mkdir("tmp")
    struct=json.loads(open("stack.json").read())
    with open("tmp/stack.json", 'w') as f:
        f.write(json.dumps(struct,
                           indent=2))

list_bucket.py

from botocore.exceptions import ClientError

import boto3, os, re, sys

def hungarorise(text):
    return "".join([tok.capitalize()
                    for tok in re.split("\\-|\\_", text)])

def fetch_outputs(cf, stackname):
    outputs={}
    for stack in cf.describe_stacks()["Stacks"]:
        if (stack["StackName"].startswith(stackname) and
            "Outputs" in stack):
            for output in stack["Outputs"]:
                outputs[output["OutputKey"]]=output["OutputValue"]
    return outputs

if __name__=="__main__":
    try:
        props=dict([tuple(row.split("="))
                    for row in open("app.props").read().split("\n")
                    if row!=''])
        stackname=props["AppName"]
        cf=boto3.client("cloudformation")
        outputs=fetch_outputs(cf, stackname)
        bucketkey=hungarorise("app-bucket")
        if bucketkey not in outputs:
            raise RuntimeError("bucket not found")
        bucketname=outputs[bucketkey]
        s3=boto3.client("s3")
        paginator=s3.get_paginator("list_objects_v2")
        pages=paginator.paginate(Bucket=bucketname)
        for struct in pages:
            if "Contents" in struct:
                for obj in struct["Contents"]:
                    print (obj["Key"])
    except RuntimeError as error:
        print ("Error: %s" % str(error))
    except ClientError as error:
        print ("Error: %s" % str(error))

list_events.sh

#!/usr/bin/env bash

. app.props

aws cloudformation describe-stack-events --stack-name $AppName --query "StackEvents[].{\"1.Timestamp\":Timestamp,\"2.Id\":LogicalResourceId,\"3.Type\":ResourceType,\"4.Status\":ResourceStatus,\"5.Reason\":ResourceStatusReason}"

list_outputs.sh

#!/usr/bin/env bash

. app.props

aws cloudformation describe-stacks --stack-name $AppName --query 'Stacks[0].Outputs' --output table

list_resources.sh

#!/usr/bin/env bash

. app.props

aws cloudformation describe-stack-resources --stack-name $AppName --query "StackResources[].{\"1.Timestamp\":Timestamp,\"2.LogicalId\":LogicalResourceId,\"3.PhysicalId\":PhysicalResourceId,\"4.Type\":ResourceType,\"5.Status\":ResourceStatus}"

list_stacks.sh

#!/usr/bin/env bash

aws cloudformation describe-stacks --query "Stacks[].{\"1.Name\":StackName,\"2.Status\":StackStatus}"

NOTES.md

requirements.txt

awscli
boto3
botocore
pyyaml

setenv.sh

#!/usr/bin/env bash

export AWS_DEFAULT_OUTPUT=table
export AWS_PROFILE=#{your-aws-profile-here}
export AWS_REGION=#{your-aws-region-here}
export SLACK_WEBHOOK_URL=#{your-slack-webhook-url-here}

stack.json

{
  "Outputs": {
    "AppBucket": {
      "Value": {
        "Ref": "AppBucket"
      }
    },
    "AppProject": {
      "Value": {
        "Ref": "AppProject"
      }
    }
  },
  "Parameters": {
    "SlackWebhookUrl": {
      "Type": "String"
    }
  },
  "Resources": {
    "AppBucket": {
      "Properties": {},
      "Type": "AWS::S3::Bucket"
    },
    "AppPolicy": {
      "Properties": {
        "PolicyDocument": {
          "Statement": [
            {
              "Action": [
                "events:PutEvents"
              ],
              "Effect": "Allow",
              "Resource": "*"
            },
            {
              "Action": [
                "logs:CreateLogGroup",
                "logs:CreateLogStream",
                "logs:PutLogEvents"
              ],
              "Effect": "Allow",
              "Resource": "*"
            },
            {
              "Action": [
                "codebuild:*"
              ],
              "Effect": "Allow",
              "Resource": {
                "Fn::GetAtt": [
                  "AppProject",
                  "Arn"
                ]
              }
            },
            {
              "Action": [
                "s3:PutObject",
                "s3:ListBucket"
              ],
              "Effect": "Allow",
              "Resource": {
                "Fn::Sub": "${AppBucket.Arn}/*"
              }
            }
          ],
          "Version": "2012-10-17"
        },
        "PolicyName": {
          "Fn::Sub": "app-policy-${AWS::StackName}"
        },
        "Roles": [
          {
            "Ref": "AppRole"
          }
        ]
      },
      "Type": "AWS::IAM::Policy"
    },
    "AppProject": {
      "Properties": {
        "Artifacts": {
          "Location": {
            "Ref": "AppBucket"
          },
          "Name": "artifacts.zip",
          "NamespaceType": "BUILD_ID",
          "Packaging": "ZIP",
          "Path": "build",
          "Type": "S3"
        },
        "Environment": {
          "ComputeType": "BUILD_GENERAL1_SMALL",
          "Image": "aws/codebuild/standard:6.0",
          "Type": "LINUX_CONTAINER"
        },
        "LogsConfig": {
          "S3Logs": {
            "Location": {
              "Fn::Sub": "${AppBucket}/logs"
            },
            "Status": "ENABLED"
          }
        },
        "ServiceRole": {
          "Fn::GetAtt": [
            "AppRole",
            "Arn"
          ]
        },
        "Source": {
          "BuildSpec": "---\nphases:\n  install:\n    commands:\n    - pip install --upgrade pip\n    - mkdir -p build/python\n    - pip install --upgrade --target build/python $PIP_TARGET\n    runtime-versions:\n      python: $PYTHON_RUNTIME\nartifacts:\n  base-directory: build\n  files:\n  - '**/*'\nversion: '0.2'\n",
          "Type": "NO_SOURCE"
        }
      },
      "Type": "AWS::CodeBuild::Project"
    },
    "AppRole": {
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Statement": [
            {
              "Action": [
                "sts:AssumeRole"
              ],
              "Effect": "Allow",
              "Principal": {
                "Service": "codebuild.amazonaws.com"
              }
            }
          ],
          "Version": "2012-10-17"
        }
      },
      "Type": "AWS::IAM::Role"
    },
    "DemoErrorSubscriptionFilter": {
      "DependsOn": [
        "DemoLogStream",
        "SlackErrorPermission"
      ],
      "Properties": {
        "DestinationArn": {
          "Fn::GetAtt": [
            "SlackErrorFunction",
            "Arn"
          ]
        },
        "FilterPattern": "ERROR",
        "LogGroupName": {
          "Fn::Sub": "/aws/lambda/${DemoFunction}"
        }
      },
      "Type": "AWS::Logs::SubscriptionFilter"
    },
    "DemoEventInvokeConfig": {
      "Properties": {
        "FunctionName": {
          "Ref": "DemoFunction"
        },
        "MaximumRetryAttempts": 0,
        "Qualifier": "$LATEST"
      },
      "Type": "AWS::Lambda::EventInvokeConfig"
    },
    "DemoFunction": {
      "Properties": {
        "Code": {
          "ZipFile": "\nimport logging\n\nlogger=logging.getLogger()\nlogger.setLevel(logging.INFO)\n\ndef handler(event, context=None):\n    logger.warning(str(event))\n"
        },
        "Handler": "index.handler",
        "MemorySize": 512,
        "Role": {
          "Fn::GetAtt": [
            "DemoRole",
            "Arn"
          ]
        },
        "Runtime": "python3.10",
        "Timeout": 5
      },
      "Type": "AWS::Lambda::Function"
    },
    "DemoLogGroup": {
      "Properties": {
        "LogGroupName": {
          "Fn::Sub": "/aws/lambda/${DemoFunction}"
        },
        "RetentionInDays": 3
      },
      "Type": "AWS::Logs::LogGroup"
    },
    "DemoLogStream": {
      "DependsOn": [
        "DemoLogGroup"
      ],
      "Properties": {
        "LogGroupName": {
          "Fn::Sub": "/aws/lambda/${DemoFunction}"
        }
      },
      "Type": "AWS::Logs::LogStream"
    },
    "DemoPermission": {
      "Properties": {
        "Action": "lambda:InvokeFunction",
        "FunctionName": {
          "Ref": "DemoFunction"
        },
        "Principal": "events.amazonaws.com",
        "SourceArn": {
          "Fn::GetAtt": [
            "DemoRule",
            "Arn"
          ]
        }
      },
      "Type": "AWS::Lambda::Permission"
    },
    "DemoPolicy": {
      "Properties": {
        "PolicyDocument": {
          "Statement": [
            {
              "Action": [
                "logs:CreateLogGroup",
                "logs:CreateLogStream",
                "logs:PutLogEvents"
              ],
              "Effect": "Allow",
              "Resource": "*"
            }
          ],
          "Version": "2012-10-17"
        },
        "PolicyName": {
          "Fn::Sub": "demo-policy-${AWS::StackName}"
        },
        "Roles": [
          {
            "Ref": "DemoRole"
          }
        ]
      },
      "Type": "AWS::IAM::Policy"
    },
    "DemoRole": {
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Statement": [
            {
              "Action": [
                "sts:AssumeRole"
              ],
              "Effect": "Allow",
              "Principal": {
                "Service": "lambda.amazonaws.com"
              }
            }
          ],
          "Version": "2012-10-17"
        }
      },
      "Type": "AWS::IAM::Role"
    },
    "DemoRule": {
      "Properties": {
        "EventPattern": {
          "detail": {
            "completed-phase": [
              "SUBMITTED",
              "PROVISIONING",
              "DOWNLOAD_SOURCE",
              "INSTALL",
              "PRE_BUILD",
              "BUILD",
              "POST_BUILD",
              "UPLOAD_ARTIFACTS",
              "FINALIZING"
            ],
            "completed-phase-status": [
              "TIMED_OUT",
              "STOPPED",
              "FAILED",
              "SUCCEEDED",
              "FAULT",
              "CLIENT_ERROR"
            ],
            "project-name": [
              {
                "Ref": "AppProject"
              }
            ]
          },
          "detail-type": [
            "CodeBuild Build Phase Change"
          ],
          "source": [
            "aws.codebuild"
          ]
        },
        "State": "ENABLED",
        "Targets": [
          {
            "Arn": {
              "Fn::GetAtt": [
                "DemoFunction",
                "Arn"
              ]
            },
            "Id": {
              "Fn::Sub": "demo-rule-${AWS::StackName}"
            }
          }
        ]
      },
      "Type": "AWS::Events::Rule"
    },
    "DemoWarningSubscriptionFilter": {
      "DependsOn": [
        "DemoLogStream",
        "SlackWarningPermission"
      ],
      "Properties": {
        "DestinationArn": {
          "Fn::GetAtt": [
            "SlackWarningFunction",
            "Arn"
          ]
        },
        "FilterPattern": "%WARNING|Task timed out%",
        "LogGroupName": {
          "Fn::Sub": "/aws/lambda/${DemoFunction}"
        }
      },
      "Type": "AWS::Logs::SubscriptionFilter"
    },
    "SlackErrorFunction": {
      "Properties": {
        "Code": {
          "ZipFile": "import base64, gzip, json, os, urllib.request\n\n# https://colorswall.com/palette/3\n\nLevels={\"info\":  \"#5bc0de\",\n        \"warning\": \"#f0ad4e\",\n        \"error\": \"#d9534f\"}\n\ndef post_webhook(struct, url):\n    req = urllib.request.Request(url, method = \"POST\")\n    req.add_header(\"Content-Type\", \"application/json\")\n    data = json.dumps(struct).encode()\n    return urllib.request.urlopen(req, data = data).read()\n\ndef handler(event, context = None,\n            colour = Levels[os.environ[\"SLACK_LOGGING_LEVEL\"]],\n            webhookurl = os.environ[\"SLACK_WEBHOOK_URL\"]):\n    struct = json.loads(gzip.decompress(base64.b64decode(event[\"awslogs\"][\"data\"])))\n    text = json.dumps(struct)\n    struct = {\"attachments\": [{\"text\": text,\n                               \"color\": colour}]}\n    post_webhook(struct, webhookurl)\n"
        },
        "Environment": {
          "Variables": {
            "SLACK_LOGGING_LEVEL": "error",
            "SLACK_WEBHOOK_URL": {
              "Ref": "SlackWebhookUrl"
            }
          }
        },
        "Handler": "index.handler",
        "MemorySize": 512,
        "Role": {
          "Fn::GetAtt": [
            "SlackErrorRole",
            "Arn"
          ]
        },
        "Runtime": "python3.10",
        "Timeout": 5
      },
      "Type": "AWS::Lambda::Function"
    },
    "SlackErrorPermission": {
      "Properties": {
        "Action": "lambda:InvokeFunction",
        "FunctionName": {
          "Ref": "SlackErrorFunction"
        },
        "Principal": "logs.amazonaws.com"
      },
      "Type": "AWS::Lambda::Permission"
    },
    "SlackErrorPolicy": {
      "Properties": {
        "PolicyDocument": {
          "Statement": [
            {
              "Action": [
                "logs:CreateLogGroup",
                "logs:CreateLogStream",
                "logs:PutLogEvents"
              ],
              "Effect": "Allow",
              "Resource": "*"
            }
          ],
          "Version": "2012-10-17"
        },
        "PolicyName": {
          "Fn::Sub": "slack-error-policy-${AWS::StackName}"
        },
        "Roles": [
          {
            "Ref": "SlackErrorRole"
          }
        ]
      },
      "Type": "AWS::IAM::Policy"
    },
    "SlackErrorRole": {
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Statement": [
            {
              "Action": [
                "sts:AssumeRole"
              ],
              "Effect": "Allow",
              "Principal": {
                "Service": "lambda.amazonaws.com"
              }
            }
          ],
          "Version": "2012-10-17"
        }
      },
      "Type": "AWS::IAM::Role"
    },
    "SlackWarningFunction": {
      "Properties": {
        "Code": {
          "ZipFile": "import base64, gzip, json, os, urllib.request\n\n# https://colorswall.com/palette/3\n\nLevels={\"info\":  \"#5bc0de\",\n        \"warning\": \"#f0ad4e\",\n        \"error\": \"#d9534f\"}\n\ndef post_webhook(struct, url):\n    req = urllib.request.Request(url, method = \"POST\")\n    req.add_header(\"Content-Type\", \"application/json\")\n    data = json.dumps(struct).encode()\n    return urllib.request.urlopen(req, data = data).read()\n\ndef handler(event, context = None,\n            colour = Levels[os.environ[\"SLACK_LOGGING_LEVEL\"]],\n            webhookurl = os.environ[\"SLACK_WEBHOOK_URL\"]):\n    struct = json.loads(gzip.decompress(base64.b64decode(event[\"awslogs\"][\"data\"])))\n    text = json.dumps(struct)\n    struct = {\"attachments\": [{\"text\": text,\n                               \"color\": colour}]}\n    post_webhook(struct, webhookurl)\n"
        },
        "Environment": {
          "Variables": {
            "SLACK_LOGGING_LEVEL": "warning",
            "SLACK_WEBHOOK_URL": {
              "Ref": "SlackWebhookUrl"
            }
          }
        },
        "Handler": "index.handler",
        "MemorySize": 512,
        "Role": {
          "Fn::GetAtt": [
            "SlackWarningRole",
            "Arn"
          ]
        },
        "Runtime": "python3.10",
        "Timeout": 5
      },
      "Type": "AWS::Lambda::Function"
    },
    "SlackWarningPermission": {
      "Properties": {
        "Action": "lambda:InvokeFunction",
        "FunctionName": {
          "Ref": "SlackWarningFunction"
        },
        "Principal": "logs.amazonaws.com"
      },
      "Type": "AWS::Lambda::Permission"
    },
    "SlackWarningPolicy": {
      "Properties": {
        "PolicyDocument": {
          "Statement": [
            {
              "Action": [
                "logs:CreateLogGroup",
                "logs:CreateLogStream",
                "logs:PutLogEvents"
              ],
              "Effect": "Allow",
              "Resource": "*"
            }
          ],
          "Version": "2012-10-17"
        },
        "PolicyName": {
          "Fn::Sub": "slack-warning-policy-${AWS::StackName}"
        },
        "Roles": [
          {
            "Ref": "SlackWarningRole"
          }
        ]
      },
      "Type": "AWS::IAM::Policy"
    },
    "SlackWarningRole": {
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Statement": [
            {
              "Action": [
                "sts:AssumeRole"
              ],
              "Effect": "Allow",
              "Principal": {
                "Service": "lambda.amazonaws.com"
              }
            }
          ],
          "Version": "2012-10-17"
        }
      },
      "Type": "AWS::IAM::Role"
    }
  }
}

start_build.py

from botocore.exceptions import ClientError

import boto3, os, re, sys

def hungarorise(text):
    return "".join([tok.capitalize()
                    for tok in re.split("\\-|\\_", text)])

def fetch_outputs(cf, stackname):
    outputs={}
    for stack in cf.describe_stacks()["Stacks"]:
        if (stack["StackName"].startswith(stackname) and
            "Outputs" in stack):
            for output in stack["Outputs"]:
                outputs[output["OutputKey"]]=output["OutputValue"]
    return outputs

if __name__=="__main__":
    try:
        import sys
        if len(sys.argv) < 2:
            raise RuntimeError("please enter pip target")
        piptarget=sys.argv[1]
        props=dict([tuple(row.split("="))
                    for row in open("app.props").read().split("\n")
                    if row!=''])
        stackname=props["AppName"]
        cf=boto3.client("cloudformation")
        outputs=fetch_outputs(cf, stackname)
        projectkey=hungarorise("app-project")
        if projectkey not in outputs:
            raise RuntimeError("project not found")
        projectname=outputs[projectkey]
        cb=boto3.client("codebuild")
        env=[{"name": k,
              "value": v,
              "type": "PLAINTEXT"}
             for k, v in [("PIP_TARGET", piptarget),
                          ("PYTHON_RUNTIME", "3.10")]]
        print (cb.start_build(projectName=projectname,
                              environmentVariablesOverride=env))
    except RuntimeError as error:
        print ("Error: %s" % str(error))
    except ClientError as error:
        print ("Error: %s" % str(error))

TODO.md

short

done

• s3 PutObject/ListBucket permissions

• policy resource arn for s3 permissions is incorrectly specified

  • needs /* at end of ARN

• Bucket needs to be visible

  • probably true of all base state resources and builder project

• LogsConfig needs S3Logs subdir

• CodeBuild not Codebuild

  • needs exception

• empty bucket script

• list bucket script

CLIENT_ERROR: \\nError in UPLOAD_ARTIFACTS phase: AccessDenied: Access Denied\\n\\tstatus code: 403, request id: D9RC4XBY0ZWZ8J1H, host id: D6SFZpGBMBsWOk9CLSdNYfvfwjG4mmHu4n25FhuDS+UnK7k6qSXnbpo7fFXhv67y33vRnjS3ahWzIfsWdKK84JFah0ac5OUD'], 'start-time': 'Apr 1, 2024 2:10:14 PM', 'end-time': 'Apr 1, 2024 2:10:14 PM', 'duration-in-seconds': 0.0, 'phase-type': 'UPLOAD_ARTIFACTS', 'phase-status': 'FAILED'}, {'start-time': 'Apr 1, 2024 2:10:14 PM', 'phase-type': 'FINALIZING'}], 'queued-timeout-in-minutes': 480.0}, 'completed-phase-status': 'FAILED', 'completed-phase-duration-seconds': 0.0, 'version': '1', 'completed-phase-start': 'Apr 1, 2024 2:10:14 PM', 'completed-phase-end': 'Apr 1, 2024 2:10:14 PM'}}\n"}]}

• script to populate and start builder

|  2024-04-01T13:53:48.897Z|  AppProject             |  AWS::CodeBuild::Project        |  CREATE_FAILED        |  Property validation failure: [Encountered unsupported properties in {/LogsConfig}: [Status, Location]]