We install from git, and run all of this in a Docker container, within Jenkins via a Jenkinsfile. Our Jenkinsfile and Makefile are attached.
Note that in order to get the mailer to recognize custom templates, they must be copied into the msg-templates/ directory under the c7n_mailer source. Note line 25 of the Makefile.
We use Terraform to manage the IAM Role for the lambda function and the CloudWatch Log Group.
| #!/usr/bin/env groovy | |
| // internal/private shared library; not terribly important to the functionality | |
| @Library('re-pipeline-library@master') _ | |
| node { | |
| deleteDir() | |
| checkout scm | |
| stash name: 'configs', includes: '*.yml, templates/*.html.j2' | |
| // wraps is a global variable from our internal re-pipeline-library; it just applies | |
| // timestamp and ANSI color build wrappers to the closure within it. | |
| wraps { | |
| def environment = docker.image('python:2-wheezy') | |
| // run as user 0 group 0 - see comment in Validate stage | |
| environment.inside('-u 0:0') { | |
| stage('Setup Virtualenv') { | |
| /* | |
| * If we run the container as 1000:1000 (which Jenkins will do by | |
| * default), we can't pip install from a git URL in the container, | |
| * because we're running as a user not present in /etc/passwd. But | |
| * running as 0:0 has the side effect that any files we create | |
| * in `pwd` (the workspace, mounted RW into the container) will be | |
| * owned 0:0 (even on the host)... which means deleteDir() will fail | |
| * with permissions errors. The simple solution is to not touch | |
| * anything in the workspace at all; copy it to a path that exists | |
| * only in the container and mess with it there. | |
| */ | |
| sh "mkdir /app && cp -a . /app && cd /app && virtualenv --no-site-packages -p python2.7 ." | |
| } | |
| stage('Validate') { | |
| sh "cd /app && make validate" | |
| archiveArtifacts artifacts: "custodian.yml" | |
| } | |
| stage('Dry Run') { | |
| sh "cd /app && make dryrun" | |
| sh "cp -a /app/dryrun . && chown -R 1000:1000 ." | |
| archiveArtifacts artifacts: "dryrun/**/*" | |
| } | |
| if (env.BRANCH_NAME == 'master') { | |
| stage('Install and Run') { | |
| sh "cd /app && make run mailer" | |
| } | |
| } else { | |
| stage('Install Mailer') { | |
| // not master - install deps but don't run mailer | |
| sh "cd /app && make mailerdeps && cat mailer.yml" | |
| } | |
| } | |
| } //environment | |
| } // wrapper | |
| } |
| .PHONY: docs clean | |
| PROJECT=$(shell grep "project" terraform/terraform.tfvars | cut -d= -f2 | tr -d '[[="=]],[[:space:]]') | |
| ENVIRONMENT=$(shell grep "environment" terraform/terraform.tfvars | cut -d= -f2 | tr -d '[[="=]],[[:space:]]') | |
| ACCOUNT_ID=$(shell aws sts get-caller-identity --region=$(REGION) --output text --query 'Account') | |
| BUCKET=s3://$(PROJECT)-$(ACCOUNT_ID) | |
| REGION=us-east-1 | |
| MARKDOWN = pandoc --from markdown_github --to html --standalone | |
| INSTALL_REPO=git+https://github.com/capitalone/cloud-custodian.git | |
| INSTALL_REF=0.8.24.3 | |
| virtualenv: | |
| if [ ! -e "./bin/activate_this.py" ] ; then virtualenv --clear .; fi | |
| deps: virtualenv | |
| PYTHONPATH=. ; . ./bin/activate && \ | |
| pip install -e "$(INSTALL_REPO)@$(INSTALL_REF)#egg=c7n" | |
| mailerdeps: deps | |
| PYTHONPATH=. ; . ./bin/activate && \ | |
| pip install -r src/c7n/tools/c7n_mailer/requirements.txt && \ | |
| cd src/c7n/tools/c7n_mailer && \ | |
| python setup.py develop && \ | |
| cd ../../../../ | |
| cp templates/* src/c7n/tools/c7n_mailer/msg-templates/ | |
| clean: clean-build clean-pyc clean-test | |
| clean-build: | |
| rm -fr build/ | |
| rm -fr dist/ | |
| rm -fr .eggs/ | |
| find . -name '*.egg-info' -exec rm -rf {} + | |
| find . -name '*.egg' -exec rm -rf {} + | |
| clean-pyc: | |
| find . -name '*.pyc' -exec rm -f {} + | |
| find . -name '*.pyo' -exec rm -f {} + | |
| find . -name '*~' -exec rm -f {} + | |
| find . -name '__pycache__' -exec rm -fr {} + | |
| clean-test: | |
| rm -fr .tox/ | |
| rm -f .coverage | |
| rm -fr htmlcov/ | |
| install: clean | |
| PYTHONPATH=$PYTHONPATH:.:. ; . ./bin/activate && python setup.py install | |
| policies: deps | |
| PYTHONPATH=. ; . ./bin/activate && ./policygen.py | |
| validate: policies | |
| echo "Running custodian in ${ACCOUNT}" | |
| PYTHONPATH=$PYTHONPATH:.:. ; . ./bin/activate && custodian validate -c custodian.yml | |
| dryrun: validate | |
| PYTHONPATH=$PYTHONPATH:.:. ; . ./bin/activate && custodian run --region '$(REGION)' --dryrun --metrics -v -s dryrun -c custodian.yml --cache '/tmp/.cache/cloud-custodian.cache' | |
| run: validate | |
| PYTHONPATH=$PYTHONPATH:.:. ; . ./bin/activate && custodian run --region '$(REGION)' --metrics -v -s $(BUCKET)/logs --log-group=/cloud-custodian/$(ACCOUNT_ID)/$(REGION) -c custodian.yml --cache '/tmp/.cache/cloud-custodian.cache' | |
| config: | |
| @echo $(PROJECT) | |
| @echo $(ENVIRONMENT) | |
| @echo $(BUCKET) | |
| mailer: mailerdeps | |
| PYTHONPATH=$PYTHONPATH:.:. ; . ./bin/activate && c7n-mailer -c mailer.yml |
| <!DOCTYPE html> | |
| <html lang="en"> | |
| {# | |
| Template customizations: | |
| - additional parameters for slack channel and email for questions | |
| - link to our internal docs | |
| - case-insensitive tag lookups in the getTag macro | |
| - formatting that renders correctly in Office365 and acceptably in GMail | |
| - switch the default tags in the message from Application and Owner to our internal tags (Project, Component, Environment, OwnerEmail) plus aws:autoscaling:groupName | |
| #} | |
| {# | |
| Sample Policy that can be used with this template: | |
| Additional parameters can be passed in from the policy - i.e. action_desc, violation_desc | |
| - name: delete-unencrypted-ec2 | |
| resource: ec2 | |
| filters: | |
| - type: ebs | |
| key: Encrypted | |
| value: false | |
| actions: | |
| - terminate | |
| - type: notify | |
| template: redefault.html | |
| subject: "[custodian {{ account }}] Delete Unencrypted EC2 - {{ region }}" | |
| violation_desc: "The following EC2(s) are not encrypted:" | |
| action_desc: "The EC2(s) have been terminated" | |
| questions_email: me@example.com | |
| questions_slack: mySlackChannel | |
| to: | |
| - owner@domain.com | |
| transport: | |
| type: sqs | |
| queue: https://sqs.us-east-1.amazonaws.com/12345678910/custodian-sqs-queue | |
| #} | |
| {# You can set any mandatory tags here, and they will be formatted/outputted in the message #} | |
| {% set requiredTags = ['Project','Component','Environment','OwnerEmail','aws:autoscaling:groupName'] %} | |
| {# The macros below format some resource attributes for better presentation #} | |
| {% macro getTag(resource, tagKey) -%} | |
| {% if resource.get('Tags') %} | |
| {% for t in resource.get('Tags') %} | |
| {% if t.get('Key')|lower == tagKey|lower %} | |
| {{ t.get('Value') }} | |
| {% endif %} | |
| {% endfor %} | |
| {% endif %} | |
| {%- endmacro %} | |
| {% macro extractList(resource, column) -%} | |
| {% for p in resource.get(column) %} | |
| {{ p }}, | |
| {% endfor %} | |
| {%- endmacro %} | |
| {% macro columnHeading(columnNames, tableWidth) -%} | |
| <table style="width: {{ tableWidth }}; border-spacing: 0px; box-shadow: 5px 5px 5px grey; border-collapse:separate; border-radius: 7px;"> | |
| <tr> | |
| {% for columnName in columnNames %} | |
| {% set thstyle = "background: #a1bae2; color: white; border: 1px solid #a1bae2; text-align: center; padding: 5px;" %} | |
| {% if loop.index == 1 %} | |
| <th style="{{ thstyle }} border-top-left-radius: 7px;">{{ columnName }}</th> | |
| {% elif loop.index == columnNames|length %} | |
| <th style="{{ thstyle }} border-top-right-radius: 7px;">{{ columnName }}</th> | |
| {% else %} | |
| <th style="{{ thstyle }}">{{ columnName }}</th> | |
| {% endif %} | |
| {% endfor %} | |
| </tr> | |
| {%- endmacro %} | |
| {# This macro creates a row in the table #} | |
| {% macro tableRow(resource, columnNames, loop_idx, res_len) %} | |
| {% if loop_idx % 2 == 0 %} | |
| <tr style="background-color: #f2f2f2;"> | |
| {% else %} | |
| <tr> | |
| {% endif %} | |
| {% for columnName in columnNames %} | |
| {% set tdpart = "border: 1px solid grey; padding: 4px;" %} | |
| {% if loop_idx == res_len %} | |
| {# last row in table #} | |
| {% if loop.index == 1 %} | |
| {# first td in row #} | |
| {% set tdpart = "%s border-bottom-left-radius: 7px;" % tdpart %} | |
| {% elif loop.index == columnNames|length %} | |
| {# last td in row #} | |
| {% set tdpart = "%s border-bottom-right-radius: 7px;" % tdpart %} | |
| {% endif %} | |
| {% endif %} | |
| {% if columnName in requiredTags %} | |
| <td style="{{ tdpart }}">{{ getTag(resource,columnName) }}</td> | |
| {% elif columnName == 'tag.Name' %} | |
| <td style="{{ tdpart }}">{{ getTag(resource,'Name') }}</td> | |
| {% elif columnName == 'InstanceCount' %} | |
| <td align="center" style="{{ tdpart }}">{{ resource['Instances'] | length }}</td> | |
| {% elif columnName == 'VolumeConsumedReadWriteOps' %} | |
| <td style="{{ tdpart }}">{{ resource['c7n.metrics']['AWS/EBS.VolumeConsumedReadWriteOps.Maximum'][0]['Maximum'] }}</td> | |
| {% elif columnName == 'PublicIp' %} | |
| <td style="{{ tdpart }}">{{ resource['NetworkInterfaces'][0].get('Association')['PublicIp'] }}</td> | |
| {% else %} | |
| <td style="{{ tdpart }}">{{ resource[columnName] }}</td> | |
| {% endif %} | |
| {% endfor %} | |
| </tr> | |
| {%- endmacro %} | |
| {# The macro below creates the table: | |
| Formatting can be dependent on the column names that are passed in | |
| #} | |
| {% macro columnData(resources, columnNames) -%} | |
| {% set len = resources|length %} | |
| {% for resource in resources %} | |
| {% set idx = loop.index %} | |
| {{ tableRow(resource, columnNames, idx, len) }} | |
| {% endfor %} | |
| </table> | |
| {%- endmacro %} | |
| {# Main #} | |
| {% macro createTable(columnNames, resources, tableWidth) %} | |
| {{ columnHeading(columnNames, tableWidth) }} | |
| {{ columnData(resources, columnNames) }} | |
| {%- endmacro %} | |
| <head> | |
| <title>Cloud Custodian Notification - {{ "%s - %s" | format(account,region) }}</title> | |
| </head> | |
| <body> | |
| <h2><font color="#505151"> {{ "%s - %s" | format(account,region) }} </h2> | |
| {% if action['action_desc'] %} | |
| <h3> {{ action['violation_desc'] }} and <strong>{{ action['action_desc'] }}</strong>:</h3> | |
| {% else %} | |
| <h3> {{ action['violation_desc'] }}:</h3> | |
| {% endif %} | |
| {# Below, notifications for any resource-type can be formatted with specific columns #} | |
| {% if policy['resource'] == "ami" %} | |
| {% set columnNames = ['Name','ImageId','CreationDate'] %} | |
| {{ createTable(columnNames, resources, '60') }} | |
| {% elif policy['resource'] == "app-elb" %} | |
| {% set columnNames = ['LoadBalancerName','CreatedTime','Project','Component','Environment','OwnerEmail'] %} | |
| {{ createTable(columnNames, resources, '80') }} | |
| {% elif policy['resource'] == "asg" %} | |
| {% if resources[0]['Invalid'] is defined %} | |
| {% set columnNames = ['AutoScalingGroupName','InstanceCount','Invalid'] %} | |
| {% else %} | |
| {% set columnNames = ['AutoScalingGroupName','InstanceCount','Project','Component','Environment','OwnerEmail'] %} | |
| {% endif %} | |
| {{ createTable(columnNames, resources, '60') }} | |
| {% elif policy['resource'] == "cache-cluster" %} | |
| {% set columnNames = ['CacheClusterId','CacheClusterCreateTime','CacheClusterStatus','Project','Component','Environment','OwnerEmail'] %} | |
| {{ createTable(columnNames, resources, '80') }} | |
| {% elif policy['resource'] == "cache-snapshot" %} | |
| {% set columnNames = ['SnapshotName','CacheClusterId','SnapshotSource','Project','Component','Environment','OwnerEmail'] %} | |
| {{ createTable(columnNames, resources, '80') }} | |
| {% elif policy['resource'] == "cfn" %} | |
| {% set columnNames = ['StackName'] %} | |
| {{ createTable(columnNames, resources, '50') }} | |
| {% elif policy['resource'] == "cloudsearch" %} | |
| {% set columnNames = ['DomainName'] %} | |
| {{ createTable(columnNames, resources, '50') }} | |
| {% elif policy['resource'] == "ebs" %} | |
| {% set columnNames = ['VolumeId','CreateTime','State','Project','Component','Environment','OwnerEmail'] %} | |
| {{ createTable(columnNames, resources, '50') }} | |
| {% elif policy['resource'] == "ebs-snapshot" %} | |
| {% set columnNames = ['SnapshotId','StartTime','Project','Component','Environment','OwnerEmail'] %} | |
| {{ createTable(columnNames, resources, '80') }} | |
| {% elif policy['resource'] == "ec2" %} | |
| {% if resources[0]['MatchedFilters'] == ['PublicIpAddress'] %} | |
| {% set columnNames = ['tag.Name','PublicIp','InstanceId','ImageId','Project','Component','Environment','OwnerEmail','LaunchTime','aws:autoscaling:groupName'] %} | |
| {% else %} | |
| {% set columnNames = ['tag.Name','PrivateIpAddress','InstanceId','ImageId','Project','Component','Environment','OwnerEmail','LaunchTime','aws:autoscaling:groupName'] %} | |
| {% endif %} | |
| {{ createTable(columnNames, resources, '80') }} | |
| {% elif policy['resource'] == "efs" %} | |
| {% set columnNames = ['CreationToken','CreationTime','FileSystemId','OwnerId'] %} | |
| {{ createTable(columnNames, resources, '50') }} | |
| {% elif policy['resource'] == "elasticsearch" %} | |
| {% set columnNames = ['DomainName','Endpoint'] %} | |
| {{ createTable(columnNames, resources, '50') }} | |
| {% elif policy['resource'] == "elb" %} | |
| {% set columnNames = ['LoadBalancerName','InstanceCount','AvailabilityZones','Project','Component','Environment','OwnerEmail'] %} | |
| {{ createTable(columnNames, resources, '80') }} | |
| {% elif policy['resource'] == "emr" %} | |
| {% set columnNames = ['Id','EmrState'] %} | |
| {{ createTable(columnNames, resources, '50') }} | |
| {% elif policy['resource'] == "kinesis" %} | |
| {% set columnNames = ['KinesisName'] %} | |
| {{ createTable(columnNames, resources, '50') }} | |
| {% elif policy['resource'] == "launch-config" %} | |
| {% set columnNames = ['LaunchConfigurationName'] %} | |
| {{ createTable(columnNames, resources, '30') }} | |
| {% elif policy['resource'] == "log-group" %} | |
| {% set columnNames = ['logGroupName'] %} | |
| {{ createTable(columnNames, resources, '30') }} | |
| {% elif policy['resource'] == "rds" %} | |
| {% if resources[0]['PubliclyAccessible'] == true or resources[0]['StorageEncrypted'] == false %} | |
| {% set columnNames = ['DBInstanceIdentifier','PubliclyAccessible','StorageEncrypted','DBSubnetGroup'] %} | |
| {% else %} | |
| {% set columnNames = ['DBInstanceIdentifier','Project','Component','Environment','OwnerEmail'] %} | |
| {% endif %} | |
| {{ createTable(columnNames, resources, '80') }} | |
| {% elif policy['resource'] == "rds-snapshot" %} | |
| {% set columnNames = ['DBSnapshotIdentifier','SnapshotCreateTime','DBInstanceIdentifier','SnapshotType','Project','Component','Environment','OwnerEmail'] %} | |
| {{ createTable(columnNames, resources, '80') }} | |
| {% elif policy['resource'] == "redshift" %} | |
| {% if resources[0]['PubliclyAccessible'] == true or resources[0]['Encrypted'] == false %} | |
| {% set columnNames = ['ClusterIdentifier','NodeCount','PubliclyAccessible','Encrypted'] %} | |
| {% else %} | |
| {% set columnNames = ['ClusterIdentifier','NodeCount','Project','Component','Environment','OwnerEmail'] %} | |
| {% endif %} | |
| {{ createTable(columnNames, resources, '80') }} | |
| {% elif policy['resource'] == "redshift-snapshot" %} | |
| {% set columnNames = ['SnapshotIdentifier','DBName','Project','Component','Environment','OwnerEmail'] %} | |
| {{ createTable(columnNames, resources, '80') }} | |
| {% elif policy['resource'] == "s3" %} | |
| {% if resources[0]['GlobalPermissions'] is defined %} | |
| {% set columnNames = ['Name','GlobalPermissions'] %} | |
| {% else %} | |
| {% set columnNames = ['Name','Project','Component','Environment','OwnerEmail'] %} | |
| {% endif %} | |
| {{ createTable(columnNames, resources, '80') }} | |
| {% elif policy['resource'] == "security-group" %} | |
| {% set columnNames = ['GroupName','tag.Name','GroupId','VpcId'] %} | |
| {{ createTable(columnNames, resources, '80') }} | |
| {% elif policy['resource'] == "simpledb" %} | |
| {% set columnNames = ['DomainName'] %} | |
| {{ createTable(columnNames, resources, '60') }} | |
| {# If no special formatting is defined for a resource type, all attributes will be formatted in the email #} | |
| {% else %} | |
| {% set columnNames = resources[0].keys() %} | |
| {{ createTable(columnNames, resources, '100') }} | |
| {% endif %} | |
| <h4> | |
| For any other questions, contact <a href="mailto:{{ action['questions_email'] }}">{{ action['questions_email'] }}</a> | |
| or <a href="https://example.slack.com/messages/{{ action['questions_slack'] }}/">#{{ action['questions_slack'] }} on Slack</a>. | |
| Documentation for our cloud-custodian instance and policies can be found <a href="https://ghe.example.com/MyOrg/custodian-config/blob/master/README.md">in the README on Github</a>. | |
| </h4> | |
| <p>Generated by cloud-custodian policy: {{ policy['name'] }}</p> | |
| </body> | |
| </html> |
