Skip to content

Instantly share code, notes, and snippets.

@duwerq

duwerq/PostDeploymentElasticSearch.json

Last active August 22, 2023 23:18
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 2 You must be signed in to fork a gist
  • Save duwerq/3483dcbeadd2b7a38d1e488763547547 to your computer and use it in GitHub Desktop.
Save duwerq/3483dcbeadd2b7a38d1e488763547547 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
PostDeploymentElasticSearch.json
"ConfigureESCustom": {
"Type": "Custom::ConfigureES",
"Properties": {
"ServiceToken": {
"Fn::GetAtt": ["ConfigureES", "Arn"]
}
}
},
"ConfigureES": {
"Type": "AWS::Lambda::Function",
"Properties": {
"Environment": {
"Variables": {
"ES_ENDPOINT": {
"Fn::ImportValue": {
"Fn::Join": [
":",
[
{
"Ref": "AppSyncApiId"
},
"GetAtt",
"Elasticsearch",
"DomainEndpoint"
]
]
}
},
"ES_REGION": {
"Ref": "AWS::Region"
}
}
},
"Code": {
"ZipFile": {
"Fn::Join": [
"\n",
[
"import base64",
"import json",
"import logging",
"import string",
"import boto3",
"import os",
"import time",
"import datetime",
"import traceback",
"from urllib.parse import urlparse, quote",
"from botocore.vendored import requests",
"from botocore.auth import SigV4Auth",
"from botocore.awsrequest import AWSRequest",
"from botocore.credentials import get_credentials",
"from botocore.httpsession import URLLib3Session",
"from botocore.session import Session",
"import cfnresponse",
" ",
"logger = logging.getLogger()",
"logger.setLevel(logging.INFO)",
" ",
"# The following parameters are required to configure the ES cluster",
"ES_ENDPOINT = os.environ['ES_ENDPOINT']",
"ES_REGION = os.environ['ES_REGION']",
"# DEBUG = True if os.environ['DEBUG'] is not None else False",
" ",
"def es_put(payload, region, creds, host, path, method='PUT', proto='https://'):",
" # Put index data to ES endpoint with SigV4 signed http headers ",
" req = AWSRequest(method=method, url=proto + host + quote(path), data=payload, headers={'Host': host, 'Content-Type': 'application/json'})",
" SigV4Auth(creds, 'es', region).add_auth(req)",
" http_session = URLLib3Session()",
" res = http_session.send(req.prepare())",
" return res._content",
" ",
"def lambda_handler(event, context):",
" logger.info('got event {}'.format(event))",
" if event['RequestType'] == 'Create':",
" # Get aws_region and credentials to post signed URL to ES",
" es_region = ES_REGION or os.environ['AWS_REGION']",
" session = Session()",
" creds = get_credentials(session)",
" es_url = urlparse(ES_ENDPOINT)",
" # Extract the domain name in ES_ENDPOINT",
" es_endpoint = es_url.netloc or es_url.path",
" es_put('', es_region, creds,es_endpoint, '/')",
" action = {\"mappings\": {\"doc\": {\"properties\": {\"gps\": {\"type\": \"geo_point\"}}}}}",
" es_payload = json.dumps(action)",
" es_put(es_payload, es_region, creds, es_endpoint, '/establishment')",
" cfnresponse.send(event, context, cfnresponse.SUCCESS, {})"
]
]
}
},
"FunctionName": {
"Fn::Join": [
"-",
[
"ConfigureES",
{
"Ref": "env"
}
]
]
},
"Handler": "index.lambda_handler",
"Timeout": 30,
"Role": {
"Fn::GetAtt": ["LambdaRole", "Arn"]
},
"Runtime": "python3.6",
"Layers": [
"arn:aws:lambda:us-east-1:668099181075:layer:AWSLambda-Python-AWS-SDK:4"
]
}
},
"LambdaRole": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Service": ["lambda.amazonaws.com"]
},
"Action": ["sts:AssumeRole"]
}
]
},
"Path": "/",
"Policies": [
{
"PolicyName": "lambda-logs",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"logs:CreateLogGroup",
"logs:CreateLogStream",
"logs:PutLogEvents"
],
"Resource": ["arn:aws:logs:*:*:*"]
}
]
}
},
{
"PolicyName": "ES",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["es:*"],
"Resource": {
"Fn::Join": [
"",
[
{
"Fn::ImportValue": {
"Fn::Join": [
":",
[
{
"Ref": "AppSyncApiId"
},
"GetAtt",
"Elasticsearch",
"DomainArn"
]
]
}
},
"/*"
]
]
}
}
]
}
}
]
}
}
@d99joper
Copy link

d99joper commented Mar 6, 2023

I used the one above with some adjustments like:

-- hardcoded the endpoint
"Environment": {
"Variables": {
"ES_ENDPOINT": "https://search-amplify-opense-{my open search domain id}-{my app id}.us-west-1.es.amazonaws.com/",

"# The following parameters are required to configure the ES cluster",
"ES_ENDPOINT = os.environ['OPENSEARCH_ENDPOINT']",
"ES_REGION = os.environ['OPENSEARCH_REGION']",

-- because my region is us-west-1
"Runtime": "python3.7",
"Layers": [
"arn:aws:lambda:us-west-1:325793726646:layer:AWSLambda-Python-AWS-SDK:4"
]

-- hardcoded the resource
"PolicyName": "ES",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["es:"],
"Resource": "arn:aws:es:us-west-1:{my app id}:domain/amplify-opense-{my open search domain id}/"
}
]
}

@nairoldi
Copy link

How do we create the python layer ?

@d99joper
Copy link

How do we create the python layer ?

This helped me: https://aws.amazon.com/blogs/compute/upcoming-changes-to-the-python-sdk-in-aws-lambda/

@nairoldi
Copy link

How do we create the python layer ?

This helped me: https://aws.amazon.com/blogs/compute/upcoming-changes-to-the-python-sdk-in-aws-lambda/

Thanks @d99joper going to try this out , i was able to get it to push without the layer in there but the query isnt working so I am assuming this is why.

@ok-martin
Copy link

Hey @d99joper thank you! I got this working now. One note is that I had to change the python lambda to this:

action = {\"properties\": {\"gps\": {\"type\": \"geo_point\"}}}
es_payload = json.dumps(action)
es_put(es_payload, es_region, creds, es_endpoint, '/establishment/_mapping')

I will post a complete update once I confirm everything is working ok

@amondnet
Copy link

amondnet commented Apr 23, 2023 •
edited

If you are using opensearch, you will need to modify it as follows.

{
  "Resources": {
    "ConfigureESCustom": {
      "Type": "Custom::ConfigureES",
      "Properties": {
        "ServiceToken": {
          "Fn::GetAtt": [
            "ConfigureES",
            "Arn"
          ]
        }
      }
    },
    "ConfigureES": {
      "Type": "AWS::Lambda::Function",
      "Properties": {
        "Environment": {
          "Variables": {
            "ES_ENDPOINT": {
              "Fn::ImportValue": {
                "Fn::Join": [
                  ":",
                  [
                    {
                      "Ref": "AppSyncApiId"
                    },
                    "GetAtt",
                    "OpenSearch",
                    "DomainEndpoint"
                  ]
                ]
              }
            },
            "ES_REGION": {
              "Ref": "AWS::Region"
            }
          }
        },
        "Code": {
          "ZipFile": {
            "Fn::Join": [
              "\n",
              [
                "import base64",
                "import json",
                "import logging",
                "import string",
                "import boto3",
                "import os",
                "import time",
                "import datetime",
                "import traceback",
                "from urllib.parse import urlparse, quote",
                "from botocore.vendored import requests",
                "from botocore.auth import SigV4Auth",
                "from botocore.awsrequest import AWSRequest",
                "from botocore.credentials import get_credentials",
                "from botocore.httpsession import URLLib3Session",
                "from botocore.session import Session",
                "import cfnresponse",
                " ",
                "logger = logging.getLogger()",
                "logger.setLevel(logging.INFO)",
                " ",
                "# The following parameters are required to configure the ES cluster",
                "ES_ENDPOINT = os.environ['ES_ENDPOINT']",
                "ES_REGION = os.environ['ES_REGION']",
                "# DEBUG = True if os.environ['DEBUG'] is not None else False",
                " ",
                "def es_put(payload, region, creds, host, path, method='PUT', proto='https://'):",
                "    # Put index data to ES endpoint with SigV4 signed http headers ",
                "    req = AWSRequest(method=method, url=proto + host +    quote(path), data=payload, headers={'Host': host, 'Content-Type': 'application/json'})",
                "    SigV4Auth(creds, 'es', region).add_auth(req)",
                "    http_session = URLLib3Session()",
                "    res = http_session.send(req.prepare())",
                "    return res._content",
                " ",
                "def lambda_handler(event, context):",
                "    logger.info('got event {}'.format(event))",
                "    if event['RequestType'] == 'Create':",
                "        # Get aws_region and credentials to post signed URL to ES",
                "        es_region = ES_REGION or os.environ['AWS_REGION']",
                "        session = Session()",
                "        creds = get_credentials(session)",
                "        es_url = urlparse(ES_ENDPOINT)",
                "        # Extract the domain name in ES_ENDPOINT",
                "        es_endpoint = es_url.netloc or es_url.path",
                "        es_put('', es_region, creds,es_endpoint, '/')",
                "        action =  {\"properties\": {\"gps\": {\"type\": \"geo_point\"}}}",
                "        es_payload = json.dumps(action)",
                "        es_put(es_payload, es_region, creds, es_endpoint, '/establishment/_mapping')",
                "        cfnresponse.send(event, context, cfnresponse.SUCCESS, {})"
              ]
            ]
          }
        },
        "FunctionName": {
          "Fn::Join": [
            "-",
            [
              "ConfigureES",
              {
                "Ref": "env"
              }
            ]
          ]
        },
        "Handler": "index.lambda_handler",
        "Timeout": 30,
        "Role": {
          "Fn::GetAtt": [
            "LambdaRole",
            "Arn"
          ]
        },
        "Runtime": "python3.7",
        "Layers": [
          "arn:aws:lambda:ap-northeast-2:296580773974:layer:AWSLambda-Python-AWS-SDK:4"
        ]
      }
    },
    "LambdaRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Effect": "Allow",
              "Principal": {
                "Service": [
                  "lambda.amazonaws.com"
                ]
              },
              "Action": [
                "sts:AssumeRole"
              ]
            }
          ]
        },
        "Path": "/",
        "Policies": [
          {
            "PolicyName": "lambda-logs",
            "PolicyDocument": {
              "Version": "2012-10-17",
              "Statement": [
                {
                  "Effect": "Allow",
                  "Action": [
                    "logs:CreateLogGroup",
                    "logs:CreateLogStream",
                    "logs:PutLogEvents"
                  ],
                  "Resource": [
                    "arn:aws:logs:*:*:*"
                  ]
                }
              ]
            }
          },
          {
            "PolicyName": "ES",
            "PolicyDocument": {
              "Version": "2012-10-17",
              "Statement": [
                {
                  "Effect": "Allow",
                  "Action": [
                    "es:*"
                  ],
                  "Resource": {
                    "Fn::Join": [
                      "",
                      [
                        {
                          "Fn::ImportValue": {
                            "Fn::Join": [
                              ":",
                              [
                                {
                                  "Ref": "AppSyncApiId"
                                },
                                "GetAtt",
                                "OpenSearch",
                                "DomainArn"
                              ]
                            ]
                          }
                        },
                        "/*"
                      ]
                    ]
                  }
                }
              ]
            }
          }
        ]
      }
    }
  }
}

@majirosstefan
Copy link

nice

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment