Cloudformation Template for Migration to AWS OpenSearch Serverless Collection

opensearch-serverless-migration.template.yaml

AWSTemplateFormatVersion: '2010-09-09'
Description: An EC2 instance used to run a logstash batch job

Parameters:
  KeyName:
    Description: Key pair for SSH access
    Type: AWS::EC2::KeyPair::KeyName
    ConstraintDescription: Must be a valid EC2 key pair name
  SecurityGroups:
    Type: List<AWS::EC2::SecurityGroup::Id>
    Description: List of security groups
    ConstraintDescription: Must be a valid Security Group ID
  Subnet:
    Type: AWS::EC2::Subnet::Id
    Description: Subnet for Instance
  SourceEndpoint:
    Type: String
    Description: The source endpoint
  TargetEndpoint:
    Type: String
    Description: The target endpoint
  InstanceProfile:
    Description: IAM Profile Name for the service instances
    Type: String
Resources:
  EC2Instance:
    DependsOn:
      - IAMUserAccessKey
    Type: AWS::EC2::Instance
    Properties:
      Tags:
        - Key: Name
          Value: !Sub ${AWS::StackName}
      InstanceType: t2.large
      IamInstanceProfile: !Ref InstanceProfile
      KeyName: !Ref KeyName
      SecurityGroupIds: !Ref SecurityGroups
      ImageId: ami-04376654933b081a7
      SubnetId: !Ref Subnet
      UserData:
        Fn::Base64:
          !Sub |
            #!/bin/bash -xe
            yum update -y
            yum install -y aws-cfn-bootstrap
  
            /opt/aws/bin/cfn-init --region ${AWS::Region} --stack ${AWS::StackName} --resource EC2Instance --configsets install
            /opt/aws/bin/cfn-signal --exit-code $? --stack ${AWS::StackName} --resource EC2Instance

    Metadata:
      AWS::CloudFormation::Init:
        configSets:
          install:
            - setup
            - install_and_configure_logstash
            - do_migration
          update:
            - install_and_configure_logstash
            - do_migration
        setup:
          files:
            /etc/cfn/cfn-hup.conf:
              content: !Sub |
                [main]
                stack=${AWS::StackId}
                region=${AWS::Region}
                verbose=true
                interval=1
              mode: "000400"
              owner: root
              group: root

            /etc/cfn/hooks.d/cfn-auto-reloader.conf:
              content: !Sub |
                [cfn-auto-reloader-hook]
                triggers=post.update
                path=Resources.EC2Instance.Metadata.AWS::CloudFormation::Init
                action=/opt/aws/bin/cfn-init -v --stack ${AWS::StackName} --resource EC2Instance --configsets update --region ${AWS::Region}
              mode: '000400'
              owner: root
              group: root

            /lib/systemd/system/cfn-hup.service:
              content: |
                [Unit]
                Description=cfn-hup daemon
                
                [Service]
                Type=simple
                ExecStart=/opt/aws/bin/cfn-hup
                Restart=always
                
                [Install]
                WantedBy=multi-user.target
              mode: "000644"
              owner: root
              group: root
          commands:
            010_enable_cfn-hup:
              command: "systemctl enable cfn-hup.service"
            020_start_cfn-hup:
              command: "systemctl restart cfn-hup.service"
        install_and_configure_logstash:
          commands:
            01_install_logstash:
              command: !Sub |
                #!/bin/bash -xe
                wget https://artifacts.opensearch.org/logstash/logstash-oss-with-opensearch-output-plugin-8.9.0-linux-x64.tar.gz
                tar -zxvf logstash-oss-with-opensearch-output-plugin-8.9.0-linux-x64.tar.gz
                rm logstash-oss-with-opensearch-output-plugin-8.9.0-linux-x64.tar.gz
                cd logstash-8.9.0/
                /logstash-8.9.0/bin/logstash-plugin update logstash-output-opensearch
                /logstash-8.9.0/bin/logstash-plugin install logstash-input-opensearch
                
                SECRET_KEY=${IAMUserAccessKey.SecretAccessKey}
                
                cat > migrate.conf << EOF
                input {
                    opensearch {
                        hosts =>  ["https://${SourceEndpoint}:443"]
                        index =>  "movies"
                        query =>  '{ "query": { "match_all": {} } }'
                    }
                }
                output {
                    opensearch {
                        ecs_compatibility => disabled
                        index => "movies"
                        hosts => "${TargetEndpoint}:443"
                        auth_type => {
                            type => 'aws_iam'
                            aws_access_key_id => '${IAMUserAccessKey}'
                            aws_secret_access_key => '$SECRET_KEY'
                            region => 'eu-central-1'
                            service_name => 'aoss'
                        }
                        legacy_template => false
                        default_server_major_version => 2
                    }
                }
                EOF
        do_migration:
          commands:
            01_logstash_run:
              command: !Sub |
                #!/bin/bash -xe
                
                /logstash-8.9.0/bin/logstash --log.level=info -f /logstash-8.9.0/migrate.conf 
                

  IAMUserAccessKey:
    Type: AWS::IAM::AccessKey
    Properties:
      UserName: aossadmin
      Serial: 6

Outputs:
  EC2InstanceId:
    Description: EC2 instance ID
    Value: !Ref EC2Instance

  EC2InstancePrivateIp:
    Description: Private IP address of the EC2 instance
    Value: !GetAtt EC2Instance.PrivateIp