AWS DataSync- On-Premises Region

datasync-onprem.yaml

AWSTemplateFormatVersion: '2010-09-09'
Description: AWS DataSync- On-Premises Region
Metadata:
  AWS::CloudFormation::Interface:
    ParameterGroups:
      - Label:
          default: Security
        Parameters:
          - kp
      - Label:
          default: AMI IDs (do not edit)
        Parameters:
          - linuxAmi
          - dsAmi
    ParameterLabels:
      kp:
        default: 'Access Key Pair:'
      linuxAmi:
        default: 'Linux'
      dsAmi:
        default: 'DataSync'

Parameters:
  kp:
    Description: '(NOTE: If you don''t see a key in the list you will need to create
      one from the EC2 console in this region)'
    Type: AWS::EC2::KeyPair::KeyName
  linuxAmi:
    Type : 'AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>'
    Default: '/aws/service/ami-amazon-linux-latest/amzn2-ami-hvm-x86_64-gp2'
  dsAmi:
    Type : 'AWS::SSM::Parameter::Value<AWS::EC2::Image::Id>'
    Default: '/aws/service/datasync/ami'

Resources:

  # Create a dedicated VPC with internet connectivity
  dmVPC:
    Type: AWS::EC2::VPC
    Properties:
      CidrBlock: 10.12.14.0/24
      EnableDnsSupport: 'true'
      EnableDnsHostnames: 'true'
      InstanceTenancy: default
      Tags:
      - Key: Name
        Value: DsSmallFilesNfsWorkshopVPC
  dmSubnet1:
    Type: AWS::EC2::Subnet
    Properties:
      VpcId: !Ref 'dmVPC'
      CidrBlock: 10.12.14.0/24
      MapPublicIpOnLaunch: 'True'
      Tags:
      - Key: Name
        Value: DsSmallFilesNfsWorkshopSubnet1
  dmInternetGateway:
    Type: AWS::EC2::InternetGateway
    Properties:
      Tags:
      - Key: Name
        Value: DsSmallFilesNfsWorkshopIGW
  dmAttachGateway:
    Type: AWS::EC2::VPCGatewayAttachment
    Properties:
      VpcId: !Ref 'dmVPC'
      InternetGatewayId: !Ref 'dmInternetGateway'
  dmRouteTable:
    Type: AWS::EC2::RouteTable
    Properties:
      VpcId: !Ref 'dmVPC'
      Tags:
      - Key: Name
        Value: DsSmallFilesNfsWorkshopRouteTable
  dmSubnet1RouteAssociaton:
    Type: AWS::EC2::SubnetRouteTableAssociation
    Properties:
      SubnetId: !Ref 'dmSubnet1'
      RouteTableId: !Ref 'dmRouteTable'
  dmRoutetoInternet:
    Type: AWS::EC2::Route
    DependsOn: dmInternetGateway
    Properties:
      RouteTableId: !Ref 'dmRouteTable'
      DestinationCidrBlock: 0.0.0.0/0
      GatewayId: !Ref 'dmInternetGateway'

  dmSecurityGroup:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: DataSync Small Files NFS Workshop - Security Group for all resources
      VpcId: !Ref 'dmVPC'
      SecurityGroupIngress:
        - IpProtocol: tcp
          FromPort: '22'
          ToPort: '22'
          CidrIp: '0.0.0.0/0'
        - IpProtocol: tcp
          FromPort: '80'
          ToPort: '80'
          CidrIp: '0.0.0.0/0'

  # We use this so we can limit access on this port to the SG
  dmSecurityGroupIngress:
    Type: AWS::EC2::SecurityGroupIngress
    Properties:
      GroupId: !Ref 'dmSecurityGroup'
      IpProtocol: tcp
      ToPort: '2049'
      FromPort: '2049'
      SourceSecurityGroupId: !Ref 'dmSecurityGroup'
    DependsOn: dmSecurityGroup

  nfsServerInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles:
        - !Ref 'nfsServerIamRole'
  nfsServerIamRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
          - Action:
              - sts:AssumeRole
            Effect: Allow
            Principal:
              Service:
                - ec2.amazonaws.com
        Version: '2012-10-17'
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
  nfsServerRolePolicy:
    Type: AWS::IAM::Policy
    Properties:
      PolicyDocument:
        Statement:
          - Effect: Allow
            Action:
              - s3:GetObject
            Resource:
              - arn:aws:s3:::aws-datasync-samples/files/fs1m-small.tar
              - arn:aws:s3:::aws-datasync-samples/files/fs500k-small.tar
        Version: '2012-10-17'
      PolicyName: policy
      Roles:
        - !Ref 'nfsServerIamRole'
  nfsServer:
    Type: AWS::EC2::Instance
    CreationPolicy:
      ResourceSignal:
        Timeout: PT30M
        Count: 1
    Metadata:
      AWS::CloudFormation::Init:
        config:
          files:
            /opt/aws/cfn/initdata.sh:
              content: !Sub |
                #!/bin/bash
                mkdir -p /mnt/fs1 /mnt/fs2 /mnt/fs3
                devs=`lsblk | grep 200G | awk '{ print $1 }'`
                fsnum=1
                for d in $devs
                do
                  mkfs.xfs /dev/$d
                  mount /dev/$d /mnt/fs$fsnum
                  fsnum=$(( fsnum + 1 ))
                done
                chmod 777 /mnt/fs*
                aws s3 cp s3://aws-datasync-samples/files/fs500k-small.tar /mnt/fs1/fs500k-small.tar >> /dev/null
                aws s3 cp s3://aws-datasync-samples/files/fs500k-small.tar /mnt/fs2/fs500k-small.tar >> /dev/null
                aws s3 cp s3://aws-datasync-samples/files/fs1m-small.tar /mnt/fs3/fs1m-small.tar >> /dev/null
                cd /mnt/fs1/ && tar -xf fs500k-small.tar &
                cd /mnt/fs2/ && tar -xf fs500k-small.tar &
                cd /mnt/fs3/ && tar -xf fs1m-small.tar &
                wait
                rm -f /mnt/fs1/fs500k-small.tar /mnt/fs2/fs500k-small.tar /mnt/fs3/fs1m-small.tar
                echo READY > /home/ec2-user/datasets_ready
          commands:
            1-initData:
              command: "bash /opt/aws/cfn/initdata.sh"
            2-finishUp:
              command: !Sub "/opt/aws/bin/cfn-signal -e 0 --stack ${AWS::StackId} --resource nfsServer --region ${AWS::Region}"
    Properties:
      ImageId: !Ref linuxAmi
      InstanceType: m5.large
      IamInstanceProfile: !Ref 'nfsServerInstanceProfile'
      Tags:
        - Key: Name
          Value: !Join
            - ''
            - - NfsServer-
              - !Ref 'AWS::StackName'
      KeyName: !Ref 'kp'
      InstanceInitiatedShutdownBehavior: terminate
      BlockDeviceMappings:
        - DeviceName: /dev/xvda
          Ebs:
            VolumeSize: '8'
            DeleteOnTermination: 'true'
            VolumeType: gp2
        - DeviceName: /dev/xvdb
          Ebs:
            VolumeSize: '200'
            DeleteOnTermination: 'true'
            VolumeType: gp2
        - DeviceName: /dev/xvdc
          Ebs:
            VolumeSize: '200'
            DeleteOnTermination: 'true'
            VolumeType: gp2
        - DeviceName: /dev/xvdd
          Ebs:
            VolumeSize: '200'
            DeleteOnTermination: 'true'
            VolumeType: gp2
      NetworkInterfaces:
        - AssociatePublicIpAddress: 'true'
          DeviceIndex: '0'
          GroupSet:
            - !Ref 'dmSecurityGroup'
          SubnetId: !Ref 'dmSubnet1'
      UserData:
        Fn::Base64:
          !Sub |
            #!/bin/bash
            /opt/aws/bin/cfn-init -v -s ${AWS::StackId} -r nfsServer --region ${AWS::Region}

  dataSyncAgentInstanceProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Path: /
      Roles:
        - !Ref 'dataSyncAgentIamRole'
  dataSyncAgentIamRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Statement:
          - Action:
              - sts:AssumeRole
            Effect: Allow
            Principal:
              Service:
                - ec2.amazonaws.com
        Version: '2012-10-17'
  dataSyncAgentRolePolicy:
    Type: AWS::IAM::Policy
    Properties:
      PolicyDocument:
        Statement:
          - Effect: Allow
            Action:
              - datasync:*
            Resource:
              - '*'
          - Effect: Allow
            Action:
              - iam:PassRole
            Resource:
              - '*'
        Version: '2012-10-17'
      PolicyName: policy
      Roles:
        - !Ref 'dataSyncAgentIamRole'

  dataSyncAgent1:
    Type: AWS::EC2::Instance
    Properties:
      ImageId: !Ref dsAmi
      InstanceType: m5.large
      IamInstanceProfile: !Ref 'dataSyncAgentInstanceProfile'
      Tags:
        - Key: Name
          Value: !Join
            - ''
            - - DataSyncAgent1-
              - !Ref 'AWS::StackName'
      KeyName: !Ref 'kp'
      InstanceInitiatedShutdownBehavior: stop
      BlockDeviceMappings:
        - DeviceName: /dev/xvda
          Ebs:
            VolumeSize: '80'
            DeleteOnTermination: 'true'
            VolumeType: gp2
      NetworkInterfaces:
        - AssociatePublicIpAddress: 'true'
          DeviceIndex: '0'
          GroupSet:
            - !Ref 'dmSecurityGroup'
          SubnetId: !Ref 'dmSubnet1'

  dataSyncAgent2:
    Type: AWS::EC2::Instance
    Properties:
      ImageId: !Ref dsAmi
      InstanceType: m5.large
      IamInstanceProfile: !Ref 'dataSyncAgentInstanceProfile'
      Tags:
        - Key: Name
          Value: !Join
            - ''
            - - DataSyncAgent2-
              - !Ref 'AWS::StackName'
      KeyName: !Ref 'kp'
      InstanceInitiatedShutdownBehavior: stop
      BlockDeviceMappings:
        - DeviceName: /dev/xvda
          Ebs:
            VolumeSize: '80'
            DeleteOnTermination: 'true'
            VolumeType: gp2
      NetworkInterfaces:
        - AssociatePublicIpAddress: 'true'
          DeviceIndex: '0'
          GroupSet:
            - !Ref 'dmSecurityGroup'
          SubnetId: !Ref 'dmSubnet1'

Outputs:
  nfsServerPrivateIP:
    Description: NFS Server Private IP Address
    Value: !GetAtt nfsServer.PrivateIp
  dataSyncAgent1PublicIP:
    Description: DataSync Agent 1 - Public IP Address
    Value: !GetAtt dataSyncAgent1.PublicIp
  dataSyncAgent1PrivateIP:
    Description: DataSync Agent 1 - Private IP Address
    Value: !GetAtt dataSyncAgent1.PrivateIp
  dataSyncAgent2PublicIP:
    Description: DataSync Agent 2 - Public IP Address
    Value: !GetAtt dataSyncAgent2.PublicIp
  dataSyncAgent2PrivateIP:
    Description: DataSync Agent 2 - Private IP Address
    Value: !GetAtt dataSyncAgent2.PrivateIp