deploy s3 & IAM policy & snowflake external volume for creating iceberg tables

deploy_steps.md

1. Apply the terraform plan.

   1. It'll create AWS resources.
   2. It'll create the snowflake external volume.

2. Obtain the snowflake user & session credentials.

3. Re-apply terraform but this time only for the policy resource aws_iam_role.my_iceberg_sf_access_role.

   1. This will update the trust of your IAM role to allow the snowflake user RW access to the S3 bucket.

s3_iam.tf

# s3.tf
resource "aws_s3_bucket" "my_iceberg" {
  bucket = "my-iceberg"
}

output "bucket_name" {
  value = aws_s3_bucket.my_iceberg.bucket
}

# iam.tf
resource "aws_iam_policy" "my_iceberg_sf_access" {
  name        = "my_iceberg_sf_access"
  description = "policy for Snowflake access to S3 bucket and folder for iceberg tables"

  policy = jsonencode({
    Version   = "2012-10-17",
    Statement = [
      {
        Effect = "Allow",
        Action = [
          "s3:PutObject",
          "s3:GetObject",
          "s3:GetObjectVersion",
          "s3:DeleteObject",
          "s3:DeleteObjectVersion",
        ],
        Resource = "${aws_s3_bucket.my_iceberg.arn}/*",
      },
      {
        Effect = "Allow",
        Action = [
          "s3:ListBucket",
          "s3:GetBucketLocation",
        ],
        Resource = aws_s3_bucket.my_iceberg.arn,
      }
    ]
  })
}

resource "aws_iam_role" "my_iceberg_sf_access_role" {
  name               = "my_iceberg_sf_access_role"
  assume_role_policy = jsonencode({
    Version   = "2012-10-17",
    Statement = [
      merge(
        {
          Action = "sts:AssumeRole",
          Effect = "Allow",

          Condition = {
            StringEquals = {
              "sts:ExternalId" = var.sf_user_extern_id
            }
          }
        },
        coalesce(var.sf_user_extern_arn == null ?
        {
          Principal = {
            Service = "s3.amazonaws.com"
          }
        } : {
          Principal = {
            AWS = var.sf_user_extern_arn
          }

        })
      )
    ],
  })
}


# Attach the policy created in Step 1
resource "aws_iam_role_policy_attachment" "s3_access_attachment" {
  policy_arn = aws_iam_policy.my_iceberg_sf_access.arn
  role       = aws_iam_role.my_iceberg_sf_access_role.name
}


output "policy_arn" {
  value = aws_iam_policy.my_iceberg_sf_access.arn
}
output "role_arn" {
  value = aws_iam_role.my_iceberg_sf_access_role.arn
}

sf_extern_vol.tf

provider "snowflake" {
  role = "ACCOUNTADMIN"
}

locals {
  volume_name = "my-iceberg"
}

locals {
  volume_name_sf_identifier = replace(local.volume_name, "-", "_")
}

resource "snowflake_unsafe_execute" "iceberg_volume" {
  execute = <<EOT
CREATE OR REPLACE EXTERNAL VOLUME ${local.volume_name_sf_identifier}
   STORAGE_LOCATIONS =
      (
         (
            NAME = '${aws_s3_bucket.my_iceberg.bucket}',
            STORAGE_PROVIDER = 'S3'
            STORAGE_BASE_URL = 's3://${aws_s3_bucket.my_iceberg.bucket}'
            STORAGE_AWS_ROLE_ARN = '${aws_iam_role.my_iceberg_sf_access_role.arn}'
         )
      )
EOT

  revert = <<EOT
DROP EXTERNAL VOLUME IF EXISTS ${local.volume_name_sf_identifier}
EOT

  query = <<EOT
DESC EXTERNAL VOLUME ${local.volume_name_sf_identifier}
EOT
}

output "volume_aws_iam_user_arn" {
  value = jsondecode(snowflake_unsafe_execute.iceberg_volume.query_results[index(snowflake_unsafe_execute.iceberg_volume.query_results.*.property, "STORAGE_LOCATION_1")].property_value).STORAGE_AWS_IAM_USER_ARN
}

output "volume_external_id" {
  value = jsondecode(snowflake_unsafe_execute.iceberg_volume.query_results[index(snowflake_unsafe_execute.iceberg_volume.query_results.*.property, "STORAGE_LOCATION_1")].property_value).STORAGE_AWS_EXTERNAL_ID
}

variables.tf

variable region {}

variable "sf_user_extern_id" {
  default = "0000"
}

variable "sf_user_extern_arn" {
  default = null
}