Skip to content

Instantly share code, notes, and snippets.

@timvw
Created February 1, 2024 20:37
Show Gist options
  • Star 3 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save timvw/52f742553f691c92e69c33fc3102c2a6 to your computer and use it in GitHub Desktop.
Save timvw/52f742553f691c92e69c33fc3102c2a6 to your computer and use it in GitHub Desktop.
Build infra for creating iceberg tables in snowflake with the terraform provider
resource "snowflake_unsafe_execute" "volume" {
execute = <<EOT
CREATE OR REPLACE EXTERNAL VOLUME ${var.external_volume_name}
STORAGE_LOCATIONS =
(
(
NAME = '${var.external_volume_bucket_name}'
STORAGE_PROVIDER = 'S3'
STORAGE_BASE_URL = 's3://${var.external_volume_bucket_name}'
STORAGE_AWS_ROLE_ARN = '${var.snowflake_iam_role_arn}'
ENCRYPTION=(TYPE='AWS_SSE_KMS')
)
)
EOT
revert = <<EOT
DROP EXTERNAL VOLUME IF EXISTS ${var.external_volume_name}
EOT
query = <<EOT
DESC EXTERNAL VOLUME ${var.external_volume_name}
EOT
}
resource "snowflake_unsafe_execute" "catalog" {
execute = <<EOT
CREATE or replace CATALOG INTEGRATION ${var.catalog_name}
CATALOG_SOURCE=GLUE
CATALOG_NAMESPACE='${var.catalog_namespace}'
TABLE_FORMAT=ICEBERG
GLUE_AWS_ROLE_ARN='${var.snowflake_iam_role_arn}'
GLUE_CATALOG_ID='${var.catalog_id}'
ENABLED=TRUE
EOT
revert = <<EOT
DROP CATALOG INTEGRATION IF EXISTS ${var.catalog_name}
EOT
query = <<EOT
DESCRIBE CATALOG INTEGRATION ${var.catalog_name}
EOT
}
output "volume_aws_iam_user_arn" {
value = jsondecode(snowflake_unsafe_execute.volume.query_results[index(snowflake_unsafe_execute.volume.query_results.*.property, "STORAGE_LOCATION_1")].property_value).STORAGE_AWS_IAM_USER_ARN
}
output "volume_external_id" {
value = jsondecode(snowflake_unsafe_execute.volume.query_results[index(snowflake_unsafe_execute.volume.query_results.*.property, "STORAGE_LOCATION_1")].property_value).STORAGE_AWS_EXTERNAL_ID
}
output "catalog_aws_iam_user_arn" {
value = snowflake_unsafe_execute.catalog.query_results[index(snowflake_unsafe_execute.catalog.query_results.*.property, "GLUE_AWS_IAM_USER_ARN")].property_value
}
output "catalog_external_id" {
value = snowflake_unsafe_execute.catalog.query_results[index(snowflake_unsafe_execute.catalog.query_results.*.property, "GLUE_AWS_EXTERNAL_ID")].property_value
}
variable "external_volume_name" {
type = string
description = "Name of the external volume"
}
variable "external_volume_bucket_name" {
type = string
description = "Name of an S3 bucket that stores your data files"
}
variable "snowflake_iam_role_arn" {
type = string
description = "Specifies the Amazon Resource Name (ARN) of the AWS identity and access management (IAM) role that grants privileges on the S3 bucket containing your data files"
}
variable "catalog_name" {
type = string
description = "String that specifies the identifier (name) for the catalog integration"
}
variable "catalog_id" {
type = string
description = "Specifies the ID of your AWS account"
}
variable "catalog_namespace" {
type = string
description = "Specifies your AWS Glue Data Catalog namespace (for example, my_glue_database). This is the default namespace for all Iceberg tables that you associate with this catalog integration. You can override this value by specifying the namespace at the table level when you create a table."
default = "default"
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment