Skip to content

Instantly share code, notes, and snippets.

@rdkls
Created July 11, 2023 13:42
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save rdkls/7ed8863f2e170af68bab465a8143893e to your computer and use it in GitHub Desktop.
Save rdkls/7ed8863f2e170af68bab465a8143893e to your computer and use it in GitHub Desktop.
CREATE EXTERNAL TABLE IF NOT EXISTS vpc_flow_logs (
version int,
account string,
interfaceid string,
sourceaddress string,
destinationaddress string,
sourceport int,
destinationport int,
protocol int,
numpackets int,
numbytes bigint,
starttime int,
endtime int,
action string,
logstatus string,
vpcid string,
subnetid string,
instanceid string,
tcpflags int,
type string,
pktsrcaddr string,
pktdstaddr string,
aws_region string,
azid string,
sublocationtype string,
sublocationid string,
pktsrcawsservice string,
pktdstawsservice string,
flowdirection string,
trafficpath string
)
-- PARTITIONED BY (region string, day string)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ' '
LOCATION 's3://xxxxxxxxxxxxxx/vpc-flow-logs/AWSLogs/'
TBLPROPERTIES
(
"skip.header.line.count"="1",
"projection.enabled" = "true",
"projection.region.type" = "injected",
-- "projection.region.values" = "ap-southus-east-1,us-west-2,ap-south-1,eu-west-1",
"projection.day.type" = "date",
"projection.day.range" = "2021/01/01,NOW",
"projection.day.format" = "yyyy/MM/dd",
"projection.account_id.type" = "injected",
"storage.location.template" = "s3://xxxxxxxxxxxxxxxx/vpc-flow-logs/AWSLogs/${account_id}/vpcflowlogs/${region}/${day}"
)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment