Created
March 30, 2022 14:18
-
-
Save tmokmss/6e600c8de18e0560f65b0b0a9d84886d to your computer and use it in GitHub Desktop.
A standalone construct for DataBrew Job
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import { Construct } from 'constructs'; | |
import { aws_databrew as databrew } from 'aws-cdk-lib'; | |
import { IBucket } from 'aws-cdk-lib/aws-s3'; | |
import { IRole } from 'aws-cdk-lib/aws-iam'; | |
export interface JobProps { | |
readonly name: string; | |
readonly inputBucket: IBucket; | |
readonly inputKey: string; | |
readonly outputBucket: IBucket; | |
readonly outputKey: string; | |
readonly recipeName: string; | |
readonly role: IRole; | |
} | |
export class Job extends Construct { | |
readonly jobName : string; | |
readonly outputKey: string; | |
constructor(scope: Construct, id: string, props: JobProps) { | |
super(scope, id); | |
const inputDataset = new databrew.CfnDataset(this, 'Dataset', { | |
name: `${props.name}-input`, | |
input: { | |
s3InputDefinition: { | |
bucket: props.inputBucket.bucketName, | |
key: props.inputKey, | |
}, | |
}, | |
format: 'CSV', | |
formatOptions: { | |
csv: { | |
delimiter: ',', | |
headerRow: true, | |
}, | |
}, | |
}); | |
const cfnRecipe = new databrew.CfnRecipe(this, 'Recipe', { | |
name: props.name, | |
steps: [ | |
{ | |
action: { | |
operation: 'EXTRACT_PATTERN', | |
parameters: { | |
SourceColumn: 'Consulate', | |
Pattern: 'A', | |
TargetColumn: 'extract_pattern', | |
}, | |
}, | |
}, | |
], | |
}); | |
const cfnProject = new databrew.CfnProject(this, 'Project', { | |
datasetName: inputDataset.ref, | |
name: props.name, | |
recipeName: cfnRecipe.ref, | |
roleArn: props.role.roleArn, | |
}); | |
const job = new databrew.CfnJob(this, 'Default', { | |
name: 'mod'+props.name, | |
roleArn: props.role.roleArn, | |
type: 'RECIPE', | |
// datasetName: inputDataset.ref, | |
encryptionMode: 'SSE-S3', | |
logSubscription: 'ENABLE', | |
maxRetries: 3, | |
outputs: [ | |
{ | |
location: { | |
bucket: props.outputBucket.bucketName, | |
key: props.outputKey, | |
}, | |
format: 'CSV', | |
formatOptions: { | |
csv: { | |
delimiter: ',', | |
}, | |
}, | |
overwrite: true, | |
maxOutputFiles: 1, | |
}, | |
], | |
// recipe: { | |
// name: props.recipeName, | |
// version: 'LATEST_PUBLISHED', | |
// }, | |
projectName: cfnProject.ref, | |
}); | |
this.jobName = job.ref; | |
this.outputKey = props.outputKey; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment