Skip to content

Instantly share code, notes, and snippets.

@harlow
Last active December 28, 2019 18:58
Show Gist options
  • Star 3 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save harlow/48ba378eae55ce386846 to your computer and use it in GitHub Desktop.
Save harlow/48ba378eae55ce386846 to your computer and use it in GitHub Desktop.
Golang lambda function to send Streams data to Firehose
package main
import (
"github.com/apex/go-apex"
"github.com/apex/go-apex/kinesis"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/firehose"
)
const (
deliveryStreamName = "HarlowTest"
maxBatchSize = 400
)
func main() {
kinesis.HandleFunc(func(event *kinesis.Event, ctx *apex.Context) error {
svc := firehose.New(session.New())
records := make([]*firehose.Record, 0, maxBatchSize)
numRecords := len(event.Records)
for i, r := range event.Records {
records = append(
records,
&firehose.Record{Data: append(r.Kinesis.Data, '\n')},
)
if len(records) == maxBatchSize || i == (numRecords-1) {
_, err := svc.PutRecordBatch(
&firehose.PutRecordBatchInput{
DeliveryStreamName: aws.String(deliveryStreamName),
Records: records,
},
)
if err != nil {
return err
}
records = records[:0]
}
}
return nil
})
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment