If I use the Go v2 SDK for AWS with DynamoDB, the AWS NoSQL Workbench and a local DynamoDB on Docker, I can't PUT, GET or QUERY info on the local DynamoDB instance. The issue is operating system agnostic. Outcomes are the same so far on Linux, OSX and Windows.
I've also done packet captures and analysed information passed across to the local instance. I can't see what's going on with enough clarity to diagnose the issue. I can confirm the packets are making it to the Docker instance, but do not see (or can decode) the auth information. I do see the DynamoDB local instance however respond and reject the query.
You'll notice there are lots of unused code paths. I've left them in to show history of exploration.
Any assistance will be greatly received!
-
Install the AWS CLI and configure appropriately (for either AWS or with 'localhost' region and local credentials for DynamoDB)
Note: Using the AWS CLI I can access the local DynamoDB instance. No issue with that.
-
Instantiate a local DynamoDB on Docker (on the local machine)
docker run -d -p 8000:8000 amazon/dynamodb-local- Using the NoSQL Workbench, create a test table with data below. Commit the data to the local instance with the NoSQL Workbench and make a note of the credentials (you'll need them for the Go code and for the AWS CLI).
{
"ModelName": "test",
"ModelMetadata": {
"Author": "David Gee",
"DateCreated": "Jul 13, 2021, 09:44 AM",
"DateLastModified": "Jul 13, 2021, 09:45 AM",
"Description": "",
"AWSService": "Amazon DynamoDB",
"Version": "3.0"
},
"DataModel": [
{
"TableName": "test",
"KeyAttributes": {
"PartitionKey": {
"AttributeName": "PK",
"AttributeType": "S"
},
"SortKey": {
"AttributeName": "SK",
"AttributeType": "S"
}
},
"NonKeyAttributes": [
{
"AttributeName": "att1",
"AttributeType": "S"
}
],
"TableData": [
{
"PK": {
"S": "testpk"
},
"SK": {
"S": "testsk"
},
"att1": {
"S": "testatt1"
}
}
],
"DataAccess": {
"MySql": {}
},
"BillingMode": "PROVISIONED",
"ProvisionedCapacitySettings": {
"ProvisionedThroughput": {
"ReadCapacityUnits": 5,
"WriteCapacityUnits": 5
},
"AutoScalingRead": {
"ScalableTargetRequest": {
"MinCapacity": 1,
"MaxCapacity": 10,
"ServiceRole": "AWSServiceRoleForApplicationAutoScaling_DynamoDBTable"
},
"ScalingPolicyConfiguration": {
"TargetValue": 70
}
},
"AutoScalingWrite": {
"ScalableTargetRequest": {
"MinCapacity": 1,
"MaxCapacity": 10,
"ServiceRole": "AWSServiceRoleForApplicationAutoScaling_DynamoDBTable"
},
"ScalingPolicyConfiguration": {
"TargetValue": 70
}
}
}
}
]
}- Build, compile and run the code below, ensuring to insert the correct credentials (I've placed appropriate comments). This code works just fine for the AWS webservice, but not the local DynamoDB. Yes, the code is a little fugly, but it's functional enough to explore various options.
package main
import (
"context"
"fmt"
"log"
"strconv"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression"
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
)
// DynamoDBDescribeTableAPI defines the interface for the DescribeTable function.
// We use this interface to enable unit testing.
type DynamoDBDescribeTableAPI interface {
DescribeTable(ctx context.Context,
params *dynamodb.DescribeTableInput,
optFns ...func(*dynamodb.Options)) (*dynamodb.DescribeTableOutput, error)
}
// GetTableInfo retrieves information about the table.
func GetTableInfo(c context.Context, api DynamoDBDescribeTableAPI, input *dynamodb.DescribeTableInput) (*dynamodb.DescribeTableOutput, error) {
return api.DescribeTable(c, input)
}
func GetItems(c context.Context, api DynamoDBScanAPI, input *dynamodb.ScanInput) (*dynamodb.ScanOutput, error) {
return api.Scan(c, input)
}
type DynamoDBScanAPI interface {
Scan(ctx context.Context,
params *dynamodb.ScanInput,
optFns ...func(*dynamodb.Options)) (*dynamodb.ScanOutput, error)
}
// Item holds info about the items returned by Scan
type Item struct {
PK string `json:"PK"`
SK string `json:"SK"`
Att1 string `json:"att1"`
}
func main() {
cfg, err := config.LoadDefaultConfig(context.TODO(),
// CHANGE THIS TO us-east-1 TO USE AWS proper
config.WithRegion("localhost"),
// COMMENT OUT THE BELOW FOUR LINES TO USE AWS proper
config.WithEndpointResolver(aws.EndpointResolverFunc(
func(service, region string) (aws.Endpoint, error) {
return aws.Endpoint{URL: "http://localhost:8000"}, nil
})),
)
if err != nil {
log.Fatalf("unable to load SDK config, %v", err)
}
tableName := "test"
client := dynamodb.NewFromConfig(cfg, func(o *dynamodb.Options) {
// o.EndpointOptions.DisableHTTPS = true
// o.Region = "localhost"
// CHANGE THESE CREDENTIALS TO USE THE PUBLIC SET. IMMEDIATELY BELOW IS FOR LOCAL DYNAMODB
o.Credentials = credentials.NewStaticCredentialsProvider("x2yvg", "2uprpl", "")
// o.Credentials = credentials.NewStaticCredentialsProvider("KEYID", "KEY", "TOKEN")
})
input := &dynamodb.DescribeTableInput{
TableName: &tableName,
}
fmt.Printf("client data: %+v\n", client)
resp, err := GetTableInfo(context.TODO(), client, input)
if err != nil {
fmt.Println("failed to describe table, " + err.Error())
}
fmt.Println("Info about " + tableName + ":")
fmt.Println(" #items: ", resp.Table.ItemCount)
fmt.Println(" Size (bytes)", resp.Table.TableSizeBytes)
fmt.Println(" Status: ", string(resp.Table.TableStatus))
tables, err := client.ListTables(context.Background(), &dynamodb.ListTablesInput{})
if err != nil {
fmt.Println("failed to get tables, " + err.Error())
}
fmt.Printf("%+v\n", tables)
for _, n := range tables.TableNames {
fmt.Println("TABLE", n)
}
// Programmatically load some data
inputItems := map[string]types.AttributeValue{}
inputItems["PK"] = &types.AttributeValueMemberS{Value: "test2pk"}
inputItems["SK"] = &types.AttributeValueMemberS{Value: "test2sk"}
inputItems["att1"] = &types.AttributeValueMemberS{Value: "test2att1"}
_, err = client.PutItem(context.Background(), &dynamodb.PutItemInput{TableName: &tableName, Item: inputItems})
if err != nil {
fmt.Println("Error PutItem(): ", err)
return
}
// Let's get the stuff
filt1 := expression.Name("PK").Equal(expression.Value("test2pk"))
// Get back the title and rating (we know the year).
proj := expression.NamesList(expression.Name("PK"), expression.Name("SK"), expression.Name("att1"))
expr, err := expression.NewBuilder().WithFilter(filt1).WithProjection(proj).Build()
if err != nil {
fmt.Println("Got error building expression:")
fmt.Println(err.Error())
return
}
inputQ := &dynamodb.ScanInput{
ExpressionAttributeNames: expr.Names(),
ExpressionAttributeValues: expr.Values(),
FilterExpression: expr.Filter(),
ProjectionExpression: expr.Projection(),
TableName: &tableName,
}
resp2, err := GetItems(context.TODO(), client, inputQ)
if err != nil {
fmt.Println("Got an error scanning the table:")
fmt.Println(err.Error())
return
}
items := []Item{}
err = attributevalue.UnmarshalListOfMaps(resp2.Items, &items)
if err != nil {
fmt.Println(fmt.Sprintf("failed to unmarshal Dynamodb Scan Items, %v", err))
}
for _, item := range items {
fmt.Println("PK: ", item.PK)
fmt.Println("SK:", item.SK)
fmt.Println("att1:", item.Att1)
fmt.Println()
}
numItems := strconv.Itoa(len(items))
fmt.Println("Found", numItems)
}
Thanks Tom! Will try it out.