Skip to content

Instantly share code, notes, and snippets.

@thejuan
Last active February 15, 2021 01:26
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save thejuan/ecd0d3419c1be76905376ef2d7e3ec7c to your computer and use it in GitHub Desktop.
Save thejuan/ecd0d3419c1be76905376ef2d7e3ec7c to your computer and use it in GitHub Desktop.
S3 read-after-update consistency
import { writeS3Obj, getS3FileAsObj, listObjects, deleteS3Files } from "./S3";
/**
* S3 does not provide read-after-update consistency.
* It does provide read-after-firstWrite consistency (as long as no GET has been requested)
* We write a new file every time it changes, and we read the latest file.
* S3 guarantees list of files are sorted in ascending UTF-8 Binary Order
*
*/
const cleanUp = async (key: string) => {
const response = await listObjects({
MaxKeys: 1000,
Bucket: process.env.BUCKET_NAME!,
Prefix: key,
});
const keys = response.Contents?.map((c) => c.Key!) || [];
await deleteS3Files(keys.slice(0, keys.length - 1));
};
export const writeServiceState = async (key: string, state: any) => {
await writeS3Obj(`${key}.${Date.now()}`, state);
await cleanUp(key);
};
export const getServiceState = async (key: string, defaultVal: T): Promise => {
const response = await listObjects({
MaxKeys: 1000,
Bucket: process.env.BUCKET_NAME!,
Prefix: key,
});
if (!response.Contents || response.Contents.length === 0) {
console.log("No state file for key " + key);
return defaultVal;
}
return getS3FileAsObj(response.Contents[response.Contents.length - 1].Key!);
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment