Skip to content

Instantly share code, notes, and snippets.

@tsibley
Created November 8, 2023 20:11
Show Gist options
  • Save tsibley/2a090228891111162537856e5ef8ae05 to your computer and use it in GitHub Desktop.
Save tsibley/2a090228891111162537856e5ef8ae05 to your computer and use it in GitHub Desktop.
diff --git a/src/resourceIndex.js b/src/resourceIndex.js
index 3bcffee4..886f2a74 100644
--- a/src/resourceIndex.js
+++ b/src/resourceIndex.js
@@ -1,6 +1,7 @@
import fs from 'fs';
import * as utils from './utils/index.js';
import zlib from 'zlib';
+import { DateTime } from 'luxon';
import { promisify } from 'util';
import { NotFound, BadRequest } from './httpErrors.js';
import { RESOURCE_INDEX } from './config.js';
@@ -11,7 +12,6 @@ const gunzip = promisify(zlib.gunzip)
const LATEST = "LATEST"
let resources = {};
-let eTag;
class ResourceVersions {
constructor(sourceId, resourceType, resourcePath) {
@@ -99,17 +99,24 @@ async function updateResourceVersions() {
}
resources = JSON.parse(fileContents);
} else {
- const [Bucket, Key] = ['nextstrain-inventories', 'resources.json.gz'];
- utils.verbose(`Updating available resources index from s3://${Bucket}/${Key}`);
+ const [bucket, key] = ['nextstrain-inventories', 'resources.json.gz'];
+ utils.verbose(`Updating available resources index from s3://${bucket}/${key}`);
+
+ // Generate a signed URL that's stable (and thus cacheable) for up to a day
+ const method = "GET";
+ const issuedAt = DateTime.now().startOf("day").toUnixInteger();
+ const expiresIn = 25 * 60 * 60; // 25 hours in seconds
+ const url = await signedUrl({bucket, key, method, issuedAt, expiresIn});
+
try {
- const newETag = (await fetch(await signedUrl({bucket:Bucket, key:Key, method: 'HEAD'}), {method: 'HEAD'}))
- .headers.get('etag'); // value is enclosed in double quotes, but that doesn't matter for our purposes
- if (newETag && newETag === eTag) {
- utils.verbose("Skipping available resource update as eTag hasn't changed");
- return;
+ // Use no-cache to require revalidation but still allow cached data
+ const response = await fetch(url, {method, cache: 'no-cache'});
+
+ if (response.status !== 200) {
+ throw new Error(`response was: ${response.status} ${response.statusText}`);
}
- const newResources = JSON.parse(await gunzip(await (await fetch(await signedUrl({bucket:Bucket, key:Key, method: 'GET'}), {method: 'GET'})).buffer()));
- [resources, eTag] = [newResources, newETag];
+
+ resources = JSON.parse(await gunzip(await response.buffer()));
} catch (err) {
utils.warn(`Resource updating failed: ${err.message}`)
return;
@@ -123,4 +130,4 @@ export {
updateResourceVersions,
LATEST,
-}
\ No newline at end of file
+}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment