Created
November 8, 2023 20:11
-
-
Save tsibley/2a090228891111162537856e5ef8ae05 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
diff --git a/src/resourceIndex.js b/src/resourceIndex.js | |
index 3bcffee4..886f2a74 100644 | |
--- a/src/resourceIndex.js | |
+++ b/src/resourceIndex.js | |
@@ -1,6 +1,7 @@ | |
import fs from 'fs'; | |
import * as utils from './utils/index.js'; | |
import zlib from 'zlib'; | |
+import { DateTime } from 'luxon'; | |
import { promisify } from 'util'; | |
import { NotFound, BadRequest } from './httpErrors.js'; | |
import { RESOURCE_INDEX } from './config.js'; | |
@@ -11,7 +12,6 @@ const gunzip = promisify(zlib.gunzip) | |
const LATEST = "LATEST" | |
let resources = {}; | |
-let eTag; | |
class ResourceVersions { | |
constructor(sourceId, resourceType, resourcePath) { | |
@@ -99,17 +99,24 @@ async function updateResourceVersions() { | |
} | |
resources = JSON.parse(fileContents); | |
} else { | |
- const [Bucket, Key] = ['nextstrain-inventories', 'resources.json.gz']; | |
- utils.verbose(`Updating available resources index from s3://${Bucket}/${Key}`); | |
+ const [bucket, key] = ['nextstrain-inventories', 'resources.json.gz']; | |
+ utils.verbose(`Updating available resources index from s3://${bucket}/${key}`); | |
+ | |
+ // Generate a signed URL that's stable (and thus cacheable) for up to a day | |
+ const method = "GET"; | |
+ const issuedAt = DateTime.now().startOf("day").toUnixInteger(); | |
+ const expiresIn = 25 * 60 * 60; // 25 hours in seconds | |
+ const url = await signedUrl({bucket, key, method, issuedAt, expiresIn}); | |
+ | |
try { | |
- const newETag = (await fetch(await signedUrl({bucket:Bucket, key:Key, method: 'HEAD'}), {method: 'HEAD'})) | |
- .headers.get('etag'); // value is enclosed in double quotes, but that doesn't matter for our purposes | |
- if (newETag && newETag === eTag) { | |
- utils.verbose("Skipping available resource update as eTag hasn't changed"); | |
- return; | |
+ // Use no-cache to require revalidation but still allow cached data | |
+ const response = await fetch(url, {method, cache: 'no-cache'}); | |
+ | |
+ if (response.status !== 200) { | |
+ throw new Error(`response was: ${response.status} ${response.statusText}`); | |
} | |
- const newResources = JSON.parse(await gunzip(await (await fetch(await signedUrl({bucket:Bucket, key:Key, method: 'GET'}), {method: 'GET'})).buffer())); | |
- [resources, eTag] = [newResources, newETag]; | |
+ | |
+ resources = JSON.parse(await gunzip(await response.buffer())); | |
} catch (err) { | |
utils.warn(`Resource updating failed: ${err.message}`) | |
return; | |
@@ -123,4 +130,4 @@ export { | |
updateResourceVersions, | |
LATEST, | |
-} | |
\ No newline at end of file | |
+} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment