Created
May 27, 2022 17:53
-
-
Save vlovich/75cb7c8f6f011bc6e895c5c8a8743f92 to your computer and use it in GitHub Desktop.
Hacky R2 write IOPs benchmark
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// "aws-sdk": "^2.1100.0", | |
import pkg from 'aws-sdk' | |
const { S3 } = pkg | |
import { performance } from 'perf_hooks' | |
import { globalAgent } from 'http' | |
import { globalAgent as globalAgentSecure } from 'https' | |
globalAgent.maxSockets = 300 | |
globalAgentSecure.maxSockets = 300 | |
const creds = { | |
endpoint: 'https://<ACCOUNT>.r2.cloudflarestorage.com', | |
credentials: { | |
accessKeyId: '', | |
secretAccessKey: '', | |
}, | |
} | |
const client = new S3({ | |
region: 'us-east-1', | |
...creds, | |
retryDelayOptions: { | |
customBackoff: (): number => -1, | |
} | |
}) | |
function sync<T>(r: AWS.Request<T, AWS.AWSError>): Promise<T> { | |
return new Promise((resolve, reject) => { | |
r.send((err, data) => { | |
if (err) { | |
reject(err) | |
} else { | |
resolve(data) | |
} | |
}) | |
}) | |
} | |
const bucket = 'bench' | |
/* | |
try { | |
await sync(client.deleteBucket({ | |
Bucket: bucket | |
})) | |
} catch (e) { | |
if ((e as Error).name === 'BucketNotEmpty') { | |
while ((e as Error).name === 'BucketNotEmpty') { | |
console.error('listing') | |
const existingObjects = await sync(client.listObjectsV2({ | |
Bucket: bucket, | |
MaxKeys: 63, | |
})) | |
const toDelete = existingObjects.Contents?.map((o) => { return { Key: o.Key! }}) ?? [] | |
console.error('deleting') | |
await sync(client.deleteObjects({ | |
Bucket: bucket, | |
Delete: { | |
Objects: toDelete, | |
} | |
})) | |
try { | |
await sync(client.deleteBucket({ | |
Bucket: bucket | |
})) | |
e = undefined | |
break | |
} catch (e2) { | |
e = e2 | |
} | |
} | |
if (e !== undefined) { | |
throw e | |
} | |
} else if ((e as Error).name === 'NoSuchBucket') { | |
} else { | |
throw e | |
} | |
} | |
await sync(client.createBucket({ | |
Bucket: bucket, | |
})) | |
*/ | |
console.log(`testing bucket '${bucket}'`) | |
while (true) { | |
const ops = [] | |
const start = performance.now() | |
const num_loops = 1000 | |
for (let i = 0; i < num_loops; i++) { | |
const key = `key${(i % 5000).toString().padStart(5, '0')}` | |
ops.push(sync(client.putObject({ | |
Bucket: bucket, | |
Key: key, | |
Body: '', | |
}))) | |
} | |
const results = await Promise.allSettled(ops) | |
const failed = results.filter((r) => r.status === 'rejected') as PromiseRejectedResult[] | |
const success = results.filter((r) => r.status === 'fulfilled') | |
if (failed.length > 0) { | |
console.error('failed', failed.length, failed[0].reason) | |
} | |
const end = performance.now() | |
console.log(ops.length, 'concurrent ops took', end - start, 'milliseconds, or', success.length / ((end - start) / 1000), '(', failed.length, 'failures)') | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment