Skip to content

Instantly share code, notes, and snippets.

@vlovich
Created May 27, 2022 17:53
Show Gist options
  • Save vlovich/75cb7c8f6f011bc6e895c5c8a8743f92 to your computer and use it in GitHub Desktop.
Save vlovich/75cb7c8f6f011bc6e895c5c8a8743f92 to your computer and use it in GitHub Desktop.
Hacky R2 write IOPs benchmark
// "aws-sdk": "^2.1100.0",
import pkg from 'aws-sdk'
const { S3 } = pkg
import { performance } from 'perf_hooks'
import { globalAgent } from 'http'
import { globalAgent as globalAgentSecure } from 'https'
globalAgent.maxSockets = 300
globalAgentSecure.maxSockets = 300
const creds = {
endpoint: 'https://<ACCOUNT>.r2.cloudflarestorage.com',
credentials: {
accessKeyId: '',
secretAccessKey: '',
},
}
const client = new S3({
region: 'us-east-1',
...creds,
retryDelayOptions: {
customBackoff: (): number => -1,
}
})
function sync<T>(r: AWS.Request<T, AWS.AWSError>): Promise<T> {
return new Promise((resolve, reject) => {
r.send((err, data) => {
if (err) {
reject(err)
} else {
resolve(data)
}
})
})
}
const bucket = 'bench'
/*
try {
await sync(client.deleteBucket({
Bucket: bucket
}))
} catch (e) {
if ((e as Error).name === 'BucketNotEmpty') {
while ((e as Error).name === 'BucketNotEmpty') {
console.error('listing')
const existingObjects = await sync(client.listObjectsV2({
Bucket: bucket,
MaxKeys: 63,
}))
const toDelete = existingObjects.Contents?.map((o) => { return { Key: o.Key! }}) ?? []
console.error('deleting')
await sync(client.deleteObjects({
Bucket: bucket,
Delete: {
Objects: toDelete,
}
}))
try {
await sync(client.deleteBucket({
Bucket: bucket
}))
e = undefined
break
} catch (e2) {
e = e2
}
}
if (e !== undefined) {
throw e
}
} else if ((e as Error).name === 'NoSuchBucket') {
} else {
throw e
}
}
await sync(client.createBucket({
Bucket: bucket,
}))
*/
console.log(`testing bucket '${bucket}'`)
while (true) {
const ops = []
const start = performance.now()
const num_loops = 1000
for (let i = 0; i < num_loops; i++) {
const key = `key${(i % 5000).toString().padStart(5, '0')}`
ops.push(sync(client.putObject({
Bucket: bucket,
Key: key,
Body: '',
})))
}
const results = await Promise.allSettled(ops)
const failed = results.filter((r) => r.status === 'rejected') as PromiseRejectedResult[]
const success = results.filter((r) => r.status === 'fulfilled')
if (failed.length > 0) {
console.error('failed', failed.length, failed[0].reason)
}
const end = performance.now()
console.log(ops.length, 'concurrent ops took', end - start, 'milliseconds, or', success.length / ((end - start) / 1000), '(', failed.length, 'failures)')
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment