Skip to content

Instantly share code, notes, and snippets.

@mvolfik
Created June 22, 2022 10:01
Show Gist options
  • Save mvolfik/b7905242079fc476b3709965898208c7 to your computer and use it in GitHub Desktop.
Save mvolfik/b7905242079fc476b3709965898208c7 to your computer and use it in GitHub Desktop.
const { Actor } = require("apify");
const { Readable } = require("node:stream");
const https = require("node:https");
class MyStream extends Readable {
constructor() {
super();
this.i = 0;
}
_read() {
const i = this.i++;
if (i > 200) {
this.push(null);
return;
}
setTimeout(() => {
this.push(".".repeat(1024 * 1024 * 5) + i.toString());
console.log(i);
}, 500);
}
}
async function uploadWithSdk() {
const store = await Actor.openKeyValueStore(undefined, {
forceCloud: true,
});
await store.setValue("foobar", new MyStream(), {
contentType: "text/plain",
});
}
function uploadWithNodeHttp() {
return new Promise((resolve) => {
const request = https.request(
"https://api.apify.com/v2/key-value-stores/sQX36Y3hraqX0TUAm/records/MY_KEY?token=YOUR_APIFY_TOKEN",
{
method: "POST",
headers: {
"Content-Type": "text/plain",
"transfer-encoding": "chunked",
},
},
(response) => response.on("end", () => resolve())
);
const stream = new MyStream();
stream.pipe(request);
});
}
Actor.main(async () => {
// compare memory usage for these two
await uploadWithSdk();
// await uploadWithNodeHttp();
}, {});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment