apify-stream-upload-kv-test.js

const { Actor } = require("apify");
const { Readable } = require("node:stream");
const https = require("node:https");

class MyStream extends Readable {
    constructor() {
        super();
        this.i = 0;
    }

    _read() {
        const i = this.i++;
        if (i > 200) {
            this.push(null);
            return;
        }
        setTimeout(() => {
            this.push(".".repeat(1024 * 1024 * 5) + i.toString());
            console.log(i);
        }, 500);
    }
}

async function uploadWithSdk() {
    const store = await Actor.openKeyValueStore(undefined, {
        forceCloud: true,
    });
    await store.setValue("foobar", new MyStream(), {
        contentType: "text/plain",
    });
}

function uploadWithNodeHttp() {
    return new Promise((resolve) => {
        const request = https.request(
            "https://api.apify.com/v2/key-value-stores/sQX36Y3hraqX0TUAm/records/MY_KEY?token=YOUR_APIFY_TOKEN",
            {
                method: "POST",
                headers: {
                    "Content-Type": "text/plain",
                    "transfer-encoding": "chunked",
                },
            },
            (response) => response.on("end", () => resolve())
        );
        const stream = new MyStream();
        stream.pipe(request);
    });
}

Actor.main(async () => {
    // compare memory usage for these two
    await uploadWithSdk();
    // await uploadWithNodeHttp();
}, {});