large_file_splitting.js

import papaparse from 'https://jslib.k6.io/papaparse/5.1.1/index.js';
import { randomIntBetween, randomItem } from "https://jslib.k6.io/k6-utils/1.0.0/index.js";
import http from "k6/http";
import { sleep } from "k6";
let targetVus = 5;
let dataSlices = 100; // in how many parts are we going to split the data
export let options = {
    stages: [
        { duration: "300s", target: targetVus },
        { duration: "300s", target: targetVus }
    ],
};
if (__VU == 0) {
  open('userData_30mb.csv'); // we just open it so it is available in the cloud or if we do k6 archive
} else {
  var data = (function () {
    // separate function in order to not leak all the data in the main scope
    var all_data = papaparse.parse(open('./userData_30mb.csv'), { header: true }).data; // we load and parse the data in one go, no need for temp variables
    var part_size = all_data.length / dataSlices;
    var index = part_size * (__VU % dataSlices);
    return all_data.slice(index, index + part_size); // we return a slice for the VU.
  })();
}
// const csvData = papaparse.parse(open('./userData_30mb.csv'), { header: true });
export default function() {
  console.log(`VU=${__VU} has ${data.length} data`);
  console.log(`random item: ${JSON.stringify(randomItem(data))}`);
  http.get("http://test.k6.io");
  sleep(1);
}