Skip to content

Instantly share code, notes, and snippets.

@milinmestry
Created April 18, 2020 16:08
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save milinmestry/cf7ca21e9dffeb2354e6020218f30bdf to your computer and use it in GitHub Desktop.
Save milinmestry/cf7ca21e9dffeb2354e6020218f30bdf to your computer and use it in GitHub Desktop.
PHP Swoole Writing large dataset as json into file
<?php
// functions.php
function memoryStats($real = false) {
return memory_get_usage($real);
}
function memoryStatsHuman($real = false) {
return (memoryStats($real) / 1024) . ' MB';
}
<?php
require_once './functions.php';
require_once './vendor/autoload.php';
$allocatedMemory = memoryStatsHuman(true);
echo 'Allocated memory: ', $allocatedMemory, PHP_EOL;
$startTime = microtime(true);
echo 'Before loop memory usage: ', memoryStatsHuman(false), PHP_EOL;
$filename = __DIR__ . '/json-files/records-250000.json';
// use the factory to create a Faker\Generator instance
function generatorData() {
$faker = Faker\Factory::create();
$numRows = 250000;
for ($i = 0; $i < $numRows; $i++) {
yield [
'name' => $faker->name,
'email' => $faker->email,
'spent' => $faker->randomFloat(15),
'start_date' => $faker->date(),
'end_date' => $faker->date(),
'cpa' => $faker->randomNumber(7),
'cpm' => $faker->randomFloat(10),
'created_at' => date('Y-m-d H:i:s'),
];
// echo $i,PHP_EOL;
}
}
// Swoole\Runtime::enableCoroutine();
// $data = new SplFixedArray($numRows);
// $rawData = [
// 'records' => $data, // This fails memory exhausted when working with normal array
// 'recordCount' => $data>getSize(),
// ];
$fp = null;
$fp = fopen($filename, 'ab+');
// You need to know how you need to create JSON file and create accordingly
go(function() use ($fp) {
Swoole\Coroutine\System::fwrite($fp, '{' . PHP_EOL . '"records":[' . PHP_EOL);
foreach (generatorData() as $key => $data) {
// print_r($data);
$json = $jsonStr = '';
$json = json_encode($data);
$jsonStr = (0 !== $key) ? ',' . PHP_EOL . $json : $json;
$bytesWrote = Swoole\Coroutine\System::fwrite($fp, $jsonStr);
// echo "bytes wrote {$bytesWrote} \n";
}
// metadata
$metadata = [
'recordCount' => $key + 1 // array starts with 0, so +1 added
];
$metaStr = '],' . PHP_EOL . '"metadata":' . json_encode($metadata);
Swoole\Coroutine\System::fwrite($fp, $metaStr);
Swoole\Coroutine\System::fwrite($fp, PHP_EOL . '}' . PHP_EOL);
fclose($fp);
$memoryUsed = memoryStatsHuman(false);
$endTime = microtime(true);
$timeTaken = $endTime - $startTime;
echo 'Actual memory usage: ', $memoryUsed, PHP_EOL;
echo 'Total time taken: ', $timeTaken, ' seconds', PHP_EOL;
});
/**
Allocated memory: 2048 MB
Before loop memory usage: 1261.5859375 MB
Actual memory usage: 2946.8125 MB
Total time taken: 1587224980.5645 seconds
*/
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment