Created
September 17, 2016 15:44
-
-
Save xcaptain/57aa46eaf52a1e0233f04caef1f1df66 to your computer and use it in GitHub Desktop.
multi process php script to write csv file
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<?php | |
class Task | |
{ | |
private $data = []; | |
private $fp = null; | |
private $columns = ['id', 'amount', 'date']; | |
private $num = 10000; | |
private $workerNum = 10; | |
private $pageSize = 1; | |
private $filename = '/tmp/data1.csv'; | |
private $counter = 0; | |
public function __construct() | |
{ | |
$this->generateData(); | |
$this->pageSize = $this->devideWork($this->num, $this->workerNum); | |
@unlink($this->filename); | |
$this->fp = fopen($this->filename, 'a+'); | |
} | |
public function run() | |
{ | |
fputcsv($this->fp, $this->columns); | |
$this->runProcess(); | |
echo "total loop count is {$this->counter}\n"; // 0 | |
} | |
private function runProcess() | |
{ | |
for($i = 0; $i < $this->workerNum; $i++) { | |
$pids[$i] = pcntl_fork(); | |
if (!$pids[$i]) { | |
$this->writeCsv($i); | |
exit($i); | |
} | |
} | |
while (pcntl_waitpid(0, $status) != -1) { | |
$status = pcntl_wexitstatus($status); | |
echo "Child $status completed\n"; | |
} | |
} | |
private function writeCsv($page) | |
{ | |
$start = $page * $this->pageSize; | |
$stop = ($page+1)*$this->pageSize; | |
echo "writing page {$page}, start={$start}, stop={$stop}, pageSize={$this->pageSize}\n"; | |
for ($i = $start; $i < $stop; $i++) { | |
$row = $this->data[$i]; | |
fputcsv($this->fp, $row); | |
$this->counter++; | |
} | |
echo "counter in child process is {$this->counter}\n"; // 1000 | |
} | |
private function devideWork($total, $workNum) | |
{ | |
return (int)($total/$workNum); | |
} | |
private function generateData() | |
{ | |
for ($i = 0; $i < $this->num; $i++) { | |
$amount = random_int($i, 10*$this->num); | |
$date = date('Y-m-d H:i:s', 100 * $amount); | |
$row = [$i, $amount, $date]; | |
$this->data[] = $row; | |
} | |
} | |
public function __desctruct() | |
{ | |
fclose($this->fp); | |
unset($this->fp); | |
unset($this->data); | |
} | |
} | |
$task = new Task(); | |
$task->run(); | |
unset($task); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment