Skip to content

Instantly share code, notes, and snippets.

@qRoC
Last active June 18, 2016 18:38
Show Gist options
  • Save qRoC/d8947d180fe01fcd413e3f9f05436831 to your computer and use it in GitHub Desktop.
Save qRoC/d8947d180fe01fcd413e3f9f05436831 to your computer and use it in GitHub Desktop.
DoctrineFixturesBundle for big data
<?php
// Copyright (c) 2016 Andrey <qRoC.Work@gmail.com> Savitsky. All rights reserved.
// Site: http://qRoC.pro
namespace CommonBundle\DataFixtures;
use Doctrine\Common\DataFixtures\FixtureInterface;
use Doctrine\Common\DataFixtures\OrderedFixtureInterface;
use Symfony\Component\DependencyInjection\ContainerAwareInterface;
use Symfony\Component\DependencyInjection\ContainerInterface;
use Doctrine\Common\Persistence\ObjectManager;
abstract class ORMFixture implements FixtureInterface, ContainerAwareInterface, OrderedFixtureInterface
{
const kMemoryLimit = 32;
const kMemoryCanUsage = 0.3; // 70% резерв на flush() и clear(). Иногда нужно больше
/**
* @var ContainerInterface
*/
protected $container;
final public function setContainer(ContainerInterface $container = null)
{
$this->container = $container;
}
final public function load(ObjectManager $manager)
{
$manager->getConnection()->getConfiguration()->setSQLLogger(null);
$this->loadStandart($manager);
$this->innerLoadIteration($manager);
$manager->flush();
$manager->clear();
}
public function getOrder()
{
return 0;
}
private function innerLoadIteration(ObjectManager $manager)
{
$memory_limit = self::kMemoryLimit;
if (preg_match('/^(\d+)(.)$/', ini_get('memory_limit'), $matches))
{
$memory_limit = round($matches[1] * self::kMemoryCanUsage);
switch($matches[2])
{
case 'G':
$memory_limit *=1024;
case 'M':
$memory_limit *=1024;
case 'K':
$memory_limit *=1024;
}
}
$iteration = function(array $work_objects = []) use($memory_limit, $manager)
{
$usage = memory_get_usage();
if ($usage > $memory_limit)
{
// echo 'USAGE: ' . $usage . PHP_EOL;
// echo 'MAX: ' . $memory_limit . PHP_EOL;
$manager->flush();
foreach($work_objects as $work_object)
{
$manager->clear($work_object);
}
//echo 'AFTER CLEAR: ' . memory_get_usage() . PHP_EOL;
}
};
$this->loadIteration($manager, $iteration);
}
/**
* iteration {
*
* // ....
*
* $iteration();
* }
*
* @param ObjectManager $manager
* @param callable $iteration
*/
protected function loadIteration(ObjectManager $manager, callable $iteration)
{
}
protected function loadStandart(ObjectManager $manager)
{
}
}
<?php
// Copyright (c) 2016 Andrey <qRoC.Work@gmail.com> Savitsky. All rights reserved.
// Site: http://qRoC.pro
namespace CommonBundle\DataFixtures\ORM;
use CommonBundle\DataFixtures\ORMFixture;
use Doctrine\Common\Persistence\ObjectManager;
use CommonBundle\Entity\Country;
use CommonBundle\Entity\State;
use CommonBundle\Entity\City;
class LoadLocationData extends ORMFixture
{
public function loadIteration(ObjectManager $manager, callable $iteration)
{
$file = fopen(__DIR__ . '/data/weblocations.dat', 'r');
while(!feof($file))
{
$line = fgets($file);
// $entity = new City();
$manager->persist($entity);
// $iteration(); // Никогда не выполнять $manager->clear();
$iteration([City::class]); // $manager->clear(City::class) - нужно если вы работаете с несколькими типами сущностей одновременно.
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment