Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Eloquent Model Iterator
<?php
/* This gist is offered as is, make sure to test in your environment
*
* This class can be used to iterate over a large Eloquent query. It uses a combination of the PDO Fetch and
* the chunk methods to collect a series of items in memory. The chunk part was necessary in order to implement
* eager loading options. You can set the CHUNK_SIZE in the code to set how many rows to load in memory at a time
* again for the eager loading purposes.
*
* I am sure there are other ways to implement this technique more effectively
*
* Usage: You can use this simply as follows
* // Take any large model list, and include a with if you like
* $orders = Orders::all()->with('user','product');
* // Turn it into a ModelIterator
* $iterator = new ModelIterator($orders);
* // and finally simply use it in a foreach or anything that supports the Iterator patter
* foreach($iterator as $order) {
* // do something with the order
* print "User " . $order->user->name . " bought " . $order->product->name . " on " . $order->order_date . " for " . $order->price . ".\n";
* }
*
*/
use Illuminate\Support\Facades\DB;
class ModelIterator implements \Iterator
{
/**
* @var mixed
*/
private $current;
/**
* @var \PDOStatement
*/
private $statement;
/**
* states whether the query builder needs to handle eagerLoads, if so then we need to chunk the results
* @var boolean
*/
private $eager_loads;
/**
* If using eager_loads then chunk will contain a collection of models for loading
* @var array
*/
private $chunk;
/**
* The current position in the chunk
* @var int
*/
private $chunk_pos;
/**
* Determine if this is the last chunk or not
* @var boolean
*/
private $chunk_last;
private $connection_name;
const CHUNK_SIZE = 50;
/**
* This will need some work especially if it will be integrated into Illuminate
* but for now lets just treat it as a simple passing of an eloquent builder
* object and then we can work from there.
* @param \Illuminate\Database\Eloquent\Builder $query
*/
function __construct($builder)
{
$this->builder = $builder;
$this->eager_loads = count($builder->getEagerLoads()) > 0;
$this->connection_name = $builder->getModel()->getConnectionName();
}
/**
* This is a simple helper method in statement we will call $this->getBuilder(), but in the
* live code when integrated into the library it would just be $this or something.
*
* @return \Illuminate\Database\Eloquent\Builder
*/
protected function getBuilder() {
return $this->builder;
}
/**
* @return \Illuminate\Database\Query\Builder
*/
protected function getQuery() {
return $this->getBuilder()->getQuery();
}
/**
* @return mixed
*/
public function current()
{
if (is_null($this->current)) {
$this->next();
}
return $this->current;
}
/**
* @return scalar
*/
public function key()
{
return $this->current() !== FALSE ? $this->current()[$this->builder->getModel()->getKeyName()] : null;
}
/**
* Fetch the next row
*/
public function next()
{
$this->current = null;
if (!$this->statement) {
$this->rewind();
}
if ($this->eager_loads) {
// we need to chunk results
// if we reached the end of the chunk or this is the first time we need to fill the chunk
if (!$this->chunk or $this->chunk_pos == ModelIterator::CHUNK_SIZE) {
// init the array and position
unset($this->chunk);
$this->chunk = array();
$this->chunk_pos = 0;
$this->chunk_last = false;
for ($i = 0; $i < ModelIterator::CHUNK_SIZE; $i++) {
$row = $this->statement->fetch();
if ($row === FALSE) {
$this->chunk_last = true;
break;
} else {
$this->chunk[] = $model = $this->builder->getModel()->newFromBuilder($row);
$model->setConnection($this->connection_name);
}
}
// perform the eager load but only if there are records
if (sizeof($this->chunk) > 0) {
$this->chunk = $this->builder->eagerLoadRelations($this->chunk);
}
}
if (sizeof($this->chunk) > $this->chunk_pos) {
$this->current = $this->chunk[$this->chunk_pos++];
} elseif ($this->chunk_last) {
$this->current = FALSE;
}
} else {
// build the results one at a time
$row = $this->statement->fetch();
if ($row === FALSE) {
// we have reached the end
$this->current = FALSE;
} else {
// create the model
$this->current = $this->builder->getModel()->newFromBuilder($row);
$this->current->setConnection($this->connection_name);
}
}
}
/**
* This method initializes everything and should be called only once to execute the initial query
*/
public function rewind()
{
// get the pieces together make this more concise when the time comes
$query = $this->getBuilder()->getQuery();
$connection = $query->getConnection();
$query_str = $query->toSql();
$query_bindings = $connection->prepareBindings($query->getBindings());
// get unbuffered PDO
$connection_name = $connection->getName();
$pdo = DB::connection($connection_name . '_unbuffered')->getPdo();
$this->statement = $pdo->prepare($query_str);
$success = $this->statement->execute($query_bindings);
// Now we need to grab the first row
if ($success) {
$this->next();
} else {
// set the current row to FALSE or end of the iterator
$this->current = FALSE;
}
}
/**
* @return boolean
*/
public function valid()
{
return $this->current() !== FALSE;
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.