Skip to content

Instantly share code, notes, and snippets.

@emilushi
Forked from soyuka/ObjectListener.php
Last active February 15, 2019 12:50
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save emilushi/887e8bcd15ea49d2e54377bea80722f5 to your computer and use it in GitHub Desktop.
Save emilushi/887e8bcd15ea49d2e54377bea80722f5 to your computer and use it in GitHub Desktop.
Streaming big json files the good way with php with https://soyuka.me/streaming-big-json-files-the-good-way/
<?php
namespace Fry;
use JsonStreamingParser\Listener;
/**
* This implementation allows to process an object at a specific level
* when it has been fully parsed
*/
class ObjectListener implements Listener
{
/** @var string Current key * */
private $_key;
/** @var int Array deep level * */
private $array_level = 0;
/** @var int Object deep level * */
private $object_level = 0;
/** @var array Pointer that aliases the current array that represents an object or an array * */
private $pointer;
/**
* @var array $array_pointers Stores different array pointers according to the deep level
* @var array $object_pointers Stores different objects pointers according to the deep level
* Those are used to track pointers, it's easy to go forward or backwards by using this
* As they are only pointers, in PHP "aliases" they shouldn't eat much memory even with big objects
*/
private $array_pointers, $object_pointers;
/** @var array Main array that stores the current building object * */
private $stack = [];
private $callback, $end_callback;
/**
* @param $callback
* @param null $end_callback
*/
public function __construct($callback, $end_callback = null)
{
if ( ! is_callable( $callback )) {
throw new \InvalidArgumentException( "Callback should be a callable function" );
}
$this->callback = $callback;
$this->end_callback = $end_callback;
}
public function filePosition($line, $char)
{
}
/**
* Document start
* Init every variables and place the pointer on the stack
*
* @return void
*/
public function startDocument()
{
$this->stack = [];
$this->array_pointers = [];
$this->array_level = 0;
$this->object_level = 0;
$this->object_pointers = [];
$this->keys = [];
$this->_key = null;
$this->pointer =& $this->stack;
}
/**
* Document end (EOF)
*
* @return void
*/
public function endDocument()
{
// release memory
$this->startDocument();
if (is_callable( $this->end_callback )) {
call_user_func_array( $this->end_callback, [] );
}
}
/**
* Start object
* An object began...
*
* @return void
*/
public function startObject()
{
//Increase the object level
$this->object_level++;
//Point on the current array
$this->pointer =& $this->array_pointers[$this->array_level];
//Get the current index
$array_index = isset( $this->pointer ) ? count( $this->pointer ) : 0;
//Build an array on this index
$this->pointer[$array_index] = [];
//Pointer is now this new array
$this->pointer =& $this->pointer[$array_index];
//Store it
$this->object_pointers[$this->object_level] =& $this->pointer;
}
/**
* End Object
* An object ended
*
* @return void
*/
public function endObject()
{
$this->pointer =& $this->array_pointers[$this->array_level];
//We've reach a full object on my root array, callback
if ($this->array_level == 1 && $this->object_level == 1) {
call_user_func_array( $this->callback, [$this->stack[0]] );
array_shift( $this->stack[0] ); //release this item from memory
}
$this->object_level--;
}
/**
* Start array
* An array began...
*
* @return void
*/
public function startArray()
{
$this->array_level++;
//If we have a key it's our index
if ($this->_key) {
$index = $this->_key;
$this->_key = null;
} else {
$index = isset( $this->pointer ) ? count( $this->pointer ) : 0;
}
//This is our array, point on it
$this->pointer[$index] = [];
$this->pointer =& $this->pointer[$index];
//Store the pointer
$this->array_pointers[$this->array_level] =& $this->pointer;
}
/**
* End array
*
* Now it ended...
*
* @todo, according to both levels, point to the nearest one array or object
* @return void
*/
public function endArray()
{
//Point on the last known object
$this->pointer =& $this->object_pointers[$this->object_level];
$this->array_level--;
}
/**
* Called when a key is founded
*
* @param string $key
*
* @return void
*/
public function key($key)
{
$this->_key = $key;
}
/**
* Called when a value is founded
*
* @param mixed $value may be a string, integer, boolean, null
*
*/
public function value($value)
{
if ($this->_key) {
$this->pointer[$this->_key] = $value;
return;
}
$this->pointer[] = $value;
}
public function whitespace($whitespace)
{
}
}
<?php
namespace Fry;
use JsonStreamingParser\Parser;
$testfile = __DIR__.'/example.json';
$listener = new ObjectListener(function($obj) {
var_dump($obj);
});
$stream = fopen($testfile, 'r');
try {
$parser = new Parser($stream, $listener);
$parser->parse();
} catch (Exception $e) {
fclose($stream);
throw $e;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment