Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
from ijson import common
from ijson.backends import YAJLImportError
from cffi import FFI
ffi = FFI()
ffi.cdef("""
typedef void * (*yajl_malloc_func)(void *ctx, size_t sz);
typedef void (*yajl_free_func)(void *ctx, void * ptr);
typedef void * (*yajl_realloc_func)(void *ctx, void * ptr, size_t sz);
typedef struct
{
yajl_malloc_func malloc;
yajl_realloc_func realloc;
yajl_free_func free;
void * ctx;
} yajl_alloc_funcs;
typedef struct yajl_handle_t * yajl_handle;
typedef enum {
yajl_status_ok,
yajl_status_client_canceled,
yajl_status_error
} yajl_status;
typedef enum {
yajl_allow_comments = 0x01,
yajl_dont_validate_strings = 0x02,
yajl_allow_trailing_garbage = 0x04,
yajl_allow_multiple_values = 0x08,
yajl_allow_partial_values = 0x10
} yajl_option;
typedef struct {
int (* yajl_null)(void * ctx);
int (* yajl_boolean)(void * ctx, int boolVal);
int (* yajl_integer)(void * ctx, long long integerVal);
int (* yajl_double)(void * ctx, double doubleVal);
int (* yajl_number)(void * ctx, const char * numberVal,
size_t numberLen);
int (* yajl_string)(void * ctx, const unsigned char * stringVal,
size_t stringLen);
int (* yajl_start_map)(void * ctx);
int (* yajl_map_key)(void * ctx, const unsigned char * key,
size_t stringLen);
int (* yajl_end_map)(void * ctx);
int (* yajl_start_array)(void * ctx);
int (* yajl_end_array)(void * ctx);
} yajl_callbacks;
int yajl_version(void);
yajl_handle yajl_alloc(const yajl_callbacks *callbacks, yajl_alloc_funcs *afs, void *ctx);
int yajl_config(yajl_handle h, yajl_option opt, ...);
yajl_status yajl_parse(yajl_handle hand, const unsigned char *jsonText, size_t jsonTextLength);
yajl_status yajl_complete_parse(yajl_handle hand);
unsigned char* yajl_get_error(yajl_handle hand, int verbose, const unsigned char *jsonText, size_t jsonTextLength);
void yajl_free_error(yajl_handle hand, unsigned char * str);
void yajl_free(yajl_handle handle);
""")
yajl = ffi.dlopen('yajl')
major, rest = divmod(yajl.yajl_version(), 10000)
minor, micro = divmod(rest, 100)
if major < 2:
raise YAJLImportError('YAJL version %s.x required, found %s.%s.%s' % (2, major, minor, micro))
YAJL_OK = 0
YAJL_CANCELLED = 1
YAJL_INSUFFICIENT_DATA = 2
YAJL_ERROR = 3
# constants defined in yajl_parse.h
YAJL_ALLOW_COMMENTS = 1
YAJL_MULTIPLE_VALUES = 8
def null():
return None
def boolean(val):
return bool(val)
def integer(val):
return int(val)
def double(val):
return float(val)
def number(val, len):
return common.number(ffi.string(val, maxlen=len))
def string(val, len):
return ffi.string(val, maxlen=len)
def start_map():
return None
def map_key(key, len):
return ffi.string(key, maxlen=len)
def end_map():
return None
def start_array():
return None
def end_array():
return None
_callback_data = [
# Mapping of JSON parser events to callback C types and value converters.
# Used to define the Callbacks structure and actual callback functions
# inside the parse function.
('null', 'int(void * ctx)', null),
('boolean', 'int(void * ctx, int boolVal)', boolean),
# "integer" and "double" aren't actually yielded by yajl since "number"
# takes precedence if defined
('integer', 'int(void * ctx, long long integerVal)', integer),
('double', 'int(void * ctx, double doubleVal)', double),
('number', 'int(void * ctx, const char * numberVal, size_t numberLen)', number),
('string', 'int(void * ctx, const unsigned char * stringVal, size_t stringLen)', string),
('start_map', 'int(void * ctx)', start_map),
('map_key', 'int(void * ctx, const unsigned char * key,size_t stringLen)', map_key),
('end_map', 'int(void * ctx)', end_map),
('start_array', 'int(void * ctx)', start_array),
('end_array', 'int(void * ctx)', end_array)
]
def basic_parse(f, allow_comments=False, buf_size=64*1024,
multiple_values=False):
"""
Iterator yielding unprefixed events.
Parameters:
- f: a readable file-like object with JSON input
- allow_comments: tells parser to allow comments in JSON input
- buf_size: a size of an input buffer
- multiple_values: allows the parser to parse multiple JSON objects
"""
events = []
_store = []
def callback(event, func_type, func):
def c_callback(ctx, *args):
events.append((event, func(*args)))
return 1
cb = ffi.callback(func_type, python_callable=c_callback)
_store.append(cb)
return cb
callbacks = [callback(event, type, func) for event, type, func in _callback_data]
c_callbacks = ffi.new('yajl_callbacks*', tuple(callbacks))
handle = yajl.yajl_alloc(c_callbacks, ffi.NULL, ffi.NULL)
if allow_comments:
yajl.yajl_config(handle, YAJL_ALLOW_COMMENTS, 1)
if multiple_values:
yajl.yajl_config(handle, YAJL_MULTIPLE_VALUES, 1)
try:
while True:
buffer = f.read(buf_size)
if buffer:
result = yajl.yajl_parse(handle, buffer, len(buffer))
else:
result = yajl.yajl_complete_parse(handle)
if result != YAJL_OK:
perror = yajl.yajl_get_error(handle, 1, buffer, len(buffer))
error = ffi.string(perror)
yajl.yajl_free_error(handle, perror)
exception = common.IncompleteJSONError if result == YAJL_INSUFFICIENT_DATA else common.JSONError
raise exception(error)
if not buffer and not events:
break
for event in events:
yield event
events = []
finally:
yajl.yajl_free(handle)
def parse(file, **kwargs):
'''
Backend-specific wrapper for ijson.common.parse.
'''
return common.parse(basic_parse(file, **kwargs))
def items(file, prefix):
'''
Backend-specific wrapper for ijson.common.items.
'''
return common.items(parse(file), prefix)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment