cython buffer dtype mismatch on windows

buf_dtype_mismatch.pyx

""" Cyton module to demonstrate buffer dtype mismatch error
tested platforms:
1.
win7-64, Intel i5 core
python 2.7.8 [MSC v.1500 32 bit (Intel)]
mingw32 gcc 4.6.2 / tdm32 gcc 4.8.1 / msvc 9.0 (VS 2008)
numpy 1.8.1 win32
cython 0.20.2 win32

2.
osx 10.9.4, Intel i5 core
python 2.7.5
apple llvm-gcc 5.1-(clang 503.0.40)
numpy 1.8.2
cython 0.20.2
"""

import numpy as np
cimport numpy as np

cdef packed struct Rtg:
  np.float32_t rtg_field_0
  np.float32_t rtg_field_1
  np.float32_t rtg_field_2
  np.float32_t rtg_field_3
  np.uint8_t   rtg_field_4
  np.uint8_t   rtg_field_5
  np.uint8_t   rtg_field_6

rtg = [
  ('rtg_field_0', np.float32),
  ('rtg_field_1', np.float32),
  ('rtg_field_2', np.float32),
  ('rtg_field_3', np.float32),
  ('rtg_field_4', np.uint8),
  ('rtg_field_5', np.uint8),
  ('rtg_field_6', np.uint8),
]

cdef packed struct Gsd:
  np.float32_t gsd_field_0
  np.float32_t gsd_field_1
  np.float32_t gsd_field_2
  np.float32_t gsd_field_3
  np.float32_t gsd_field_4
  np.float32_t gsd_field_5
  np.float32_t gsd_field_6
  np.uint8_t   gsd_field_7
  np.uint8_t   gsd_field_8

gsd = [
  ('gsd_field_0', np.float32),
  ('gsd_field_1', np.float32),
  ('gsd_field_2', np.float32),
  ('gsd_field_3', np.float32),
  ('gsd_field_4', np.float32),
  ('gsd_field_5', np.float32),
  ('gsd_field_6', np.float32),
  ('gsd_field_7', np.uint8),
  ('gsd_field_8', np.uint8),
]

# buffer mismatch error with 2+4n (2,6,10,14,...,26) tg-s
NUM_TG = 26

cdef packed struct Rci:
# Rtg tg[NUM_TG] # cython compilation crashes even if NUM_TG is "cdef unsigned int"
# Rtg tg[26] # compiles with cython 0.20.2, results "Buffer dtype mismatch; next field is at offset 20 but 494 expected"
  Rtg tg_0  # ok
  Rtg tg_1  # err
  Rtg tg_2  # ok
  Rtg tg_3  # ok
  Rtg tg_4  # ok
  Rtg tg_5  # err
  Rtg tg_6  # ok
  Rtg tg_7  # ok
  Rtg tg_8  # ok
  Rtg tg_9  # err
  Rtg tg_10 # ok
  Rtg tg_11 # ok
  Rtg tg_12 # ok
  Rtg tg_13 # err
  Rtg tg_14 # ok
  Rtg tg_15 # ok
  Rtg tg_16 # ok
  Rtg tg_17 # err
  Rtg tg_18 # ok
  Rtg tg_19 # ok
  Rtg tg_20 # ok
  Rtg tg_21 # err
  Rtg tg_22 # ok
  Rtg tg_23 # ok
  Rtg tg_24 # ok
  Rtg tg_25 # err
  Gsd gsd_0

rci = [('tg_%d' %i, rtg) for i in xrange(NUM_TG)]
rci.append( ('gsd', gsd) )
rci_dtype = np.dtype(rci)

# old-style buffer
def proc(np.ndarray[Rci, ndim=1] input):
  cdef unsigned int cycle = input.size
  return cycle

# typed memoryview
def proc_mv(Rci[:] input):
  cdef unsigned int cycle = input.size
  return cycle

# DEBUGGING
# https://groups.google.com/forum/#!msg/cython-users/ytfJDZ_1DAI/3QIJXiSj1REJ

cdef Rci rci_tmp

offsets = [
    <np.intp_t>&(rci_tmp.tg_0)  - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_1)  - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_2)  - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_3)  - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_4)  - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_5)  - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_6)  - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_7)  - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_8)  - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_9)  - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_10) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_11) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_12) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_13) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_14) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_15) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_16) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_17) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_18) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_19) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_20) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_21) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_22) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_23) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_24) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.tg_25) - <np.intp_t>&rci_tmp,
    <np.intp_t>&(rci_tmp.gsd_0) - <np.intp_t>&rci_tmp,
]

# # easier offset calculation in case of "Rtg tg[26]"
# offsets = [ <np.intp_t>&(rci_tmp.tg[i]) - <np.intp_t>&rci_tmp for i in xrange(NUM_TG) ]
# offsets.append( <np.intp_t>&(rci_tmp.gsd_0) - <np.intp_t>&rci_tmp )

cdef get_tg_offsets(Rtg tg):
  tg_offsets = [
    <np.intp_t>&(tg.rtg_field_0) - <np.intp_t>&tg,
    <np.intp_t>&(tg.rtg_field_1) - <np.intp_t>&tg,
    <np.intp_t>&(tg.rtg_field_2) - <np.intp_t>&tg,
    <np.intp_t>&(tg.rtg_field_3) - <np.intp_t>&tg,
    <np.intp_t>&(tg.rtg_field_4) - <np.intp_t>&tg,
    <np.intp_t>&(tg.rtg_field_5) - <np.intp_t>&tg,
    <np.intp_t>&(tg.rtg_field_6) - <np.intp_t>&tg
  ]
  return tg_offsets

offsets_tg0 = get_tg_offsets(rci_tmp.tg_0)
offsets_tg1 = get_tg_offsets(rci_tmp.tg_1)
offsets_tg2 = get_tg_offsets(rci_tmp.tg_2)

setup.py

"""
compile with gcc:
$ python setup.py build_ext --inplace --compiler=mingw32

compile with msvc:
$ set VS90COMNTOOLS=C:\Program Files (x86)\Microsoft Visual Studio 9.0\Common7\Tools
$ python setup.py build_ext --inplace --compiler=msvc
"""

from distutils.core import setup
from Cython.Build import cythonize

import numpy

setup(
    ext_modules = cythonize("buf_dtype_mismatch.pyx"),
    include_dirs=[numpy.get_include()],
)

test.py

""" Test module to produce the error

$ python test.py
...
Traceback (most recent call last):
  File "test.py", line 26, in <module>
    proc(rci)
  File "buf_dtype_mismatch.pyx", line 103, in buf_dtype_mismatch.proc (buf_dtype_mismatch.c:1566)
    def proc(np.ndarray[Rci, ndim=1] input):
ValueError: Buffer dtype mismatch; next field is at offset 20 but 19 expected
"""

import numpy as np

from buf_dtype_mismatch import rci_dtype, proc, proc_mv, offsets, offsets_tg0

def sorted_fields(dtype):
  "sort dtype fields by offset"
  return sorted(dtype.fields.iteritems(), key=lambda (name, (dtype, offset)): offset)

# python dtype offsets
for name, (dtype, offset) in sorted_fields(rci_dtype):
  print name, offset
  if "tg" in name and offset > 20: continue
  for subname, (subdtype, suboffset) in sorted_fields(dtype):
    print "\t", subname, suboffset, subdtype

# cython offsets
print offsets
print offsets_tg0

# run test, both raise same exception
rci = np.zeros(1000, dtype=rci_dtype)
proc(rci)
proc_mv(rci)