Note: Not even close to a complete solution (at least at the moment!)
I agree with @DavidW that it is likely better if a single contiguous cython typed memoryview owns all of the data and data is copied into it from your python memoryviews. This is true especially if you plan to only create the giant cython typed memoryview once, but plan on iterating over it many times.
However, you could get a pointer to the contents of your python memoryview by using the PyMemoryView_GET_BUFFER
to get the underlying buffer belonging to that memoryview. Then, you could either memcpy
the data into a larger data structure (for faster copying) or just keep track of an array of pointers, with each element pointing to the data of a memoryview (which is slower during iteration since you would be jumping around memory from memoryview buffer pointer to memoryview buffer pointer).
Here is a way to get the pointer to the underlying data of a python memoryview object. From the cython github's cpython folder, there is no mention of PyMemoryView
, so I had to wrap it manually:
from cpython.object cimport PyObject
cdef extern from "Python.h":
Py_buffer* PyMemoryView_GET_BUFFER(PyObject *mview)
cdef object mv1 = memoryview(b'1234')
cdef Py_buffer* buf = PyMemoryView_GET_BUFFER(<PyObject*>mv1)
cdef char* buf_ptr = <char*>buf.buf
print(buf_ptr)#prints b'1234'
Update 1:
Wasn't 100% sure what the 3D array structure was supposed to look like, so I am just taking the 2D case. Since you said you did not want to introduce C++, I created this array_t
data type that behaves like a vector (well, a pointer to a bunch of void*
). Lots of ugly boilerplate, but here it goes:
from cpython.object cimport PyObject
from libc.stdlib cimport malloc, calloc, realloc, free
from libc.string cimport memcpy, memmove
cdef extern from "Python.h":
Py_buffer* PyMemoryView_GET_BUFFER(PyObject *mview)
cdef char* get_view_ptr(object view):
cdef Py_buffer* py_buf = PyMemoryView_GET_BUFFER(<PyObject*>view)
cdef char* ptr = <char*>py_buf.buf
return ptr
ctypedef struct array_t:
void** data
int max_items
int num_items
cdef void array_init(array_t* array):
array.data = NULL
array.max_items = 0
array.num_items = 0
cdef void array_add(array_t* array, void* item):
if array.max_items == 0:
array.max_items = 10
array.num_items = 0
array.data = <void**>calloc(array.max_items, sizeof(void*))
if array.max_items == array.num_items:
array.max_items *= 2
array.data = <void**>realloc(array.data, array.max_items * sizeof(void*))
array.data[array.num_items] = item
array.num_items += 1
cdef void array_set(array_t* array, int index, void *item):
if index < 0 or index >= array.max_items:
return
array.data[index] = item
cdef void* array_get(array_t* array, int index):
if index < 0 or index >= array.max_items:
return NULL
return array.data[index]
cdef void array_remove(array_t* array, int index):
cdef:
void* src
void* dest
if index < 0 or index >= array.max_items:
return
array.data[index] = NULL
if index+1 != array.max_items:
src = &array.data[index+1]
dest = &array.data[index]
memmove(dest, src, (array.max_items - index) * sizeof(void*))
array.num_items -= 1
cdef void array_free(array_t* array):
free(array.data)
cdef int i
cdef array_t a
cdef object mv1 = memoryview(b'12345')
cdef object mv2 = memoryview(b'67890')
cdef object mv3 = memoryview(b'abcde')
cdef object mv4 = memoryview(b'!@#$%')
array_init(&a)
array_add(&a, get_view_ptr(mv1))
array_add(&a, get_view_ptr(mv2))
array_add(&a, get_view_ptr(mv3))
array_add(&a, get_view_ptr(mv4))
for i in range(a.num_items):
print(i, <char*>array_get(&a, i))