I'm trying to load List[np.ndarray] into shared_memory such that other processes can directly access this shared_memory and recover the original List[np.ndarray] without copying List[np.ndarray] into every process. The detailed motivation is related to my previous question: share read-only generic complex python object with int, list of numpy array, tuple, etc. as instance field between multiprocessing

I wrote the following code(python version: 3.8.12, Numpy:1.20.3, MacOS):

encode_nd_arr_list(): given List[np.ndarray], I can get List of share_memory name. decode_nd_arr_list(): given List of share_memory name, I can recover original List[np.ndarray].

from typing import List, Tuple
import numpy as np
from multiprocessing.shared_memory import SharedMemory
from multiprocessing.managers import SharedMemoryManager


def encode_nd_arr_list(
        smm: SharedMemoryManager,
        nd_arr_list: List[np.ndarray]
):

    shm_name_list = []
    shape, dtype = nd_arr_list[0].shape, nd_arr_list[0].dtype
    print(shape)
    print(dtype)
    for nd_arr in nd_arr_list:
        shm = smm.SharedMemory(size=nd_arr.nbytes)
        shm_arr = np.ndarray(shape=shape, dtype=dtype, buffer=shm.buf)
        np.copyto(shm_arr, nd_arr)
        shm_name_list.append(shm.name)
    return shm_name_list, shape, dtype


def decode_nd_arr_list(
        shm_name_list: List[str],
        shape: Tuple[int],
        dtype: np.dtype
):

    nd_array_list = []
    for shm_name in shm_name_list:
        print("----------")
        shm = SharedMemory(shm_name)
        nd_arr = np.ndarray(shape=shape, dtype=dtype, buffer=shm.buf)
        print("nd_arr:", nd_arr)
        nd_array_list.append(nd_arr)
        print("nd_array_list:", nd_array_list)
    return nd_array_list


if __name__ == '__main__':
    arr = np.array([[1, 2, 3], [2, 3, 4], [3, 4, 5]])
    nd_arr_list = [arr, arr   1, arr   2]
    print(nd_arr_list)
    with SharedMemoryManager() as smm:
        shm_name_list, shape, dtype = encode_nd_arr_list(smm, nd_arr_list)
        print(shm_name_list)
        print(shape)
        print(dtype)
        res = decode_nd_arr_list(shm_name_list, shape, dtype)
        print("------------")
        print(res)

However, when I run it in PyCharm, the console shows Process finished with exit code 139 (interrupted by signal 11: SIGSEGV). When I run it in the terminal, it shows segmentation fault, without any error information.

My Question:

1. What's this fault meaning in my case?

2. How can I make my code work? Thanks.

CodePudding user response：

The buffers that are used in each iteration of the loop in the decode_nd_arr_list method get closed after the corresponding SharedMemory object goes out of scope and that causes the segfault. You are essentially trying to access a memory that is no longer valid.

In order to fix it, you can create a custom object that wraps around the ndarray and also stores the SharedMemory to prevent it from going out of scope.

Example:

from typing import List, Tuple
import numpy as np
from multiprocessing.shared_memory import SharedMemory
from multiprocessing.managers import SharedMemoryManager


class SHMArray(np.ndarray):

    def __new__(cls, input_array, shm=None):
        obj = np.asarray(input_array).view(cls)
        obj.shm = shm
        return obj

    def __array_finalize__(self, obj):
        if obj is None: return
        self.shm = getattr(obj, 'shm', None)

def encode_nd_arr_list(
        smm: SharedMemoryManager,
        nd_arr_list: List[np.ndarray]
):

    shm_name_list = []
    shape, dtype = nd_arr_list[0].shape, nd_arr_list[0].dtype
    print(shape)
    print(dtype)
    for nd_arr in nd_arr_list:
        shm = smm.SharedMemory(size=nd_arr.nbytes)
        shm_arr = np.ndarray(shape=shape, dtype=dtype, buffer=shm.buf)
        np.copyto(shm_arr, nd_arr)
        shm_name_list.append(shm.name)
    return shm_name_list, shape, dtype


def decode_nd_arr_list(
        shm_name_list: List[str],
        shape: Tuple[int],
        dtype: np.dtype
):

    nd_array_list = []
    for shm_name in shm_name_list:
        print("----------")
        shm = SharedMemory(shm_name)
        nd_arr = SHMArray(np.ndarray(shape=shape, dtype=dtype, buffer=shm.buf), shm)
        print("nd_arr:", nd_arr)
        nd_array_list.append(nd_arr)
        print("nd_array_list:", nd_array_list)
    return nd_array_list


if __name__ == '__main__':
    arr = np.array([[1, 2, 3], [2, 3, 4], [3, 4, 5]])
    nd_arr_list = [arr, arr   1, arr   2]
    print(nd_arr_list)
    with SharedMemoryManager() as smm:
        shm_name_list, shape, dtype = encode_nd_arr_list(smm, nd_arr_list)
        print(shm_name_list)
        print(shape)
        print(dtype)
        res = decode_nd_arr_list(shm_name_list, shape, dtype)
        print("------------")
        print(res)

Reference: https://github.com/numpy/numpy/issues/18294#issuecomment-771329575