Issue when trying to write to memory mapped file, shared between two processes (32 bit -> 64 bit)-CodePudding

As a temp solution I'm implementing a way to get some data via a 32-bit DLL to a 64-bit application.

I'm keeping it as simple as possible, it doesn't need to be high performance or the most beautiful solution. The idea is not to waste too much time on fancy handshaking, etc.

To that end, my 64-bit application loads a 64-bit DLL of my own making (which has the same exported functions as the 32-bit DLL). So, in the main app, nothing changes.

The 64-bit DLL creates a memory-mapped file and starts a 32-bit helper program, by passing the name of the memory-mapped file and some other input data on the command line.

The helper executable loads the 32-bit DLL, calls a DLL function, copies the data into the shared memory-mapped file, exits, and .. Bob's your uncle. That is to say .. Bob is having a bad day , it doesn't work. WriteFile() to the memory-mapped file fails with Windows API error 6 (ERROR_INVALID_HANDLE), as reported by GetLastError().

Everything works except for writing to the memory-mapped file (reading probably fails as well, but haven't been able to get to that yet).

I'm sure the issue is with how I create the memory-mapped file, I'm thoroughly out of my depth here, so I appreciate a fresh pair of eyes.

Relevant code in the 64-bit DLL:

char MemoryFileName[] = "unique_name_with_extra_random_chars";

HANDLE MemoryFile = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, 
PAGE_EXECUTE_READWRITE, 0, (256 * 1024), MemoryFileName) ;

if (MemoryFile)
    {
    void *View = MapViewOfFile(MemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0) ;

    char Exe[] = "Hlpr.exe" ;
    char Cmd[2048] = "Hlpr.exe " ;

    strcat(Cmd, "\"") ;
    strcat(Cmd, MemoryFileName) ;
    strcat(Cmd, "\" \"") ;
    strcat(Cmd, "262144") ; // (256 * 1024)
    strcat(Cmd, "\" \"") ;
    strcat(Cmd, file_name_path_of_data_the_dll_needs_to_parse) ;
    strcat(Cmd, "\"") ;

    STARTUPINFO StartupInfo = {sizeof StartupInfo} ; // Init
    PROCESS_INFORMATION ProcessInfo ;

    if (CreateProcess(Exe, Cmd, NULL, NULL, false, 0, NULL, NULL, 
        &StartupInfo, &ProcessInfo))
        {
        if (WaitForSingleObject(ProcessInfo.hProcess, INFINITE) == WAIT_OBJECT_0)
            {
            DWORD ExitCode ;
            if (GetExitCodeProcess(ProcessInfo.hProcess, &ExitCode))
                {
                Status = (ExitCode >> 24) ; // Feedback from the dll function

                if (Status == DLL_FUNCTION_OK)
                    {
                    DWORD Size = (ExitCode & 0x00FFFFFF) ;

                    if (Size)
                        {
                        BYTE *Buffer = new BYTE[Size] ;
                        Dll_function_return_Data = 
                            (Dll_function_return_Data_Type*)Buffer ;

                        SetFilePointer(MemoryFile, 0, 0, FILE_BEGIN) ;

                        DWORD BytesCopied ;
                        ReadFile(MemoryFile, Buffer, Size, &BytesCopied, NULL) ;

                        if (BytesCopied != Size)
                            Status = MY_ERROR ;
                        }
                    }
                }
            }

        CloseHandle(ProcessInfo.hProcess) ;
        CloseHandle(ProcessInfo.hThread) ;
        }
    UnmapViewOfFile(View) ;
    }

CloseHandle(MemoryFile) ;

Relevant code in the 32-bit Hlpr.exe:

if (ParamCount() >= 3)
    {
    String DllPath = StringReplace(ParamStr(0), L"Hlpr.exe", 
    L"the_32bit_dll.dll", TReplaceFlags() << rfReplaceAll) ;

    DllHandle = LoadLibraryExW(DllPath.c_str(), NULL, 0) ;

    if (DllHandle)
        {
        // Here the dll exported function pointers are collected (GetProcAddress())

        if (dll_function_pointer)
            {
            MemoryFile = OpenFileMapping(FILE_MAP_ALL_ACCESS, false, 
                         ParamStr(1).c_str()) ;

            if (MemoryFile)
                {
                void *View = MapViewOfFile(MemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0);

                DWORD SizeOfMappedFile = ParamStr(2).ToInt() ;

                if (SizeOfMappedFile)
                    {
                    AnsiString ImageFileNameA(ParamStr(3)) ;

                    DWORD Size ;
                    Dll_function_return_Data_Type *Dll_function_return_Data ;

                    Status = dll_function_pointer(ImageFileNameA.c_str(), 
                             Dll_function_return_Data, Size) ; 

                    if (Status == DLL_FUNCTION_OK)
                        {
                        if (SizeOfMappedFile >= Size)
                            {
                            SetFilePointer(MemoryFile, 0, 0, FILE_BEGIN) ;

                            if (WriteFile(MemoryFile, (BYTE*)Dll_function_return_Data,
                                Size, &BytesCopied, NULL)) // FAILS
                                {
                                if (Size != BytesCopied)
                                    Status = MY_ERROR ;
                                }
                            else
                                {
                                LOG("Error: %u", GetLastError()) ; // Returns error 6
                                Status = MY_ERROR ;
                                }
                            }
                        else
                            Status = MY_ERROR ;
                        }
                    }
                UnmapViewOfFile(View) ;
                }
            }
        }
    }

The program exits with this exit code:

return ((Status << 24) | BytesCopied) ;

CodePudding user response：

Based on the comments I realized I misunderstood the use of a FileMapping HANDLE and the purpose of MapViewOfFile().

Instead of using ReadFile() and WriteFile() on the memory-mapped file I can simply use memcpy() on the pointer returned by MapViewOfFile()

Here is the fixed code:

Relevant code in the 64-bit DLL:

char MemoryFileName[] = "unique_name_with_extra_random_chars";

HANDLE MemoryFile = CreateFileMapping(INVALID_HANDLE_VALUE, NULL, 
PAGE_EXECUTE_READWRITE, 0, (256 * 1024), MemoryFileName) ;

if (MemoryFile)
    {
    void *Mem = MapViewOfFile(MemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0) ;

    char Exe[] = "Hlpr.exe" ;
    char Cmd[2048] = "Hlpr.exe " ;

    strcat(Cmd, "\"") ;
    strcat(Cmd, MemoryFileName) ;
    strcat(Cmd, "\" \"") ;
    strcat(Cmd, "262144") ; // (256 * 1024)
    strcat(Cmd, "\" \"") ;
    strcat(Cmd, file_name_path_of_data_the_dll_needs_to_parse) ;
    strcat(Cmd, "\"") ;

    STARTUPINFO StartupInfo = {sizeof StartupInfo} ; // Init
    PROCESS_INFORMATION ProcessInfo ;

    if (Mem && CreateProcess(Exe, Cmd, NULL, NULL, false, 0, NULL, NULL, 
        &StartupInfo, &ProcessInfo))
        {
        if (WaitForSingleObject(ProcessInfo.hProcess, INFINITE) == WAIT_OBJECT_0)
            {
            DWORD ExitCode ;
            if (GetExitCodeProcess(ProcessInfo.hProcess, &ExitCode))
                {
                Status = (ExitCode >> 24) ; // Feedback from the dll function

                if (Status == DLL_FUNCTION_OK)
                    {
                    DWORD Size = (ExitCode & 0x00FFFFFF) ;

                    if (Size)
                        {
                        BYTE *Buffer = new BYTE[Size] ;
                        Dll_function_return_Data = 
                            (Dll_function_return_Data_Type*)Buffer ;

                        memcpy(Buffer, Mem, Size) ;
                        }
                    }
                }
            }

        CloseHandle(ProcessInfo.hProcess) ;
        CloseHandle(ProcessInfo.hThread) ;
        }
    UnmapViewOfFile(Mem) ;
    }

CloseHandle(MemoryFile) ;

Relevant code in the 32-bit Hlpr.exe:

if (ParamCount() >= 3)
    {
    String DllPath = StringReplace(ParamStr(0), L"Hlpr.exe", 
    L"the_32bit_dll.dll", TReplaceFlags() << rfReplaceAll) ;

    DllHandle = LoadLibraryExW(DllPath.c_str(), NULL, 0) ;

    if (DllHandle)
        {
        // Here the dll exported function pointers are collected (GetProcAddress())

        if (dll_function_pointer)
            {
            MemoryFile = OpenFileMapping(FILE_MAP_ALL_ACCESS, false, 
                         ParamStr(1).c_str()) ;

            if (MemoryFile)
                {
                void *Mem = MapViewOfFile(MemoryFile, FILE_MAP_ALL_ACCESS, 0, 0, 0) ;

                DWORD SizeOfMappedFile = ParamStr(2).ToInt() ;

                if (Mem && SizeOfMappedFile)
                    {
                    AnsiString ImageFileNameA(ParamStr(3)) ;

                    DWORD Size ;
                    Dll_function_return_Data_Type *Dll_function_return_Data ;

                    Status = dll_function_pointer(ImageFileNameA.c_str(), 
                             Dll_function_return_Data, Size) ; 

                    if (Status == DLL_FUNCTION_OK)
                        {
                        if (SizeOfMappedFile >= Size)
                            {
                            memcpy(Mem, (BYTE*)Dll_function_return_Data, Size) ; 
                            BytesCopied = Size ;
                            }
                        else
                            Status = MY_ERROR ;
                        }
                    }
                UnmapViewOfFile(Mem) ;
                }
            }
        }
    }

The program exits with this exit code:

return ((Status << 24) | BytesCopied) ;