i want to intercept win32 api CreateFileW, but i meet an error "stack overflow". i don't know what happend, can someone help me?

error: Exception thrown at 0x00007FFA76204170 (KernelBase.dll) in detoursExample.exe: 0xC00000FD: Stack overflow (parameters: 0x0000000000000001, 0x00000094A6A03FF0). Unhandled exception at 0x00007FFA76204170 (KernelBase.dll) in detoursExample.exe: 0xC00000FD: Stack overflow (parameters: 0x0000000000000001, 0x00000094A6A03FF0).

#include <Windows.h>
#include <string>
#include<stdio.h>
#include<stdlib.h>
#include<Windows.h>

#include"detours.h"  
#pragma comment(lib,"detours.lib") 

void myCreateFile(LPCWSTR pathName) {
    // Open a handle to the file
    HANDLE hFile = CreateFileW(
        pathName, // L".\\NewFile.txt",     // Filename
        GENERIC_WRITE,          // Desired access
        FILE_SHARE_READ,        // Share mode
        NULL,                   // Security attributes
        CREATE_NEW,             // Creates a new file, only if it doesn't already exist
        FILE_ATTRIBUTE_NORMAL,  // Flags and attributes
        NULL);                  // Template file handle

    if (hFile == INVALID_HANDLE_VALUE)
    {
        // Failed to open/create file
        throw("failed to open/create file\n");
        return ;
    }

    // Write data to the file
    std::string strText = "Hello World!"; // For C use LPSTR (char*) or LPWSTR (wchar_t*)
    DWORD bytesWritten;
    WriteFile(
        hFile,            // Handle to the file
        strText.c_str(),  // Buffer to write
        strText.size(),   // Buffer size
        &bytesWritten,    // Bytes written
        nullptr);         // Overlapped

     // Close the handle once we don't need it.
    CloseHandle(hFile);
}

                                      
HANDLE (*oldCreateFile)(LPCWSTR,    
     DWORD,
     DWORD,
     LPSECURITY_ATTRIBUTES,
     DWORD,
     DWORD,
     HANDLE) = CreateFileW;

HANDLE WINAPI newCreateFile(
    _In_ LPCWSTR lpFileName,
    _In_ DWORD dwDesiredAccess,
    _In_ DWORD dwShareMode,
    _In_opt_ LPSECURITY_ATTRIBUTES lpSecurityAttributes,
    _In_ DWORD dwCreationDisposition,
    _In_ DWORD dwFlagsAndAttributes,
    _In_opt_ HANDLE hTemplateFile
) {
    printf("hook success!\n");
    return CreateFileW(
        //L".\\newFiles.txt", // L".\\NewFile.txt",     // Filename
        lpFileName,
        dwDesiredAccess,          // Desired access
        dwShareMode,        // Share mode
        lpSecurityAttributes,                   // Security attributes
        dwCreationDisposition,             // Creates a new file, only if it doesn't already exist
        dwFlagsAndAttributes,  // Flags and attributes
        hTemplateFile);
}

void hook() {
    DetourRestoreAfterWith();                              
    DetourTransactionBegin();                             
    DetourUpdateThread(GetCurrentThread());                
    //DetourAttach((void**)&poldsystem, newsystem);         
    DetourAttach((void**)&oldCreateFile, newCreateFile);         
    DetourTransactionCommit();                             
}

int main() {
    hook();
    myCreateFile(L".\\text.txt");
    getchar();
    return 0;

}

CodePudding user response：

In newCreateFile, you need to call oldCreateFile, not CreateFileW. The way you're doing it, your hook ends up calling into itself forever.