Firstly, I want to hook CreateFile() and rewrite it. Then I want to recode the callstack of my new CreateFile() function. But when I use SymInitialize() to Initialize a handle, it falls into an endless loop. Through my debug, the reason is SymInitialize() invokes CreateFile(). So why does SymInitialize() involve CreateFile()? How to avoid this loop? Is there any alternative method to record callstack information to avoid this loop?

#include <windows.h>
#include <stdio.h>
#include "detours.h"
#include <fstream>
#include <io.h>   

#pragma comment(lib, "detours.lib")
#include <DbgHelp.h>                    //SymInitialize
#pragma comment(lib,"dbghelp.lib")
#define STACK_INFO_LEN  200

struct stackInfo {
    PDWORD hashValue; // hash value to identify same stack
    char* szBriefInfo; // callstack info
};

stackInfo ShowTraceStack(char* szBriefInfo)
{
    static const int MAX_STACK_FRAMES = 12;
    void* pStack[MAX_STACK_FRAMES];
    static char szStackInfo[STACK_INFO_LEN * MAX_STACK_FRAMES];
    static char szFrameInfo[STACK_INFO_LEN];

    HANDLE process = GetCurrentProcess(); // The handle used must be unique to avoid sharing a session with another component,
    SymInitialize(process, NULL, TRUE);
    PDWORD hashValue = (PDWORD)malloc(sizeof(DWORD)); // allow memory for hashVavlue, it will be rewrited in function CaptureStackBackTrace
    WORD frames = CaptureStackBackTrace(0, MAX_STACK_FRAMES, pStack, hashValue);
    //printf("hash value is: %ud \n", &hashValue);
    if (szBriefInfo == NULL) {
        strcpy_s(szStackInfo, "stack traceback:\n");
    }
    else {
        strcpy_s(szStackInfo, szBriefInfo);
    }

    for (WORD i = 0; i < frames;   i) {
        DWORD64 address = (DWORD64)(pStack[i]);

        DWORD64 displacementSym = 0;
        char buffer[sizeof(SYMBOL_INFO)   MAX_SYM_NAME * sizeof(TCHAR)];
        PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)buffer;
        pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO);
        pSymbol->MaxNameLen = MAX_SYM_NAME;

        DWORD displacementLine = 0;
        IMAGEHLP_LINE64 line;
        line.SizeOfStruct = sizeof(IMAGEHLP_LINE64);

        if (SymFromAddr(process, address, &displacementSym, pSymbol) &&
            SymGetLineFromAddr64(process, address, &displacementLine, &line))
        {
            _snprintf_s(szFrameInfo, sizeof(szFrameInfo), "\t%s() at %s:%d(0x%x)\n",
                pSymbol->Name, line.FileName, line.LineNumber, pSymbol->Address);
        }
        else
        {
            _snprintf_s(szFrameInfo, sizeof(szFrameInfo), "\terror: %d\n", GetLastError());
        }
        strcat_s(szStackInfo, szFrameInfo);
    }
    stackInfo traceStackInfo;
    traceStackInfo.hashValue = hashValue;
    traceStackInfo.szBriefInfo = szStackInfo;
    return traceStackInfo;
    //printf("%s", szStackInfo); 
}

HANDLE (*__stdcall oldCreateFile)(LPCWSTR,
    DWORD,
    DWORD,
    LPSECURITY_ATTRIBUTES,
    DWORD,
    DWORD,
    HANDLE) = CreateFileW;

HANDLE WINAPI newCreateFile(
    _In_ LPCWSTR lpFileName,
    _In_ DWORD dwDesiredAccess,
    _In_ DWORD dwShareMode,
    _In_opt_ LPSECURITY_ATTRIBUTES lpSecurityAttributes,
    _In_ DWORD dwCreationDisposition,
    _In_ DWORD dwFlagsAndAttributes,
    _In_opt_ HANDLE hTemplateFile
) {
    ShowTraceStack((char*)"trace information.\n");
    return oldCreateFile(
        L".\\newFiles.txt", // L".\\NewFile.txt",     // Filename
        //lpFileName,
        dwDesiredAccess,          // Desired access
        dwShareMode,        // Share mode
        lpSecurityAttributes,                   // Security attributes
        dwCreationDisposition,             // Creates a new file, only if it doesn't already exist
        dwFlagsAndAttributes,  // Flags and attributes
        NULL);
}

void hook() {
    DetourRestoreAfterWith();
    DetourTransactionBegin();
    DetourUpdateThread(GetCurrentThread());
    DetourAttach(&(PVOID&)oldCreateFile, newCreateFile);
    DetourTransactionCommit();
}

void unhook()
{
    DetourTransactionBegin();
    DetourUpdateThread(GetCurrentThread());
    DetourDetach(&(PVOID&)oldCreateFile, newCreateFile);
    DetourTransactionCommit();
}

void myProcess() {
    HANDLE hFile = CreateFile(TEXT(".\\CreateFileDemo.txt"),     
        GENERIC_WRITE | GENERIC_READ,          
        0,                    
        NULL,                  
        CREATE_ALWAYS,          
        FILE_ATTRIBUTE_NORMAL,         NULL);                 
    if (hFile == INVALID_HANDLE_VALUE)
    {
        OutputDebugString(TEXT("CreateFile fail!\r\n"));
    }
    // write to file 
    const int BUFSIZE = 4096;
    char chBuffer[BUFSIZE];
    memcpy(chBuffer, "Test", 4);
    DWORD dwWritenSize = 0;
    BOOL bRet = WriteFile(hFile, chBuffer, 4, &dwWritenSize, NULL);
    if (bRet) {
        OutputDebugString(TEXT("WriteFile success!\r\n"));
    }
}

int main(){
    hook();
    myProcess();
    unhook();
}

CodePudding user response：

The main problem is the call to SymInitialize where you pass through "TRUE" for fInvadeProcess parameter. This is causing it to SymLoadModuleEx to be called for each loaded module. This will cause a lot of file access to download / create / open PDB files for each loaded module. This is the reason for your infinite loop.

The "quick" fix for this sample is to move the call to SymInitialize into your main before the hook call as it only needs to be called once. This means all the PDB modules are loaded before the hooking / call to ShowTraceStack.

The other problems are:

• dbghelp API is NOT thread safe - so this example will not work in a multi-threaded application
• SymFromAddr may call CreateFile as well for the same reason to load a newly loaded module PDB information - so your hook not passing through the filename will cause PDB information to not work

If you are trying to make someone more useful I would:

• Move SymInitialize to main before the hooking (called only once)
• Only call CaptureStackBackTrace in the hook and queue the thread stack information to be processed at a later time
• Create a separate thread the takes the CaptureStackBackTrace stack information output and convert it to a stack trace - this would is the only thread calling the dbghlp API making calls to dbghlp API thread safe
• In your hook detect when being called from the dbghlp API usage thread and don't do the stack trace and don't modify the CreateFile parameters so you don't get into a infinite loop