A C dll has:

struct CredPair{
    char usr[127];
    char pas[127];
};

struct CredData{
    enum CredType {
        PAIR,
        KEY
    } credType;

    void* CredVal;
};

struct EndpointData {
    enum EndpointType {
        DIRECT  
    } endpointType;
  
    void* endpointVal;
};

struct EndpointDirect {
    char url[127];
};

I need to call a function in this dll from my C# code, which has the following signature: __declspec(dllexport) MyErrCode CheckUser(const struct CredData* cred_data, const struct EndpointData* endpoint_data);

Here is what I have tried:

I first declared the corresponding types in C# :

public struct CredPair
    {
        public string usr;
        public string pas;
    }
public enum CredType 
    {
        PAIR,
        KEY
    } 

public struct EndpointDirect 
    {
        public string url;
    }

public enum EndpointType
    {
        DIRECT
    }

public struct CredData
    {
        public CredType credType;
        public IntPtr credVal;
    }

public struct EndpointData {
        public EndpointType endpointType;
        public IntPtr endpointVal;
    }

Later declared the function as:

[DllImport("mydll.dll")]
public static extern MyErrCode CheckUser(CredData cred_Data, EndpointData endpoint_data);

and then call the function as:

CredData objCredData = new CredData();
            objCredData.credType = CredType.PAIR;
            CredPair objPair = new CredPair();
            objPair.usr = "[email protected]";
            objPair.pas = "admin@1234";
            IntPtr pnt = Marshal.AllocHGlobal(Marshal.SizeOf(objPair));
            Marshal.StructureToPtr(objPair, pnt, false);
            objCredData.credentialValue = pnt;

            EndpointData objData = new EndpointData ();
            objData.endpointType = EndpointType.DIRECT;
EndpointDirect epd = new EndpointDirect ();
            epd.url = "example.com";
            IntPtr urlptr = Marshal.AllocHGlobal(Marshal.SizeOf(epd));
            Marshal.StructureToPtr(epd, urlptr, false);
    objData.endpointValue = urlptr;

error_code = CheckUser(objCredData, objData);

But looks like parameters are not received correctly. What parameters are wrong here? I believed void* in struct will become IntPtr in C#. The functions also need pointers to structure. Will that also have to be converted to IntPtr?

CodePudding user response：

You have a number of issues with your PInvoke declarations

• You need to make sure to free your HGlobal memory, otherwise it will leak.
• The strings are declared as fixed size char arrays in C, so need to be CharSet.Ansi...
• ... and they need to be declared [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 127)] so they have fixed nested size.
• The parameters to the function need to be ref, possibly with [In] attribute also.
• You need CallingConvention = CallingConvention.Cdecl

[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)]
public struct CredPair
    {
        [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 127)]
        public string usr;
        [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 127)]
        public string pas;
    }

public enum CredType 
    {
        PAIR,
        KEY
    } 

public enum EndpointType
    {
        DIRECT
    }

public struct CredData
    {
        public CredType credType;
        public IntPtr credVal;
    }

[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Ansi)]
public struct EndpointDirect
    {
        [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 127)]
        public string url;
    }

public struct EndpointData
    {
        public EndpointType endpointType;
        public IntPtr endpointVal;
    }

[DllImport("mydll.dll", CallingConvention = CallingConvention.Cdecl)]
public static extern MyErrCode CheckUser([In] ref CredData cred_Data, [In] ref EndpointData endpoint_data);

IntPtr pnt;
IntPtr urlptr;
try
{
    CredPair objPair = new CredPair
    {
        usr = "[email protected]"
        pas = "admin@1234",
    };
    pnt = Marshal.AllocHGlobal(Marshal.SizeOf(objPair));
    Marshal.StructureToPtr(objPair, pnt, false);
    CredData objCredData = new CredData
    {
        credType = CredType.PAIR,
        credVal = pnt,
    };

    endpointValue = new EndpointDirect
    {
        url = "example.com",
    };
    urlptr = Marshal.AllocHGlobal(Marshal.SizeOf(epd));
    Marshal.StructureToPtr(epd, urlptr, false);
    EndpointData objData = new EndpointData
    {
        endpointType = EndpointType.DIRECT,
        endpointValue = urlptr,
    };

    error_code = CheckUser(ref objCredData, ref objData);
}
finally
{
    Marshal.FreeHGlobal(pnt);
    Marshal.FreeHGlobal(urlptr);
}