test_DispCallFunc.cpp

#include <windows.h>
#include <oleauto.h>
#include <conio.h>
#include <stdio.h>

#pragma comment(lib,"OleAut32.lib")

/*
	This is an example of testing OleAut32.DispCallFunc from C with a VB6 ActiveX object.

	It was easier to debug and test from C and removes a translation layer with the VB6 api Declare.
	plus VC has better low level debug tools and control over args.

	You will have to compile your own  Vb6 ActiveX dll ProjectName.ClassName = VBTest.CSample
	Source used in this example is below.

	References:
		DispCallFunc source from Wine: 
			https://github.com/wine-mirror/wine/blob/master/dlls/oleaut32/typelib.c#L6358

		Test func: 
			https://github.com/wine-mirror/wine/blob/master/dlls/oleaut32/tests/tmarshal.c#L3497

	
	C output: vt 0x13 = VT_UI4  
		9759d8
		hr = 0 result.vt=13  ret=0, outVal=99887766
		hr = 0 result.vt=13  ret=0, outVal=11223344
		hr = 0 result.vt=13  ret=0, outVal=55667788

	VTable Dump: 
		[0] IUnknown.QueryInterface
		[4] IUnknown.AddRef
		[8] IUnknown.Release
		[C] IDispatch.GetTypeInfoCount
		[10] IDispatch.GetTypeInfo
		[14] IDispatch.GetIDsOfNamese
		[18] IDispatch.Invoke
		[Get 1C] Public i As Long  [+34] 
		[Let 20] Public i As Long  [+34] 
		[24] - MemID:60030000 - Public Function myFunc(ByVal x As String)  As Long
		[28] - MemID:60030001 - Public Function myFuncRef(x As String)  As Long
		[2C] - CSample.proc_11001D80

	Source:
		Public i As Long

		Public Function myFunc(ByVal x As String) As Long
			MsgBox "myfunc: " & x & " i= " & i
			myFunc = &H11223344
		End Function

		Public Function myFuncRef(ByRef x As String) As Long
			MsgBox "myfuncRef: " & x & " i= " & i
			myFuncRef = &H99887766
		End Function

		Private Function privFunc(ByVal x As String) As Long
			MsgBox "privfunc: " & x & " i= " & i
			privFunc = &H55667788
		End Function

		Private Sub Class_Initialize()
			i = 21
		End Sub

	'asm example of how vb would call this internally..

		set d = new CSample
		d.i = 1 'make sure class is created is ready and our instance var is passed correctly..
		DebugBreak
		d.myFunc "test"

	'	Public Function myFunc(ByRef x As String) As Long
	'    00402637   8B45 E8          MOV EAX,DWORD PTR SS:[EBP-18]
	'    0040263A   8B08             MOV ECX,DWORD PTR DS:[EAX]
	'    0040263C   8D55 E4          LEA EDX,DWORD PTR SS:[EBP-1C]
	'    0040263F   52               PUSH EDX                                 ; retval address
	'    00402640   68 041D4000      PUSH 401D04                              ; UNICODE "test"
	'    00402645   50               PUSH EAX                                 ; this pointer
	'    00402646   8BF0             MOV ESI,EAX
	'    00402648   FF51 24          CALL DWORD PTR DS:[ECX+24]               ; Project1.0040159A

	'for strings, ByVal is the same for the caller, the vb method itself adds a __vbaStrCopy in its own body..

*/

IDispatch* GetVBClass(){
	
	HRESULT	   hr;
	CLSID      clsid;
    	LPOLESTR   p = OLESTR("VBTest.CSample");
    	IDispatch  *IDisp;

     	hr = CoInitialize(NULL);

	 hr = CLSIDFromProgID(p, &clsid);
	 if( hr != S_OK  ){
		 MessageBox(0,"Failed to get Clsid from string\n","",0);
		 return 0;
	 }

	 hr =  CoCreateInstance(clsid, NULL, CLSCTX_INPROC_SERVER, IID_IDispatch, (void**)&IDisp );
	 if ( hr != S_OK )
	 {
	   MessageBox(0,"CoCreate failed","",0);
	   return 0;
	 }

	 return IDisp;
}

void Test_ByRef(IDispatch* IDisp){
	
	int outVal=0;
	int vOff=0x28;
   
	VARIANTARG varresult; //never seeing this used...had to add an extra arg manually for retval
	VariantInit(&varresult);

    	VARIANTARG vararg[2];
	VARIANTARG *rgpvarg[2] = { &vararg[0], &vararg[1] };
	VARTYPE rgvt[] = { VT_BSTR | VT_BYREF, VT_I4 };

	BSTR b = SysAllocString(L"AAAA");
	V_VT(&vararg[0]) = VT_BSTR | VT_BYREF;
    	V_BSTRREF(&vararg[0]) = &b;

	V_VT(&vararg[1]) = VT_I4;
   	 V_I4(&vararg[1]) = (int)&outVal;

    	HRESULT hr = DispCallFunc(IDisp, vOff, CC_STDCALL, VT_UI4, 2, rgvt, rgpvarg, &varresult);
    	printf("hr = %x result.vt=%x  ret=%x, outVal=%x \n", hr, varresult.vt, varresult.ulVal, outVal);

    	VariantClear(&varresult);
	SysFreeString(b);
}

void Test_ByVal(IDispatch* IDisp){
	
	int outVal=0;
	int vOff = 0x24;

    	VARIANTARG varresult; //never seeing this used...had to add an extra arg manually for retval
	VariantInit(&varresult);

    	VARIANTARG vararg[2];
	VARIANTARG *rgpvarg[2] = { &vararg[0], &vararg[1] };
	VARTYPE rgvt[] = { VT_BSTR, VT_I4 };

    	V_VT(&vararg[0]) = VT_BSTR;
	V_BSTR(&vararg[0]) = SysAllocString(L"BBBB");

	V_VT(&vararg[1]) = VT_I4;
    	V_I4(&vararg[1]) = (int)&outVal;

    	HRESULT hr = DispCallFunc(IDisp, vOff, CC_STDCALL, VT_UI4, 2, rgvt, rgpvarg, &varresult);
    	printf("hr = %x result.vt=%x  ret=%x, outVal=%x \n", hr, varresult.vt, varresult.ulVal, outVal);

    	VariantClear(&varresult);
	SysFreeString(vararg[0].bstrVal);
}

void ManualTest_Private_ByVal(IDispatch* IDisp){
	
	int outVal=0;
	int vOff = 0x2C;                             //IDisp = 012E2F38  value @ (110032E4 + 0x2c) = 11003310
	int lpVTableOffset = (*(int*)IDisp) + vOff;  //vtable + 0x2c = 11003310  value @ 11001478  
	int lpMethod = *(int*)lpVTableOffset;        //finally we have actual method address from vtable = 11001478

	//printf("privMethod: %x\n", lpMethod);
	//getch();

    	VARIANTARG varresult; //never seeing this used...had to add an extra arg manually for retval
	VariantInit(&varresult);

    	VARIANTARG vararg[3];
	VARIANTARG *rgpvarg[3] = { &vararg[0], &vararg[1], &vararg[2]};
	VARTYPE rgvt[] = {VT_I4, VT_BSTR, VT_I4 };
	 
	V_VT(&vararg[0]) = VT_I4;
    	V_I4(&vararg[0]) = (int)IDisp;

    	V_VT(&vararg[1]) = VT_BSTR;
	V_BSTR(&vararg[1]) = SysAllocString(L"CCCC");

	V_VT(&vararg[2]) = VT_I4;
    	V_I4(&vararg[2]) = (int)&outVal;

    	HRESULT hr = DispCallFunc(0, lpMethod, CC_STDCALL, VT_UI4, 3, rgvt, rgpvarg, &varresult);
    	printf("hr = %x result.vt=%x  ret=%x, outVal=%x \n", hr, varresult.vt, varresult.ulVal, outVal);

    	VariantClear(&varresult);
	SysFreeString(vararg[1].bstrVal);
}

void main(void){
	
	VARIANTARG varresult;
    	HRESULT hr;
	int vOff=0;

	IDispatch *IDisp = GetVBClass();
	printf("%x\n", IDisp);

	if(IDisp==0){
		getch();
		exit(0);
	}

	Test_ByRef(IDisp);
	Test_ByVal(IDisp);
	ManualTest_Private_ByVal(IDisp); //without using the DispCallFunc com object support feature we manually pass this pointer..

    	getch();
	IDisp->Release();
}	

/*  example from wine tmarshal.c#L3497 source code test...grep source for more..
	static const WCHAR szEmpty[] = { 0 };
    VARTYPE rgvt[] = { VT_R8, VT_BSTR, VT_BSTR, VT_VARIANT|VT_BYREF };
    VARIANTARG vararg[4];
    VARIANTARG varref;
    VARIANTARG *rgpvarg[4] = { &vararg[0], &vararg[1], &vararg[2], &vararg[3] };
    VARIANTARG varresult;
    HRESULT hr;
    IUnknown *pWidget = NULL;
    V_VT(&vararg[0]) = VT_R8;
    V_R8(&vararg[0]) = 3.141;
    V_VT(&vararg[1]) = VT_BSTR;
    V_BSTRREF(&vararg[1]) = (BSTR*)CoTaskMemAlloc(sizeof(BSTR));
    V_VT(&vararg[2]) = VT_BSTR;
    V_BSTR(&vararg[2]) = SysAllocString(szEmpty);
    V_VT(&vararg[3]) = VT_VARIANT|VT_BYREF;
    V_VARIANTREF(&vararg[3]) = &varref;
    V_VT(&varref) = VT_ERROR;
    V_ERROR(&varref) = DISP_E_PARAMNOTFOUND;
    VariantInit(&varresult);
    hr = DispCallFunc(pWidget, 9*sizeof(void*), CC_STDCALL, VT_UI4, 4, rgvt, rgpvarg, &varresult);
    //ok_ole_success(hr, DispCallFunc);
    VariantClear(&varresult);
    SysFreeString(*V_BSTRREF(&vararg[1]));
    CoTaskMemFree(V_BSTRREF(&vararg[1]));
    VariantClear(&vararg[2]);
    //IWidget_Release(pWidget);
*/