Skip to content

Instantly share code, notes, and snippets.

@dzzie
Last active January 24, 2023 23:02
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save dzzie/b778ce622342cf1ddc4eead452d5d22c to your computer and use it in GitHub Desktop.
Save dzzie/b778ce622342cf1ddc4eead452d5d22c to your computer and use it in GitHub Desktop.
#include <windows.h>
#include <oleauto.h>
#include <conio.h>
#include <stdio.h>
#pragma comment(lib,"OleAut32.lib")
/*
This is an example of testing OleAut32.DispCallFunc from C with a VB6 ActiveX object.
It was easier to debug and test from C and removes a translation layer with the VB6 api Declare.
plus VC has better low level debug tools and control over args.
You will have to compile your own Vb6 ActiveX dll ProjectName.ClassName = VBTest.CSample
Source used in this example is below.
References:
DispCallFunc source from Wine:
https://github.com/wine-mirror/wine/blob/master/dlls/oleaut32/typelib.c#L6358
Test func:
https://github.com/wine-mirror/wine/blob/master/dlls/oleaut32/tests/tmarshal.c#L3497
C output: vt 0x13 = VT_UI4
9759d8
hr = 0 result.vt=13 ret=0, outVal=99887766
hr = 0 result.vt=13 ret=0, outVal=11223344
hr = 0 result.vt=13 ret=0, outVal=55667788
VTable Dump:
[0] IUnknown.QueryInterface
[4] IUnknown.AddRef
[8] IUnknown.Release
[C] IDispatch.GetTypeInfoCount
[10] IDispatch.GetTypeInfo
[14] IDispatch.GetIDsOfNamese
[18] IDispatch.Invoke
[Get 1C] Public i As Long [+34]
[Let 20] Public i As Long [+34]
[24] - MemID:60030000 - Public Function myFunc(ByVal x As String) As Long
[28] - MemID:60030001 - Public Function myFuncRef(x As String) As Long
[2C] - CSample.proc_11001D80
Source:
Public i As Long
Public Function myFunc(ByVal x As String) As Long
MsgBox "myfunc: " & x & " i= " & i
myFunc = &H11223344
End Function
Public Function myFuncRef(ByRef x As String) As Long
MsgBox "myfuncRef: " & x & " i= " & i
myFuncRef = &H99887766
End Function
Private Function privFunc(ByVal x As String) As Long
MsgBox "privfunc: " & x & " i= " & i
privFunc = &H55667788
End Function
Private Sub Class_Initialize()
i = 21
End Sub
'asm example of how vb would call this internally..
set d = new CSample
d.i = 1 'make sure class is created is ready and our instance var is passed correctly..
DebugBreak
d.myFunc "test"
' Public Function myFunc(ByRef x As String) As Long
' 00402637 8B45 E8 MOV EAX,DWORD PTR SS:[EBP-18]
' 0040263A 8B08 MOV ECX,DWORD PTR DS:[EAX]
' 0040263C 8D55 E4 LEA EDX,DWORD PTR SS:[EBP-1C]
' 0040263F 52 PUSH EDX ; retval address
' 00402640 68 041D4000 PUSH 401D04 ; UNICODE "test"
' 00402645 50 PUSH EAX ; this pointer
' 00402646 8BF0 MOV ESI,EAX
' 00402648 FF51 24 CALL DWORD PTR DS:[ECX+24] ; Project1.0040159A
'for strings, ByVal is the same for the caller, the vb method itself adds a __vbaStrCopy in its own body..
*/
IDispatch* GetVBClass(){
HRESULT hr;
CLSID clsid;
LPOLESTR p = OLESTR("VBTest.CSample");
IDispatch *IDisp;
hr = CoInitialize(NULL);
hr = CLSIDFromProgID(p, &clsid);
if( hr != S_OK ){
MessageBox(0,"Failed to get Clsid from string\n","",0);
return 0;
}
hr = CoCreateInstance(clsid, NULL, CLSCTX_INPROC_SERVER, IID_IDispatch, (void**)&IDisp );
if ( hr != S_OK )
{
MessageBox(0,"CoCreate failed","",0);
return 0;
}
return IDisp;
}
void Test_ByRef(IDispatch* IDisp){
int outVal=0;
int vOff=0x28;
VARIANTARG varresult; //never seeing this used...had to add an extra arg manually for retval
VariantInit(&varresult);
VARIANTARG vararg[2];
VARIANTARG *rgpvarg[2] = { &vararg[0], &vararg[1] };
VARTYPE rgvt[] = { VT_BSTR | VT_BYREF, VT_I4 };
BSTR b = SysAllocString(L"AAAA");
V_VT(&vararg[0]) = VT_BSTR | VT_BYREF;
V_BSTRREF(&vararg[0]) = &b;
V_VT(&vararg[1]) = VT_I4;
V_I4(&vararg[1]) = (int)&outVal;
HRESULT hr = DispCallFunc(IDisp, vOff, CC_STDCALL, VT_UI4, 2, rgvt, rgpvarg, &varresult);
printf("hr = %x result.vt=%x ret=%x, outVal=%x \n", hr, varresult.vt, varresult.ulVal, outVal);
VariantClear(&varresult);
SysFreeString(b);
}
void Test_ByVal(IDispatch* IDisp){
int outVal=0;
int vOff = 0x24;
VARIANTARG varresult; //never seeing this used...had to add an extra arg manually for retval
VariantInit(&varresult);
VARIANTARG vararg[2];
VARIANTARG *rgpvarg[2] = { &vararg[0], &vararg[1] };
VARTYPE rgvt[] = { VT_BSTR, VT_I4 };
V_VT(&vararg[0]) = VT_BSTR;
V_BSTR(&vararg[0]) = SysAllocString(L"BBBB");
V_VT(&vararg[1]) = VT_I4;
V_I4(&vararg[1]) = (int)&outVal;
HRESULT hr = DispCallFunc(IDisp, vOff, CC_STDCALL, VT_UI4, 2, rgvt, rgpvarg, &varresult);
printf("hr = %x result.vt=%x ret=%x, outVal=%x \n", hr, varresult.vt, varresult.ulVal, outVal);
VariantClear(&varresult);
SysFreeString(vararg[0].bstrVal);
}
void ManualTest_Private_ByVal(IDispatch* IDisp){
int outVal=0;
int vOff = 0x2C; //IDisp = 012E2F38 value @ (110032E4 + 0x2c) = 11003310
int lpVTableOffset = (*(int*)IDisp) + vOff; //vtable + 0x2c = 11003310 value @ 11001478
int lpMethod = *(int*)lpVTableOffset; //finally we have actual method address from vtable = 11001478
//printf("privMethod: %x\n", lpMethod);
//getch();
VARIANTARG varresult; //never seeing this used...had to add an extra arg manually for retval
VariantInit(&varresult);
VARIANTARG vararg[3];
VARIANTARG *rgpvarg[3] = { &vararg[0], &vararg[1], &vararg[2]};
VARTYPE rgvt[] = {VT_I4, VT_BSTR, VT_I4 };
V_VT(&vararg[0]) = VT_I4;
V_I4(&vararg[0]) = (int)IDisp;
V_VT(&vararg[1]) = VT_BSTR;
V_BSTR(&vararg[1]) = SysAllocString(L"CCCC");
V_VT(&vararg[2]) = VT_I4;
V_I4(&vararg[2]) = (int)&outVal;
HRESULT hr = DispCallFunc(0, lpMethod, CC_STDCALL, VT_UI4, 3, rgvt, rgpvarg, &varresult);
printf("hr = %x result.vt=%x ret=%x, outVal=%x \n", hr, varresult.vt, varresult.ulVal, outVal);
VariantClear(&varresult);
SysFreeString(vararg[1].bstrVal);
}
void main(void){
VARIANTARG varresult;
HRESULT hr;
int vOff=0;
IDispatch *IDisp = GetVBClass();
printf("%x\n", IDisp);
if(IDisp==0){
getch();
exit(0);
}
Test_ByRef(IDisp);
Test_ByVal(IDisp);
ManualTest_Private_ByVal(IDisp); //without using the DispCallFunc com object support feature we manually pass this pointer..
getch();
IDisp->Release();
}
/* example from wine tmarshal.c#L3497 source code test...grep source for more..
static const WCHAR szEmpty[] = { 0 };
VARTYPE rgvt[] = { VT_R8, VT_BSTR, VT_BSTR, VT_VARIANT|VT_BYREF };
VARIANTARG vararg[4];
VARIANTARG varref;
VARIANTARG *rgpvarg[4] = { &vararg[0], &vararg[1], &vararg[2], &vararg[3] };
VARIANTARG varresult;
HRESULT hr;
IUnknown *pWidget = NULL;
V_VT(&vararg[0]) = VT_R8;
V_R8(&vararg[0]) = 3.141;
V_VT(&vararg[1]) = VT_BSTR;
V_BSTRREF(&vararg[1]) = (BSTR*)CoTaskMemAlloc(sizeof(BSTR));
V_VT(&vararg[2]) = VT_BSTR;
V_BSTR(&vararg[2]) = SysAllocString(szEmpty);
V_VT(&vararg[3]) = VT_VARIANT|VT_BYREF;
V_VARIANTREF(&vararg[3]) = &varref;
V_VT(&varref) = VT_ERROR;
V_ERROR(&varref) = DISP_E_PARAMNOTFOUND;
VariantInit(&varresult);
hr = DispCallFunc(pWidget, 9*sizeof(void*), CC_STDCALL, VT_UI4, 4, rgvt, rgpvarg, &varresult);
//ok_ole_success(hr, DispCallFunc);
VariantClear(&varresult);
SysFreeString(*V_BSTRREF(&vararg[1]));
CoTaskMemFree(V_BSTRREF(&vararg[1]));
VariantClear(&vararg[2]);
//IWidget_Release(pWidget);
*/
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment