==========================================================================================
DLL source: myprocs.asm
==========================================================================================
; DLL creation example

format PE GUI 4.0 DLL
entry DllEntryPoint

include '%include%\win32a.inc'

section '.data' data readable writeable
result db "Return this", 0

section '.code' code readable executable
proc DllEntryPoint, hinstDLL,fdwReason,lpvReserved
  enter
  mov         eax,TRUE
  return

proc ShowErrorMessage, hWnd,dwError
  .lpBuffer dd ?
  enter
  lea         eax,[.lpBuffer]
  invoke  FormatMessage,FORMAT_MESSAGE_ALLOCATE_BUFFER+FORMAT_MESSAGE_FROM_SYSTEM,0,[dwError],LANG_NEUTRAL,eax,0,0
  invoke  MessageBox,[hWnd],[.lpBuffer],NULL,MB_ICONERROR+MB_OK
  invoke  LocalFree,[.lpBuffer]
  return

; VOID ShowLastError(HWND hWnd);
proc ShowLastError, hWnd
  enter
  ;invoke  GetLastError
  mov eax, 21
  stdcall ShowErrorMessage,[hWnd],eax
  return

proc SubTest, lngvalue
  enter
  mov edi, [lngvalue]
  mov eax, 99
  mov [edi+0], eax
  return

proc FunctionTest
  enter
  ;mov edi, [lngvalue]
  ;mov edi, eax
  mov eax, 99
  ;mov [edi+0], eax
  ;mov [edi+0], eax
  ;return
  return


section '.idata' import data readable writeable
  library kernel,'KERNEL32.DLL',\
          user,'USER32.DLL'
  import kernel,\
  GetLastError,'GetLastError',\
    SetLastError,'SetLastError',\
    FormatMessage,'FormatMessageA',\
         LocalFree,'LocalFree'
  import user,\
         MessageBox,'MessageBoxA'

section '.edata' export data readable
  export 'myprocs.dll',\
    FunctionTest,'FunctionTest',\
    ShowErrorMessage,'ShowErrorMessage',\
    ShowLastError,'ShowLastError',\
  SubTest,'SubTest'
section '.reloc' fixups data discardable




==============================================================================================
VB module source: 
==============================================================================================
Private Declare Sub SubTest Lib "myprocs.dll" (vLong As Long)
Private Declare Function FunctionTest Lib "myprocs.dll" () As Long
Sub Main()
   'both result& and dasdf& is a Long (4 byte signed numeric) variable
   
   ' case 1: this works
   ' only 1 call made to myprocs.dll
   SubTest result&
   
   
   ' case 2: this works
   ' only 1 call made to myprocs.dll
   'dasdf& = FunctionTest
   
   
   'case 3: not work  >>  Bad DLL calling convention
   'SubTest result&
   'dasdf& = FunctionTest
End Sub

    

; store dword value 99 at address on stack
SubTest:
   mov eax,[esp+4]
     mov dword [eax], 99
 retn 4


; return dword value
FunctionTest:
 mov eax, 99
 retn    

Do you need to define in VB that a function is stdcall? Maybe you can try to modify your DLL functions to be ccall and see if VB is happy about that.

BTW: Why are there no endp's in your code? Are you using a custom set of macros? Also, enter and return are not used in the current fasm macros. Perhaps you can consider downloading the latest version?

I don't remember what the "&" in VB means? In the first case it appears that an address is being sent to SubTest, and then FunctionTest returns an address? Since your FASM code is consistent, I believe the error lies elsewhere. Yet, try the following code to bypass possible macro problems:

Code:

; store dword value 99 at address on stack
SubTest:
       mov eax,[esp+4]
     mov dword [eax], 99
 retn 4


; return dword value
FunctionTest:
 mov eax, 99
 retn    

Are you able to confirm the operation of each by printing the result? Not just that the code executes, but also that the intended consequences are achieved?

proc SubTest, lngvalue 
  enter
  mov ecx,edi
  mov edi, [lngvalue] 
  mov eax, 99 
  mov [edi+0], eax 
  mov edi,ecx
  return    

If you are gonna use enter then don't you need to use leave otherwise you screw up the stack. Plus the proc macro will do an enter for you anyway and the end of the proc bit will do the leave, so enter is not needed either way.