ToggleMathZone:
       sub esp,sizeof.CHARFORMAT2W - 4
     pushd sizeof.CHARFORMAT2W

       ; return address and three arguments to SendMessage, plus structure on stack
        label .hWndRE dword at esp+16+sizeof.CHARFORMAT2W

       invoke SendMessage,[.hWndRE],EM_GETCHARFORMAT,SCF_SELECTION or SCF_ONLYCFEFFECTS,esp
        btc [esp+CHARFORMAT2W.dwEffects+3],4 ; CFE_MATH ; test & Toggle math zone flag
  jc .off
     ; Turn on LineServices
      invoke SendMessage,[.hWndRE],EM_SETTYPOGRAPHYOPTIONS,TO_ADVANCEDTYPOGRAPHY,TO_ADVANCEDTYPOGRAPHY
    ; Enable built-in math autocorrect
  invoke SendMessage,[.hWndRE],EM_SETADJUSTTEXTPROC,1,0

   virtual at 0
                du "Cambria Math"
         du LF_FACESIZE dup (0)
              ; create set of data constants
              rept 16 i:0 { load ___data#i dword from 4*i }
         macro SETFACE dest { rept 16 i:0 \{ mov dword[(dest)+4*i],___data\#i \} }
        end virtual
 SETFACE esp+CHARFORMAT2W.szFaceName

     mov [esp+CHARFORMAT2W.lcid],MATH_LCID
       mov [esp+CHARFORMAT2W.dwMask],CFM_MATH or CFM_FACE or CFM_LCID
      invoke SendMessage,[.hWndRE],EM_SETCHARFORMAT,SCF_SELECTION or SCF_ONLYCFEFFECTS,esp
        add esp,sizeof.CHARFORMAT2W
 retn 4

.off: mov [esp+CHARFORMAT2W.dwMask],CFM_MATH
      invoke SendMessage,[.hWndRE],EM_SETCHARFORMAT,SCF_SELECTION or SCF_ONLYCFEFFECTS,esp
        add esp,sizeof.CHARFORMAT2W
 retn 4    

http://research.microsoft.com/workshops/fs2006/presentations/17_Sargent_071706.ppt

You should learn how to use your home page.

.hWndRE is a local label - the first argument on the stack.

Might want to read the blog entry linked to in the first post. Not only does it set the font (special font, btw), but it enables some advanced typography features and autocorrection. Basically, you need a RichEdit control handle.