check   (mode > 16 & rm = 5) | (mode = 16 & rm = 6)
                        jyes    no_displacement_compression
    

$ ~/fasm/fasm test.asm test1 && hd test1
flat assembler  version 1.73.30  (16384 kilobytes memory)
1 passes, 33 bytes.
00000000  62 f1 f5 48 59 80 01 00  00 00 cc 62 d1 f5 48 59  |b..HY......b..HY|
00000010  85 01 00 00 00 cc 62 f1  f5 48 59 85 01 00 00 00  |......b..HY.....|
00000020  cc                                                |.|
00000021

$ ~/fasm2/fasm2-master/fasm2 test.asm test2 && hd test2
flat assembler  version g.l4gs
test.asm [8]:
        vmulpd zmm0,zmm1,zword[rbp+1]
macro vmulpd? [1] macro AVX_512.basic_instruction_bcst_er [13] AVX_512.store_instruction@src2 [188] (CALM)
Processed: AVX_512.store_instruction@src2 @dest.size,VEX_66_0F_W0,EVEX_W1+EVEX_VL,59h,@dest.mask,@dest.rm,@src.rm
Error: the value of symbol 'AVX_512.store_instruction@src2:compressed_displacement' is not valid for this use.

$ cat test.asm
use64
org 0
vmulpd zmm0,zmm1,zword[rax+1]
int3
vmulpd zmm0,zmm1,zword[r13+1]
int3
vmulpd zmm0,zmm1,zword[rbp+1]
int3
    

seems to me it should be mod != 0 & mod != 3 for mode > 16

vmulpd zmm0,zmm1,rax{1to8}