First, export the QTDIR variable, it should contain the path to your Qt 5 directory:
QTDIR="/Applications/Qt/5.9.3/clang_64"
Now, configure VLC:
PATH="${QTDIR}/bin" \
QT_LIBS="\
-F${QTDIR}/lib \
-framework QtCore \
-framework QtNetwork \
-framework QtGui \
-framework QtWidgets \
-framework QtSvg" \
QT_CFLAGS="\
${QT_LIBS} \
-I${QTDIR}/lib/QtCore.framework/Versions/5/Headers/ \
-I${QTDIR}/lib/QtNetwork.framework/Versions/5/Headers/ \
-I${QTDIR}/lib/QtGui.framework/Versions/5/Headers/ \
-I${QTDIR}/lib/QtWidgets.framework/Versions/5/Headers/ \
-I${QTDIR}/lib/QtSvg.framework/Versions/5/Headers/" \
../extras/package/macosx/configure.sh --disable-nls --enable-qt --host=x86_64-apple-darwin16 --build=x86_64-apple-darwin16 CFLAGS="-g" CXXFLAGS="-g" OBJCFLAGS="-g"