fixed.cpp

class A {
  class DecodingState;
  class B;
};

template <class> struct ReturnTypeHelperImp;
template <class S, class R> struct ReturnTypeHelperImp<R(S::*)()> {
        using type = R;
};
template <class T> using ReturnTypeHelper = typename ReturnTypeHelperImp<T>::type;

class A::DecodingState {
  void Enter();
protected:
  template <class S> ReturnTypeHelper<decltype(&S::Enter)> SetState();
};
class A::B : DecodingState {
  void Step();
};
void A::B::Step() { SetState<DecodingState>(); }