policy.hpp

template<typename DecomposableFunctionType>
class SGD
{
...
...
double Optimize(arma::mat& iterate);
template<typename Policy>
double Optimize(arma::mat& iterate)
...
...
private:
...
}
---------------------------------------------------------------------------------------
Policy Class
---------------------------------------------------------------------------------------
/**
 * Implementation of the NoDecaySGD Policy
 */
class NoDecay
{
  public:

  NoDecay(double stepSize,double decay_rate, int decay_step_size)
  {}

  /**
   * Decay the learning rate based on the policy
   * 
   * @param stepSize: Step size of SGD
   * @param decay_rate: Decay rate for the step size
   * @param decay_step_size: Step size for the step policy
   * @param iteration: Iteration index
   */
  inline double decay_learning_rate(size_t& iteration);
  {
    return stepSize;
  }
};//similarly i have implement 3 other policies
----------------------------------------------------------------------------------
sgd_impl.hpp
----------------------------------------------------------------------------------
template<typename DecomposableFunctionType>
SGD<DecomposableFunctionType>::SGD(DecomposableFunctionType& function,
{
...
template<typename DecomposableFunctionType, DecayType>
double SGD<DecomposableFunctionType>::Optimize(arma::mat& iterate)
{
...
Old Code //Don't change anything here
...
}
double SGD<DecomposableFunctionType>::Optimize<PolicyType>(arma::mat& iterate)
{
PolicyType P;
for(size_t i = .... )
{
  //Iteration loop
  stepSize = P.decay_learning_rate(iteraion)
  iterate -= stepSize  * gradient;
}
...
...
}

This would work indepent of arun reddy momentum work. But would require code duplication