One-Cycle Policy as proposed in fastai library

one_cycle_policy.py

class OneCycleSchedule:

    def __init__(self, t, linear_pct=0.2, eta_max=1.0, eta_min=None,
                 div_factor=100, decay_to_zero=True):

        if eta_min is None:
            eta_min = eta_max / div_factor

        self.t = t
        self.linear_pct = linear_pct
        self.eta_max = eta_max
        self.eta_min = eta_min

        self.t_cosine = int(math.ceil(t * (1 - linear_pct))) + 1
        self.t_linear = int(math.floor(t * linear_pct))

        self.cosine = CosineAnnealingSchedule(
            eta_min=0 if decay_to_zero else eta_min,
            eta_max=eta_max,
            t_max=self.t_cosine, t_mult=1)
        self.linear = lambda x: x * (eta_max - eta_min) / self.t_linear + eta_min

        self.iter = 0

    def update(self, **kwargs):
        self.iter += 1
        if self.iter <= self.t_linear:
            return self.linear(self.iter)
        else:
            return self.cosine.update()