Dynamic K-Max pooling in PyTorch (Kalchbrenner et al. 2014)

dynamic_k_max.py

# Based on https://arxiv.org/pdf/1404.2188.pdf
# Anton Melnikov

import torch
from torch import nn
import torch.nn.functional as F
import numpy as np

class DynamicKMaxPooling(nn.Module):
    def __init__(self, k_top, L):
        super().__init__()
        # "L is the total  number  of  convolutional  layers
        # in  the  network;
        # ktop is the fixed pooling parameter for the
        # topmost  convolutional  layer" 
        self.k_top = k_top
        self.L = L
    
    def forward(self, X, l):
        # l is the current convolutional layer
        # X is the input sequence
        # s is the length of the sequence
        # (for conv layers, the length dimension is last)
        s = X.size()[2]
        k_ll = ((self.L - l) / self.L) * s
        k_l = round(max(self.k_top, np.ceil(k_ll)))
        out = F.adaptive_max_pool1d(X, k_l)
        return out

Hi, I was benchmarking this implementation against my own, and found that F.adaptive_max_pool1d produces repeated values.

Example:

import torch
x = torch.rand(3,5,10)
x[0]
tensor([[ 0.5765,  0.7369,  0.7149,  0.3125,  0.4035,  0.1077,  0.9161,
          0.9798,  0.3747,  0.4397],
        [ 0.4552,  0.1228,  0.2114,  0.9486,  0.4408,  0.3796,  0.0478,
          0.9157,  0.3079,  0.7311],
        [ 0.7224,  0.3733,  0.9877,  0.1144,  0.6265,  0.2836,  0.7259,
          0.5716,  0.7938,  0.6956],
        [ 0.5337,  0.2950,  0.0439,  0.7125,  0.4352,  0.0608,  0.1453,
          0.7154,  0.9430,  0.8394],
        [ 0.1790,  0.6305,  0.5095,  0.9600,  0.0667,  0.7244,  0.4464,
          0.0268,  0.4660,  0.6764]])
import torch.nn.functional as F
z = F.adaptive_max_pool1d(x, 3)
z[0]
tensor([[ 0.7369,  0.9161,  0.9798],
        [ **0.9486,  0.9486**,  0.9157],
        [ 0.9877,  0.7259,  0.7938],
        [ **0.7125,  0.7125,**  0.9430],
        [ **0.9600,  0.9600**,  0.6764]])

The second, fourth and fifth rows contain repeats of the maximum value in each sequence, rather than the K largest values in order. I expected the output to be:

tensor([[ 0.7369,  0.9161,  0.9798],
        [ 0.9486,  0.9157,  0.7311],
        [ 0.9877,  0.7259,  0.7938],
        [ 0.7154,  0.9430,  0.8394],
        [ 0.9600,  0.7244,  0.6764]])

https://discuss.pytorch.org/t/resolved-how-to-implement-k-max-pooling-for-cnn-text-classification/931/3

I hope this will help