Skip to content

Instantly share code, notes, and snippets.

View naturale0's full-sized avatar

Sihyung Park naturale0

View GitHub Profile
import itertools
from torchtext.experimental.datasets import TextClassificationDataset
from torchtext.vocab import build_vocab_from_iterator
from torchtext.experimental.functional import sequential_transforms
from torchtext.experimental.datasets import IMDB
from torchtext.data.utils import get_tokenizer
from torch.nn.utils.rnn import pack_padded_sequence
def build_char_vocab(data, index, bow="<w>", eow="</w>"):