Skip to content

Instantly share code, notes, and snippets.

@erikinfo
Created March 19, 2021 19:06
Show Gist options
  • Save erikinfo/19257669ae46e2be1e30beeb22f4dcfb to your computer and use it in GitHub Desktop.
Save erikinfo/19257669ae46e2be1e30beeb22f4dcfb to your computer and use it in GitHub Desktop.
Requirements Gpt2
import os
import io
import requests
import numpy as np
import pandas as pd
import re
import zipfile
import random
import time
import csv
import datetime
%%time
%%capture
!pip install transformers
!git lfs install # !sudo apt-get install git-lfs (if using Google Colab)
from itertools import compress
from collections import Counter, defaultdict
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from transformers import AutoTokenizer, AutoConfig, AutoModelForPreTraining, \
AdamW, get_linear_schedule_with_warmup, \
TrainingArguments, BeamScorer, \
Trainer, TrainingArguments
import torch
from torch.utils.data import Dataset, random_split, DataLoader, \
RandomSampler, SequentialSampler
from IPython.display import clear_output
print(f"PyTorch version: {torch.__version__}")
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment