Skip to content

Instantly share code, notes, and snippets.

View edgartanaka's full-sized avatar
🎵

Edgar Tanaka edgartanaka

🎵
View GitHub Profile
@edgartanaka
edgartanaka / convert_mbart_to_longformer.py
Last active May 18, 2022 10:06
Converting MBart to Longformer
import argparse
import logging
import os
import copy
from transformers import MBart50Tokenizer
from transformers import MBartForConditionalGeneration, AutoTokenizer
# from transformers.modeling_bart import shift_tokens_right
from longformer_encoder_decoder import LongformerSelfAttentionForMBart, LongformerEncoderDecoderConfig
from longformer_encoder_decoder import LongformerEncoderDecoderForConditionalGeneration
import pandas as pd
import urllib.request
from tqdm import tqdm
from multiprocessing.pool import ThreadPool
import os.path
#
# This gist downloads all series from TMDB. You can easily modify this to download all movies.
# It uses threads to parallelize downloads and speed up this process.
# Depends on python 3. Tested on anaconda.