nbroad1881

import argparse
from itertools import chain

import evaluate
from datasets import load_dataset
from transformers import AutoTokenizer, AutoModelForMaskedLM, Trainer, TrainingArguments, DataCollatorForLanguageModeling



if __name__ == "__main__":

nbroad1881

import os
import json
from typing import Optional, Any, Union, Dict


import mlflow
from transformers import TrainingArguments

from accelerate.tracking import GeneralTracker
from accelerate.logging import get_logger

nbroad1881

# Generic LM
roberta-base
roberta-large
microsoft/deberta-v3-base
microsoft/deberta-v3-large
microsoft/deberta-v3-xsmall

# Long LM
allenai/longformer-base-4096
google/bigbird-roberta-base

nbroad1881

function SUMMARIZE(input, repo_id="google/pegasus-xsum", use_gpu=false) {

  // other models to consider
  
  // short sequences
  // sshleifer/distilbart-cnn-12-6
  // knkarthick/MEETING_SUMMARY

  // long sequences
  // google/bigbird-pegasus-large-bigpatent

nbroad1881

import torch
from torch import nn


class MultiSampleDropout(nn.Module):
    def __init__(self, dropout_probs, problem_type, num_labels) -> None:
        super().__init__()

        self.dropouts = [nn.Dropout(p=p) for p in dropout_probs]
        self.problem_type = problem_type

nbroad1881

from typing import Any, Optional, Union, Tuple

import torch
from torch import nn
from transformers.activations import ACT2FN
from transformers.models.deberta.modeling_deberta import (
    DebertaPreTrainedModel,
    DebertaModel,
)
from transformers.models.deberta_v2.modeling_deberta_v2 import (

nbroad1881

def reinit_model_weights(model, n_layers, config):

    # use whatever you named your transformer module
    backbone = model.backbone
    
    encoder_layers = backbone.encoder.layer
    reinit_layers(encoder_layers, n_layers, std)

    # use whatever you named the output
    reinit_modules([model.output], std)

nbroad1881

from torch import nn
from transformers import AutoModel

class Model(nn.Module):
    
    def __init__(self, config):
        super().__init__()

        self.model = AutoModel.from_pretrained(...)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

nbroad1881

# full name to abbreviation here: https://gist.github.com/mshafrir/2646763

ste_to_reg = {
        'AA': 'Other',
        'AE': 'Other',
        'AP': 'Other',
        'AK': 'West',
        'AL': 'South',
        'AR': 'South',
        'AS': 'Other',

nbroad1881

# see here https://huggingface.co/docs/datasets/faiss_and_ea.html#adding-a-faiss-index

# I loaded my dataset from a Pandas dataframe
import pandas as pd
df = pd.read_csv("dataset.csv")


from transformers import DPRContextEncoder, DPRContextEncoderTokenizerFast
import torch
torch.set_grad_enabled(False)