Skip to content

Instantly share code, notes, and snippets.

View kusal1990's full-sized avatar

KUSAL BERA kusal1990

6 followers · 1 following
View GitHub Profile
@kusal1990
kusal1990 / html.py
Created June 28, 2022 05:43
# to open/create a new html file in the write mode
f = open('index8051.html', 'w')
# the html code which will go in the file GFG.html
html_template = """<html>
<head>
<title></title>
</head>
<body>
@kusal1990
kusal1990 / app.py
Last active June 28, 2022 05:41
from flask import Flask, jsonify, request
# In[ ]:
import flask
app = Flask(__name__)
@kusal1990
kusal1990 / T5_model.py
Created June 25, 2022 07:41
from transformers import T5ForConditionalGeneration, AdamW
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = T5ForConditionalGeneration.from_pretrained("allenai/unifiedqa-t5-base")
model.cuda()
from transformers import get_linear_schedule_with_warmup
# Parameters:
lr = 1e-4
max_grad_norm = 1.0
@kusal1990
kusal1990 / T5_token.py
Created June 25, 2022 07:30
from transformers import AutoTokenizer, T5ForConditionalGeneration
tokenizer = AutoTokenizer.from_pretrained('t5-small')
def arc_preprocessor(dataset, tokenizer):
'''
This function will convert a given context, question, choices in a format:
input: question \n options \n context </s>
target: label </s>
@kusal1990
kusal1990 / XLNET_MODEL.py
Created June 25, 2022 07:09
from transformers import TFXLNetForMultipleChoice
easy_train_dict = {'input_tokens':easy_train_input_ids,
'attention_mask':easy_train_attention_mask}
viola = tf.data.Dataset.from_tensor_slices((easy_train_dict,tf.keras.utils.to_categorical(easy_train_labels.values)))
viola = viola.shuffle(32).batch(8).cache().prefetch(tf.data.experimental.AUTOTUNE)
easy_dev_dict = {'input_tokens':easy_dev_input_ids,
'attention_mask':easy_dev_attention_mask}
viola_dev = tf.data.Dataset.from_tensor_slices((easy_dev_dict,tf.keras.utils.to_categorical(easy_dev_labels.values, num_classes=5)))
@kusal1990
kusal1990 / XLNet_Tokenization.py
Created June 25, 2022 07:04
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained('xlnet-base-cased', do_lower_case=True)
def arc_preprocessor(dataset, tokenizer):
'''
This function will convert a given article, question, choices in a format:
article <sep> question choices[0] <sep> <cls>
article <sep> question choices[1] <sep> <cls>
article <sep> question choices[2] <sep> <cls>
@kusal1990
kusal1990 / Roberta_model.py
Created June 25, 2022 06:51
pre_trained_model = TFRobertaForMultipleChoice.from_pretrained('roberta-base')
model_input_ids = Input(shape=(5,128,), name='input_tokens', dtype='int32')
masks_input = Input(shape=(5,128,), name='attention_mask', dtype='int32')
x = {'input_ids':model_input_ids,
'attention_mask':masks_input}
x = pre_trained_model(x)['logits']
outputs = Dense(5, activation='softmax')(x)
@kusal1990
kusal1990 / roberta_tokenization.py
Created June 25, 2022 06:47
from transformers import RobertaTokenizer, TFRobertaForMultipleChoice
tokenizer = RobertaTokenizer.from_pretrained('roberta-base', do_lower_case=True)
def arc_preprocessor(dataset, tokenizer):
'''
This function will convert a given article, question, choices in a format:
<s> article </s> </s> question </s> </s> choices[0] </s>
<s> article </s> </s> question </s> </s> choices[1] </s>
<s> article </s> </s> question </s> </s> choices[2] </s>
<s> article </s> </s> question </s> </s> choices[3] </s>
@kusal1990
kusal1990 / BERT_MODEL.py
Created June 25, 2022 06:36
tf.keras.backend.clear_session()
pre_trained_model = TFBertForMultipleChoice.from_pretrained('bert-base-uncased')
model_input_ids = Input(shape=(5,512,), name='input_tokens', dtype='int32')
masks_input = Input(shape=(5,512,), name='attention_mask', dtype='int32')
model_token_type_ids = Input(shape=(5,512,), name='token_type_ids', dtype='int32')
x = {'input_ids':model_input_ids,
'attention_mask':masks_input,
'token_type_ids':model_token_type_ids}
@kusal1990
kusal1990 / bert_model_ tokenizer.py
Created June 25, 2022 06:31
from transformers import BertTokenizer, TFBertForMultipleChoice
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
def arc_preprocessor(dataset, tokenizer):
'''
This function will convert a given context, question, choices in a format:
[CLS] context [SEP] question choices[0] [SEP]
[CLS] context [SEP] question choices[1] [SEP]
[CLS] context [SEP] question choices[2] [SEP]