minesh1291

"""
This model integrates the MoE concept within a Transformer architecture. Each token's
representation is processed by a subset of experts, determined by the gating mechanism.
This architecture allows for efficient and specialized handling of different aspects of the
data, aiming for the adaptability and efficiency noted in the Mixtral 8x7B model's design
philosophy. The model activates only a fraction of the available experts for each token,
significantly reducing the computational resources needed compared to activating all experts
for all tokens.
"""

minesh1291

# Works on gluonts dev branch as of May 30th, 2023
# Assumes "m5-forecasting-accuracy" folder with data next to the script
# Data is obtained from https://www.kaggle.com/c/m5-forecasting-accuracy

import pandas as pd
from pathlib import Path
from gluonts.dataset.pandas import PandasDataset

# Load data from csv files

minesh1291

from os import path
from bokeh.models import Button, Div
from bokeh.layouts import column
from bokeh.document import without_document_lock
from bokeh.io import curdoc
from zmq_subprocess import ZmqSubProcessClient


ok_button = Button(label="ok")
div = Div()

minesh1291

How to install CUDA & cuDNN on Ubuntu 22.04

Install NVIDIA drivers

Update & upgrade

sudo apt update && sudo apt upgrade

Remove previous NVIDIA installation

minesh1291

from transformers import Trainer, TrainingArguments

training_args = TrainingArguments(
    output_dir="./logs/model_name",
    logging_dir="./logs/runs",
    overwrite_output_dir=True,
    do_train=True,
    per_device_train_batch_size=32,
    num_train_epochs=1,
    evaluate_during_training=True,

minesh1291

from multimodal_transformers.model import AutoModelWithTabular, TabularConfig
from transformers import AutoConfig

num_labels = len(np.unique(torch_dataset, labels))
config = AutoConfig.from_pretrained('bert-base-uncased')
tabular_config = TabularConfig(
    num_labels=num_labels,
    cat_feat_dim=torch_dataset.cat_feats.shape[1],
    numerical_feat_dim=torch_dataset.numerical_feats.shape[1],
    combine_feat_method='weighted_feature_sum_on_transformer_cat_and_numerical_feats',

minesh1291

import pandas as pd
from multimodal_transformers.data import load_data
from transformers import AutoTokenizer

data_df = pd.read_csv('Womens Clothing E-Commerce Reviews.csv')
text_cols = ['Title', 'Review Text']
# The label col is expected to contain integers from 0 to N_classes - 1
label_col = 'Recommended IND' 
categorical_cols = ['Clothing ID', 'Division Name', 'Department Name', 'Class Name']
numerical_cols = ['Rating', 'Age', 'Positive Feedback Count']

minesh1291

minesh1291

"""
An implementation of the pytorch Subset that returns an instance of the original dataset with a reduced number of items.
This has two benefits:
  - It allows to stil access the attributes of the Dataset class, such as methods, or properties.
  - You can use the usual python index notation with slices to chunk the dataset, rather than creating a list of indices

"""
class Dataset(object):
  def __init__(self, iterable):
    self.items = iterable

minesh1291

from pycocotools.coco import COCO
import numpy as np
import cv2

import os

coco_dataset_path = "/export/public/MS-COCO-2017/"
coco = COCO(coco_dataset_path + "annotations/instances_val2017.json")