View packing.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import torch | |
import numpy as np | |
import logging | |
IMAGE_TOKEN = "<image>" | |
FAKE_TOKEN_AROUND_IMAGE_V2 = "<fake_token_around_image>" | |
_MIN_LENGTH_DOCUMENTS_TO_PACK = ( | |
5 # Minimum lengths of documents to pack together (lenghts is measures in number of tokens) | |
) |
View finetuning.slurm
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
#SBATCH --job-name=idefics_zero3_finetuning_multinode # name | |
#SBATCH --nodes=3 # nodes | |
#SBATCH --ntasks-per-node=1 # crucial - only 1 task per dist per node! | |
#SBATCH --cpus-per-task=96 # number of cores per tasks | |
#SBATCH --gres=gpu:8 # number of gpus | |
#SBATCH --output=%x-%j.out # output file name | |
export GPUS_PER_NODE=8 | |
export MASTER_ADDR=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1) |
View gcp_ubuntu1804_cuda.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Cuda for Ubuntu18.04 | |
CUDA_REPO_PKG=cuda-repo-ubuntu1804_10.1.243-1_amd64.deb | |
wget -O /tmp/${CUDA_REPO_PKG} http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/${CUDA_REPO_PKG} | |
sudo dpkg -i /tmp/${CUDA_REPO_PKG} | |
sudo apt-key adv --fetch-keys http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/7fa2af80.pub | |
rm -f /tmp/${CUDA_REPO_PKG} | |
sudo apt-get update | |
sudo apt-get install cuda-drivers -y | |
sudo apt-get install cuda -y |
View rough_draft.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import torchvision.models as models | |
resnet18 = models.resnet18() | |
from transformers import BertEmbeddings, BertEncoder | |
class MMBDEmbeddings(nn.Module): | |
def __init__(self, | |
text_mod_embds = BertEmbeddings, # Or your favorite bidirectional transformer | |
vision_mod_embds = resnet18): # Or your favorite vision model |
View tensorflow_serving_transformers.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import tensorflow as tf | |
from transformers import BertTokenizer, TFBertForSequenceClassification | |
import numpy as np | |
# seq_length = 128 | |
# nb_examples = 1 | |
# voc_size = 25000 | |
# input_ids = tf.random.uniform((nb_examples,seq_length), | |
# maxval=voc_size, |
View kd.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import torch | |
import torch.nn as nn | |
import torch.nn.functional as F | |
from torch.optim import Optimizer | |
KD_loss = nn.KLDivLoss(reduction='batchmean') | |
def kd_step(teacher: nn.Module, | |
student: nn.Module, | |
temperature: float, |
View wild_eval_bot.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Copyright (c) 2017-present, Moscow Institute of Physics and Technology. | |
# All rights reserved. | |
# This source code is licensed under the BSD-style license found in the | |
# LICENSE file in the root directory of this source tree. An additional grant | |
# of patent rights can be found in the PATENTS file in the same directory. | |
from parlai.core.params import ParlaiParser | |
from parlai.core.agents import Agent | |
from parlai.core.utils import display_messages | |
from projects.convai2.models.ftlm.wild_eval_world import ConvAIWorld |
View relation_extraction.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# coding: utf-8 | |
import logging | |
import math | |
from typing import Any, Dict, List, Optional, Tuple | |
import torch | |
import torch.nn as nn | |
import torch.nn.functional as F | |
from torch.nn.parameter import Parameter, Variable |